[autofix.ci] apply automated fixes

# Conflicts:
#	api/tests/unit_tests/core/rag/extractor/test_word_extractor.py

.github/actions/setup-web/action.yml

@@ -0,0 +1,33 @@
+name: Setup Web Environment
+description: Setup pnpm, Node.js, and install web dependencies.
+
+inputs:
+  node-version:
+    description: Node.js version to use
+    required: false
+    default: "22"
+  install-dependencies:
+    description: Whether to install web dependencies after setting up Node.js
+    required: false
+    default: "true"
+
+runs:
+  using: composite
+  steps:
+    - name: Install pnpm
+      uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0
+      with:
+        package_json_file: web/package.json
+        run_install: false
+
+    - name: Setup Node.js
+      uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
+      with:
+        node-version: ${{ inputs.node-version }}
+        cache: pnpm
+        cache-dependency-path: ./web/pnpm-lock.yaml
+
+    - name: Install dependencies
+      if: ${{ inputs.install-dependencies == 'true' }}
+      shell: bash
+      run: pnpm --dir web install --frozen-lockfile

.github/dependabot.yml

@@ -24,14 +24,34 @@ updates:
     schedule:
       interval: "weekly"
     open-pull-requests-limit: 2
+    ignore:
+      - dependency-name: "ky"
+      - dependency-name: "tailwind-merge"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "tailwindcss"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "react-syntax-highlighter"
+        update-types: ["version-update:semver-major"]
+      - dependency-name: "react-window"
+        update-types: ["version-update:semver-major"]
     groups:
+      lexical:
+        patterns:
+          - "lexical"
+          - "@lexical/*"
       storybook:
         patterns:
           - "storybook"
           - "@storybook/*"
+      eslint-group:
+        patterns:
+          - "*eslint*"
       npm-dependencies:
         patterns:
           - "*"
         exclude-patterns:
+          - "lexical"
+          - "@lexical/*"
           - "storybook"
           - "@storybook/*"
+          - "*eslint*"

.github/workflows/anti-slop.yml

@@ -0,0 +1,17 @@
+name: Anti-Slop PR Check
+
+on:
+  pull_request_target:
+    types: [opened, edited, synchronize]
+
+permissions:
+  pull-requests: write
+  contents: read
+
+jobs:
+  anti-slop:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: peakoss/anti-slop@v0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/api-tests.yml

@@ -22,12 +22,12 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
@@ -51,7 +51,7 @@ jobs:
         run: sh .github/workflows/expose_service_ports.sh
 
       - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@v2
+        uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml

.github/workflows/autofix.yml

@@ -12,22 +12,22 @@ jobs:
     if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
 
       - name: Check Docker Compose inputs
         id: docker-compose-changes
-        uses: tj-actions/changed-files@v47
+        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             docker/generate_docker_compose
             docker/.env.example
             docker/docker-compose-template.yaml
             docker/docker-compose.yaml
-      - uses: actions/setup-python@v6
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@v7
+      - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
 
       - name: Generate Docker Compose
         if: steps.docker-compose-changes.outputs.any_changed == 'true'
@@ -84,4 +84,14 @@ jobs:
         run: |
           uvx --python 3.13 mdformat . --exclude ".agents/skills/**"
 
-      - uses: autofix-ci/action@635ffb0c9798bd160680f18fd73371e355b85f27
+      - name: Setup web environment
+        uses: ./.github/actions/setup-web
+        with:
+          node-version: "24"
+
+      - name: ESLint autofix
+        run: |
+          cd web
+          pnpm eslint --concurrency=2 --prune-suppressions
+
+      - uses: autofix-ci/action@7a166d7532b277f34e16238930461bf77f9d7ed8 # v1.3.3

.github/workflows/build-push.yml

@@ -53,26 +53,26 @@ jobs:
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
       - name: Extract metadata for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
         with:
           images: ${{ env[matrix.image_name_env] }}
 
       - name: Build Docker image
         id: build
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
         with:
           context: "{{defaultContext}}:${{ matrix.context }}"
           platforms: ${{ matrix.platform }}
@@ -91,7 +91,7 @@ jobs:
           touch "/tmp/digests/${sanitized_digest}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*
@@ -113,21 +113,21 @@ jobs:
             context: "web"
     steps:
       - name: Download digests
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
         with:
           path: /tmp/digests
           pattern: digests-${{ matrix.context }}-*
           merge-multiple: true
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata for Docker
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
         with:
           images: ${{ env[matrix.image_name_env] }}
           tags: |

.github/workflows/db-migration-test.yml

@@ -13,13 +13,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: true
           python-version: "3.12"
@@ -40,7 +40,7 @@ jobs:
           cp middleware.env.example middleware.env
 
       - name: Set up Middlewares
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
@@ -63,13 +63,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: true
           python-version: "3.12"
@@ -94,7 +94,7 @@ jobs:
           sed -i 's/DB_USERNAME=postgres/DB_USERNAME=mysql/' middleware.env
 
       - name: Set up Middlewares
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml

.github/workflows/deploy-agent-dev.yml

@@ -19,7 +19,7 @@ jobs:
       github.event.workflow_run.head_branch == 'deploy/agent-dev'
     steps:
       - name: Deploy to server
-        uses: appleboy/ssh-action@v1
+        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
         with:
           host: ${{ secrets.AGENT_DEV_SSH_HOST }}
           username: ${{ secrets.SSH_USER }}

.github/workflows/deploy-dev.yml

@@ -16,7 +16,7 @@ jobs:
       github.event.workflow_run.head_branch == 'deploy/dev'
     steps:
       - name: Deploy to server
-        uses: appleboy/ssh-action@v1
+        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
         with:
           host: ${{ secrets.SSH_HOST }}
           username: ${{ secrets.SSH_USER }}

.github/workflows/deploy-hitl.yml

@@ -16,7 +16,7 @@ jobs:
       github.event.workflow_run.head_branch == 'build/feat/hitl'
     steps:
       - name: Deploy to server
-        uses: appleboy/ssh-action@v1
+        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
         with:
           host: ${{ secrets.HITL_SSH_HOST }}
           username: ${{ secrets.SSH_USER }}

.github/workflows/docker-build.yml

@@ -32,13 +32,13 @@ jobs:
             context: "web"
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
       - name: Build Docker Image
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
         with:
           push: false
           context: "{{defaultContext}}:${{ matrix.context }}"

.github/workflows/labeler.yml

@@ -9,6 +9,6 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v6
+      - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
         with:
           sync-labels: true

.github/workflows/main-ci.yml

@@ -27,8 +27,8 @@ jobs:
       vdb-changed: ${{ steps.changes.outputs.vdb }}
       migration-changed: ${{ steps.changes.outputs.migration }}
     steps:
-      - uses: actions/checkout@v6
-      - uses: dorny/paths-filter@v3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: changes
         with:
           filters: |
@@ -39,6 +39,7 @@ jobs:
             web:
               - 'web/**'
               - '.github/workflows/web-tests.yml'
+              - '.github/actions/setup-web/**'
             vdb:
               - 'api/core/rag/datasource/**'
               - 'docker/**'

.github/workflows/pyrefly-diff-comment.yml

@@ -21,7 +21,7 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.pull_requests[0].head.repo.full_name != github.repository }}
     steps:
       - name: Download pyrefly diff artifact
-        uses: actions/github-script@v8
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -49,7 +49,7 @@ jobs:
         run: unzip -o pyrefly_diff.zip
 
       - name: Post comment
-        uses: actions/github-script@v8
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/pyrefly-diff.yml

@@ -17,12 +17,12 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout PR branch
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
 
       - name: Setup Python & UV
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: true
 
@@ -55,7 +55,7 @@ jobs:
           echo ${{ github.event.pull_request.number }} > pr_number.txt
 
       - name: Upload pyrefly diff
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: pyrefly_diff
           path: |
@@ -64,7 +64,7 @@ jobs:
 
       - name: Comment PR with pyrefly diff
         if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-        uses: actions/github-script@v8
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/semantic-pull-request.yml

@@ -16,6 +16,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check title
-        uses: amannn/action-semantic-pull-request@v6.1.1
+        uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -18,7 +18,7 @@ jobs:
       pull-requests: write
 
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
         with:
           days-before-issue-stale: 15
           days-before-issue-close: 3

.github/workflows/style.yml

@@ -19,13 +19,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v47
+        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             api/**
@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup UV and Python
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: false
           python-version: "3.12"
@@ -67,36 +67,22 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v47
+        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             web/**
             .github/workflows/style.yml
+            .github/actions/setup-web/**
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup NodeJS
-        uses: actions/setup-node@v6
+      - name: Setup web environment
         if: steps.changed-files.outputs.any_changed == 'true'
-        with:
-          node-version: 22
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Web dependencies
-        if: steps.changed-files.outputs.any_changed == 'true'
-        working-directory: ./web
-        run: pnpm install --frozen-lockfile
+        uses: ./.github/actions/setup-web
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -134,14 +120,14 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v47
+        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             **.sh
@@ -152,7 +138,7 @@ jobs:
             .editorconfig
 
       - name: Super-linter
-        uses: super-linter/super-linter/slim@v8
+        uses: super-linter/super-linter/slim@61abc07d755095a68f4987d1c2c3d1d64408f1f9 # v8.5.0
         if: steps.changed-files.outputs.any_changed == 'true'
         env:
           BASH_SEVERITY: warning

.github/workflows/tool-test-sdks.yaml

@@ -21,12 +21,12 @@ jobs:
         working-directory: sdks/nodejs-client
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Use Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
         with:
           node-version: 22
           cache: ''

.github/workflows/translate-i18n-claude.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -48,18 +48,10 @@ jobs:
           git config --global user.name "github-actions[bot]"
           git config --global user.email "github-actions[bot]@users.noreply.github.com"
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
+      - name: Setup web environment
+        uses: ./.github/actions/setup-web
         with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: 22
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
+          install-dependencies: "false"
 
       - name: Detect changed files and generate diff
         id: detect_changes
@@ -130,7 +122,7 @@ jobs:
 
       - name: Run Claude Code for Translation Sync
         if: steps.detect_changes.outputs.CHANGED_FILES != ''
-        uses: anthropics/claude-code-action@v1
+        uses: anthropics/claude-code-action@26ec041249acb0a944c0a47b6c0c13f05dbc5b44 # v1.0.70
         with:
           anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/trigger-i18n-sync.yml

@@ -21,7 +21,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
 
@@ -59,7 +59,7 @@ jobs:
 
       - name: Trigger i18n sync workflow
         if: steps.detect.outputs.has_changes == 'true'
-        uses: peter-evans/repository-dispatch@v3
+        uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           event-type: i18n-sync

.github/workflows/vdb-tests.yml

@@ -19,19 +19,19 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Free Disk Space
-        uses: endersonmenezes/free-disk-space@v3
+        uses: endersonmenezes/free-disk-space@7901478139cff6e9d44df5972fd8ab8fcade4db1 # v3.2.2
         with:
           remove_dotnet: true
           remove_haskell: true
           remove_tool_cache: true
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v7
+        uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
@@ -60,7 +60,7 @@ jobs:
 #            tiflash
 
       - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase, OceanBase)
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@4894d2492015c1774ee5a13a95b1072093087ec3 # v2.5.0
         with:
           compose-file: |
             docker/docker-compose.yaml

.github/workflows/web-tests.yml

@@ -26,32 +26,19 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: 22
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+      - name: Setup web environment
+        uses: ./.github/actions/setup-web
 
       - name: Run tests
         run: pnpm vitest run --reporter=blob --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --coverage
 
       - name: Upload blob report
         if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: blob-report-${{ matrix.shardIndex }}
           path: web/.vitest-reports/*
@@ -70,28 +57,15 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: 22
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Install dependencies
-        run: pnpm install --frozen-lockfile
+      - name: Setup web environment
+        uses: ./.github/actions/setup-web
 
       - name: Download blob reports
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
         with:
           path: web/.vitest-reports
           pattern: blob-report-*
@@ -419,7 +393,7 @@ jobs:
 
       - name: Upload Coverage Artifact
         if: steps.coverage-summary.outputs.has_coverage == 'true'
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
         with:
           name: web-coverage-report
           path: web/coverage
@@ -435,36 +409,22 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v6
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v47
+        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
         with:
           files: |
             web/**
             .github/workflows/web-tests.yml
+            .github/actions/setup-web/**
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup NodeJS
-        uses: actions/setup-node@v6
+      - name: Setup web environment
         if: steps.changed-files.outputs.any_changed == 'true'
-        with:
-          node-version: 22
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Web dependencies
-        if: steps.changed-files.outputs.any_changed == 'true'
-        working-directory: ./web
-        run: pnpm install --frozen-lockfile
+        uses: ./.github/actions/setup-web
 
       - name: Web build check
         if: steps.changed-files.outputs.any_changed == 'true'

README.md

@@ -56,7 +56,7 @@
   <a href="./docs/bn-BD/README.md"><img alt="README in বাংলা" src="https://img.shields.io/badge/বাংলা-d9d9d9"></a>
 </p>
 
-Dify is an open-source platform for developing LLM applications. Its intuitive interface combines agentic AI workflows, RAG pipelines, agent capabilities, model management, observability features, and more—allowing you to quickly move from prototype to production.
+Dify is an open-source LLM app development platform. Its intuitive interface combines AI workflow, RAG pipeline, agent capabilities, model management, observability features (including [Opik](https://www.comet.com/docs/opik/integrations/dify), [Langfuse](https://docs.langfuse.com), and [Arize Phoenix](https://docs.arize.com/phoenix)) and more, letting you quickly go from prototype to production. Here's a list of the core features:
 
 ## Quick start
 
@@ -133,7 +133,7 @@ Star Dify on GitHub and be instantly notified of new releases.
 
 ### Custom configurations
 
-If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker-compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
+If you need to customize the configuration, please refer to the comments in our [.env.example](docker/.env.example) file and update the corresponding values in your `.env` file. Additionally, you might need to make adjustments to the `docker-compose.yaml` file itself, such as changing image versions, port mappings, or volume mounts, based on your specific deployment environment and requirements. After making any changes, please re-run `docker compose up -d`. You can find the full list of available environment variables [here](https://docs.dify.ai/getting-started/install-self-hosted/environments).
 
 #### Customizing Suggested Questions
 

api/.importlinter

@@ -44,9 +44,7 @@ forbidden_modules =
 allow_indirect_imports = True
 ignore_imports =
     dify_graph.nodes.agent.agent_node -> extensions.ext_database
-    dify_graph.nodes.llm.file_saver -> extensions.ext_database
     dify_graph.nodes.llm.node -> extensions.ext_database
-    dify_graph.nodes.tool.tool_node -> extensions.ext_database
     dify_graph.model_runtime.model_providers.__base.ai_model -> extensions.ext_redis
     dify_graph.model_runtime.model_providers.model_provider_factory -> extensions.ext_redis
 
@@ -112,9 +110,7 @@ ignore_imports =
     dify_graph.nodes.parameter_extractor.parameter_extractor_node -> core.model_manager
     dify_graph.nodes.question_classifier.question_classifier_node -> core.model_manager
     dify_graph.nodes.tool.tool_node -> core.tools.utils.message_transformer
-    dify_graph.nodes.tool.tool_node -> models
     dify_graph.nodes.agent.agent_node -> models.model
-    dify_graph.nodes.llm.file_saver -> core.helper.ssrf_proxy
     dify_graph.nodes.llm.node -> core.helper.code_executor
     dify_graph.nodes.llm.node -> core.llm_generator.output_parser.errors
     dify_graph.nodes.llm.node -> core.llm_generator.output_parser.structured_output
@@ -135,9 +131,7 @@ ignore_imports =
     dify_graph.nodes.llm.file_saver -> core.tools.tool_file_manager
     dify_graph.nodes.tool.tool_node -> core.tools.errors
     dify_graph.nodes.agent.agent_node -> extensions.ext_database
-    dify_graph.nodes.llm.file_saver -> extensions.ext_database
     dify_graph.nodes.llm.node -> extensions.ext_database
-    dify_graph.nodes.tool.tool_node -> extensions.ext_database
     dify_graph.nodes.agent.agent_node -> models
     dify_graph.nodes.llm.node -> models.model
     dify_graph.nodes.agent.agent_node -> services

api/AGENTS.md

@@ -62,6 +62,22 @@ This is the default standard for backend code in this repo. Follow it for new co
 
 - Code should usually include type annotations that match the repo’s current Python version (avoid untyped public APIs and “mystery” values).
 - Prefer modern typing forms (e.g. `list[str]`, `dict[str, int]`) and avoid `Any` unless there’s a strong reason.
+- For dictionary-like data with known keys and value types, prefer `TypedDict` over `dict[...]` or `Mapping[...]`.
+- For optional keys in typed payloads, use `NotRequired[...]` (or `total=False` when most fields are optional).
+- Keep `dict[...]` / `Mapping[...]` for truly dynamic key spaces where the key set is unknown.
+
+```python
+from datetime import datetime
+from typing import NotRequired, TypedDict
+
+
+class UserProfile(TypedDict):
+    user_id: str
+    email: str
+    created_at: datetime
+    nickname: NotRequired[str]
+```
+
 - For classes, declare member variables at the top of the class body (before `__init__`) so the class shape is obvious at a glance:
 
 ```python

api/commands.py

@@ -30,6 +30,7 @@ from extensions.ext_redis import redis_client
 from extensions.ext_storage import storage
 from extensions.storage.opendal_storage import OpenDALStorage
 from extensions.storage.storage_type import StorageType
+from libs.datetime_utils import naive_utc_now
 from libs.db_migration_lock import DbMigrationAutoRenewLock
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
@@ -2598,15 +2599,29 @@ def migrate_oss(
 @click.option(
     "--start-from",
     type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
-    required=True,
+    required=False,
+    default=None,
     help="Lower bound (inclusive) for created_at.",
 )
 @click.option(
     "--end-before",
     type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
-    required=True,
+    required=False,
+    default=None,
     help="Upper bound (exclusive) for created_at.",
 )
+@click.option(
+    "--from-days-ago",
+    type=int,
+    default=None,
+    help="Relative lower bound in days ago (inclusive). Must be used with --before-days.",
+)
+@click.option(
+    "--before-days",
+    type=int,
+    default=None,
+    help="Relative upper bound in days ago (exclusive). Required for relative mode.",
+)
 @click.option("--batch-size", default=1000, show_default=True, help="Batch size for selecting messages.")
 @click.option(
     "--graceful-period",
@@ -2618,8 +2633,10 @@ def migrate_oss(
 def clean_expired_messages(
     batch_size: int,
     graceful_period: int,
-    start_from: datetime.datetime,
-    end_before: datetime.datetime,
+    start_from: datetime.datetime | None,
+    end_before: datetime.datetime | None,
+    from_days_ago: int | None,
+    before_days: int | None,
     dry_run: bool,
 ):
     """
@@ -2630,18 +2647,70 @@ def clean_expired_messages(
     start_at = time.perf_counter()
 
     try:
+        abs_mode = start_from is not None and end_before is not None
+        rel_mode = before_days is not None
+
+        if abs_mode and rel_mode:
+            raise click.UsageError(
+                "Options are mutually exclusive: use either (--start-from,--end-before) "
+                "or (--from-days-ago,--before-days)."
+            )
+
+        if from_days_ago is not None and before_days is None:
+            raise click.UsageError("--from-days-ago must be used together with --before-days.")
+
+        if (start_from is None) ^ (end_before is None):
+            raise click.UsageError("Both --start-from and --end-before are required when using absolute time range.")
+
+        if not abs_mode and not rel_mode:
+            raise click.UsageError(
+                "You must provide either (--start-from,--end-before) or (--before-days [--from-days-ago])."
+            )
+
+        if rel_mode:
+            assert before_days is not None
+            if before_days < 0:
+                raise click.UsageError("--before-days must be >= 0.")
+            if from_days_ago is not None:
+                if from_days_ago < 0:
+                    raise click.UsageError("--from-days-ago must be >= 0.")
+                if from_days_ago <= before_days:
+                    raise click.UsageError("--from-days-ago must be greater than --before-days.")
+
         # Create policy based on billing configuration
         # NOTE: graceful_period will be ignored when billing is disabled.
         policy = create_message_clean_policy(graceful_period_days=graceful_period)
 
         # Create and run the cleanup service
-        service = MessagesCleanService.from_time_range(
-            policy=policy,
-            start_from=start_from,
-            end_before=end_before,
-            batch_size=batch_size,
-            dry_run=dry_run,
-        )
+        if abs_mode:
+            assert start_from is not None
+            assert end_before is not None
+            service = MessagesCleanService.from_time_range(
+                policy=policy,
+                start_from=start_from,
+                end_before=end_before,
+                batch_size=batch_size,
+                dry_run=dry_run,
+            )
+        elif from_days_ago is None:
+            assert before_days is not None
+            service = MessagesCleanService.from_days(
+                policy=policy,
+                days=before_days,
+                batch_size=batch_size,
+                dry_run=dry_run,
+            )
+        else:
+            assert before_days is not None
+            assert from_days_ago is not None
+            now = naive_utc_now()
+            service = MessagesCleanService.from_time_range(
+                policy=policy,
+                start_from=now - datetime.timedelta(days=from_days_ago),
+                end_before=now - datetime.timedelta(days=before_days),
+                batch_size=batch_size,
+                dry_run=dry_run,
+            )
         stats = service.run()
 
         end_at = time.perf_counter()
@@ -2668,3 +2737,77 @@ def clean_expired_messages(
         raise
 
     click.echo(click.style("messages cleanup completed.", fg="green"))
+
+
+@click.command("export-app-messages", help="Export messages for an app to JSONL.GZ.")
+@click.option("--app-id", required=True, help="Application ID to export messages for.")
+@click.option(
+    "--start-from",
+    type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
+    default=None,
+    help="Optional lower bound (inclusive) for created_at.",
+)
+@click.option(
+    "--end-before",
+    type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
+    required=True,
+    help="Upper bound (exclusive) for created_at.",
+)
+@click.option(
+    "--filename",
+    required=True,
+    help="Base filename (relative path). Do not include suffix like .jsonl.gz.",
+)
+@click.option("--use-cloud-storage", is_flag=True, default=False, help="Upload to cloud storage instead of local file.")
+@click.option("--batch-size", default=1000, show_default=True, help="Batch size for cursor pagination.")
+@click.option("--dry-run", is_flag=True, default=False, help="Scan only, print stats without writing any file.")
+def export_app_messages(
+    app_id: str,
+    start_from: datetime.datetime | None,
+    end_before: datetime.datetime,
+    filename: str,
+    use_cloud_storage: bool,
+    batch_size: int,
+    dry_run: bool,
+):
+    if start_from and start_from >= end_before:
+        raise click.UsageError("--start-from must be before --end-before.")
+
+    from services.retention.conversation.message_export_service import AppMessageExportService
+
+    try:
+        validated_filename = AppMessageExportService.validate_export_filename(filename)
+    except ValueError as e:
+        raise click.BadParameter(str(e), param_hint="--filename") from e
+
+    click.echo(click.style(f"export_app_messages: starting export for app {app_id}.", fg="green"))
+    start_at = time.perf_counter()
+
+    try:
+        service = AppMessageExportService(
+            app_id=app_id,
+            end_before=end_before,
+            filename=validated_filename,
+            start_from=start_from,
+            batch_size=batch_size,
+            use_cloud_storage=use_cloud_storage,
+            dry_run=dry_run,
+        )
+        stats = service.run()
+
+        elapsed = time.perf_counter() - start_at
+        click.echo(
+            click.style(
+                f"export_app_messages: completed in {elapsed:.2f}s\n"
+                f"  - Batches: {stats.batches}\n"
+                f"  - Total messages: {stats.total_messages}\n"
+                f"  - Messages with feedback: {stats.messages_with_feedback}\n"
+                f"  - Total feedbacks: {stats.total_feedbacks}",
+                fg="green",
+            )
+        )
+    except Exception as e:
+        elapsed = time.perf_counter() - start_at
+        logger.exception("export_app_messages failed")
+        click.echo(click.style(f"export_app_messages: failed after {elapsed:.2f}s - {e}", fg="red"))
+        raise

api/configs/enterprise/__init__.py

@@ -18,3 +18,7 @@ class EnterpriseFeatureConfig(BaseSettings):
         description="Allow customization of the enterprise logo.",
         default=False,
     )
+
+    ENTERPRISE_REQUEST_TIMEOUT: int = Field(
+        ge=1, description="Maximum timeout in seconds for enterprise requests", default=5
+    )

api/controllers/console/datasets/datasets.py

@@ -807,7 +807,7 @@ class DatasetApiKeyApi(Resource):
             console_ns.abort(
                 400,
                 message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
-                code="max_keys_exceeded",
+                custom="max_keys_exceeded",
             )
 
         key = ApiToken.generate_api_key(self.token_prefix, 24)

api/controllers/files/tool_files.py

@@ -10,7 +10,6 @@ from controllers.common.file_response import enforce_download_for_html
 from controllers.files import files_ns
 from core.tools.signature import verify_tool_file_signature
 from core.tools.tool_file_manager import ToolFileManager
-from extensions.ext_database import db as global_db
 
 DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
@@ -57,7 +56,7 @@ class ToolFileApi(Resource):
             raise Forbidden("Invalid request.")
 
         try:
-            tool_file_manager = ToolFileManager(engine=global_db.engine)
+            tool_file_manager = ToolFileManager()
             stream, tool_file = tool_file_manager.get_file_generator_by_tool_file_id(
                 file_id,
             )

api/controllers/service_api/dataset/document.py

@@ -1,8 +1,9 @@
 import json
+from contextlib import ExitStack
 from typing import Self
 from uuid import UUID
 
-from flask import request
+from flask import request, send_file
 from flask_restx import marshal
 from pydantic import BaseModel, Field, field_validator, model_validator
 from sqlalchemy import desc, select
@@ -100,6 +101,15 @@ class DocumentListQuery(BaseModel):
     status: str | None = Field(default=None, description="Document status filter")
 
 
+DOCUMENT_BATCH_DOWNLOAD_ZIP_MAX_DOCS = 100
+
+
+class DocumentBatchDownloadZipPayload(BaseModel):
+    """Request payload for bulk downloading uploaded documents as a ZIP archive."""
+
+    document_ids: list[UUID] = Field(..., min_length=1, max_length=DOCUMENT_BATCH_DOWNLOAD_ZIP_MAX_DOCS)
+
+
 register_enum_models(service_api_ns, RetrievalMethod)
 
 register_schema_models(
@@ -109,6 +119,7 @@ register_schema_models(
     DocumentTextCreatePayload,
     DocumentTextUpdate,
     DocumentListQuery,
+    DocumentBatchDownloadZipPayload,
     Rule,
     PreProcessingRule,
     Segmentation,
@@ -540,6 +551,46 @@ class DocumentListApi(DatasetApiResource):
         return response
 
 
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")
+class DocumentBatchDownloadZipApi(DatasetApiResource):
+    """Download multiple uploaded-file documents as a single ZIP archive."""
+
+    @service_api_ns.expect(service_api_ns.models[DocumentBatchDownloadZipPayload.__name__])
+    @service_api_ns.doc("download_documents_as_zip")
+    @service_api_ns.doc(description="Download selected uploaded documents as a single ZIP archive")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "ZIP archive generated successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Document or dataset not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def post(self, tenant_id, dataset_id):
+        payload = DocumentBatchDownloadZipPayload.model_validate(service_api_ns.payload or {})
+
+        upload_files, download_name = DocumentService.prepare_document_batch_download_zip(
+            dataset_id=str(dataset_id),
+            document_ids=[str(document_id) for document_id in payload.document_ids],
+            tenant_id=str(tenant_id),
+            current_user=current_user,
+        )
+
+        with ExitStack() as stack:
+            zip_path = stack.enter_context(FileService.build_upload_files_zip_tempfile(upload_files=upload_files))
+            response = send_file(
+                zip_path,
+                mimetype="application/zip",
+                as_attachment=True,
+                download_name=download_name,
+            )
+            cleanup = stack.pop_all()
+            response.call_on_close(cleanup.close)
+        return response
+
+
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")
 class DocumentIndexingStatusApi(DatasetApiResource):
     @service_api_ns.doc("get_document_indexing_status")
@@ -600,6 +651,35 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         return data
 
 
+@service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/download")
+class DocumentDownloadApi(DatasetApiResource):
+    """Return a signed download URL for a document's original uploaded file."""
+
+    @service_api_ns.doc("get_document_download_url")
+    @service_api_ns.doc(description="Get a signed download URL for a document's original uploaded file")
+    @service_api_ns.doc(params={"dataset_id": "Dataset ID", "document_id": "Document ID"})
+    @service_api_ns.doc(
+        responses={
+            200: "Download URL generated successfully",
+            401: "Unauthorized - invalid API token",
+            403: "Forbidden - insufficient permissions",
+            404: "Document or upload file not found",
+        }
+    )
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def get(self, tenant_id, dataset_id, document_id):
+        dataset = self.get_dataset(str(dataset_id), str(tenant_id))
+        document = DocumentService.get_document(dataset.id, str(document_id))
+
+        if not document:
+            raise NotFound("Document not found.")
+
+        if document.tenant_id != str(tenant_id):
+            raise Forbidden("No permission.")
+
+        return {"url": DocumentService.get_document_download_url(document)}
+
+
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 class DocumentApi(DatasetApiResource):
     METADATA_CHOICES = {"all", "only", "without"}

api/controllers/web/message.py

@@ -239,7 +239,7 @@ class MessageSuggestedQuestionApi(WebApiResource):
     def get(self, app_model, end_user, message_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
-            raise NotCompletionAppError()
+            raise NotChatAppError()
 
         message_id = str(message_id)
 

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -44,14 +44,13 @@ from core.app.entities.task_entities import (
 )
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.app.task_pipeline.message_cycle_manager import MessageCycleManager
+from core.app.task_pipeline.message_file_utils import prepare_file_dict
 from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk
 from core.model_manager import ModelInstance
 from core.ops.entities.trace_entity import TraceTaskName
 from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.prompt.utils.prompt_message_util import PromptMessageUtil
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
-from core.tools.signature import sign_tool_file
-from dify_graph.file import helpers as file_helpers
 from dify_graph.file.enums import FileTransferMethod
 from dify_graph.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from dify_graph.model_runtime.entities.message_entities import (
@@ -460,91 +459,40 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline):
         """
         self._task_state.metadata.usage = self._task_state.llm_result.usage
         metadata_dict = self._task_state.metadata.model_dump()
+
+        # Fetch files associated with this message
+        files = None
+        with Session(db.engine, expire_on_commit=False) as session:
+            message_files = session.scalars(select(MessageFile).where(MessageFile.message_id == self._message_id)).all()
+
+            if message_files:
+                # Fetch all required UploadFile objects in a single query to avoid N+1 problem
+                upload_file_ids = list(
+                    dict.fromkeys(
+                        mf.upload_file_id
+                        for mf in message_files
+                        if mf.transfer_method == FileTransferMethod.LOCAL_FILE and mf.upload_file_id
+                    )
+                )
+                upload_files_map = {}
+                if upload_file_ids:
+                    upload_files = session.scalars(select(UploadFile).where(UploadFile.id.in_(upload_file_ids))).all()
+                    upload_files_map = {uf.id: uf for uf in upload_files}
+
+                files_list = []
+                for message_file in message_files:
+                    file_dict = prepare_file_dict(message_file, upload_files_map)
+                    files_list.append(file_dict)
+
+                files = files_list or None
+
         return MessageEndStreamResponse(
             task_id=self._application_generate_entity.task_id,
             id=self._message_id,
             metadata=metadata_dict,
+            files=files,
         )
 
-    def _record_files(self):
-        with Session(db.engine, expire_on_commit=False) as session:
-            message_files = session.scalars(select(MessageFile).where(MessageFile.message_id == self._message_id)).all()
-            if not message_files:
-                return None
-
-            files_list = []
-            upload_file_ids = [
-                mf.upload_file_id
-                for mf in message_files
-                if mf.transfer_method == FileTransferMethod.LOCAL_FILE and mf.upload_file_id
-            ]
-            upload_files_map = {}
-            if upload_file_ids:
-                upload_files = session.scalars(select(UploadFile).where(UploadFile.id.in_(upload_file_ids))).all()
-                upload_files_map = {uf.id: uf for uf in upload_files}
-
-            for message_file in message_files:
-                upload_file = None
-                if message_file.transfer_method == FileTransferMethod.LOCAL_FILE and message_file.upload_file_id:
-                    upload_file = upload_files_map.get(message_file.upload_file_id)
-
-                url = None
-                filename = "file"
-                mime_type = "application/octet-stream"
-                size = 0
-                extension = ""
-
-                if message_file.transfer_method == FileTransferMethod.REMOTE_URL:
-                    url = message_file.url
-                    if message_file.url:
-                        filename = message_file.url.split("/")[-1].split("?")[0]  # Remove query params
-                elif message_file.transfer_method == FileTransferMethod.LOCAL_FILE:
-                    if upload_file:
-                        url = file_helpers.get_signed_file_url(upload_file_id=str(upload_file.id))
-                        filename = upload_file.name
-                        mime_type = upload_file.mime_type or "application/octet-stream"
-                        size = upload_file.size or 0
-                        extension = f".{upload_file.extension}" if upload_file.extension else ""
-                    elif message_file.upload_file_id:
-                        # Fallback: generate URL even if upload_file not found
-                        url = file_helpers.get_signed_file_url(upload_file_id=str(message_file.upload_file_id))
-                elif message_file.transfer_method == FileTransferMethod.TOOL_FILE and message_file.url:
-                    # For tool files, use URL directly if it's HTTP, otherwise sign it
-                    if message_file.url.startswith("http"):
-                        url = message_file.url
-                        filename = message_file.url.split("/")[-1].split("?")[0]
-                    else:
-                        # Extract tool file id and extension from URL
-                        url_parts = message_file.url.split("/")
-                        if url_parts:
-                            file_part = url_parts[-1].split("?")[0]  # Remove query params first
-                            # Use rsplit to correctly handle filenames with multiple dots
-                            if "." in file_part:
-                                tool_file_id, ext = file_part.rsplit(".", 1)
-                                extension = f".{ext}"
-                            else:
-                                tool_file_id = file_part
-                                extension = ".bin"
-                            url = sign_tool_file(tool_file_id=tool_file_id, extension=extension)
-                            filename = file_part
-
-                transfer_method_value = message_file.transfer_method
-                remote_url = message_file.url if message_file.transfer_method == FileTransferMethod.REMOTE_URL else ""
-                file_dict = {
-                    "related_id": message_file.id,
-                    "extension": extension,
-                    "filename": filename,
-                    "size": size,
-                    "mime_type": mime_type,
-                    "transfer_method": transfer_method_value,
-                    "type": message_file.type,
-                    "url": url or "",
-                    "upload_file_id": message_file.upload_file_id or message_file.id,
-                    "remote_url": remote_url,
-                }
-                files_list.append(file_dict)
-            return files_list or None
-
     def _agent_message_to_stream_response(self, answer: str, message_id: str) -> AgentMessageStreamResponse:
         """
         Agent message to stream response.

api/core/app/task_pipeline/message_cycle_manager.py

@@ -1,7 +1,6 @@
 import hashlib
 import logging
-import time
-from threading import Thread
+from threading import Thread, Timer
 from typing import Union
 
 from flask import Flask, current_app
@@ -96,9 +95,9 @@ class MessageCycleManager:
         if auto_generate_conversation_name and is_first_message:
             # start generate thread
             # time.sleep not block other logic
-            time.sleep(1)
-            thread = Thread(
-                target=self._generate_conversation_name_worker,
+            thread = Timer(
+                1,
+                self._generate_conversation_name_worker,
                 kwargs={
                     "flask_app": current_app._get_current_object(),  # type: ignore
                     "conversation_id": conversation_id,

api/core/app/task_pipeline/message_file_utils.py

@@ -0,0 +1,76 @@
+from core.tools.signature import sign_tool_file
+from dify_graph.file import helpers as file_helpers
+from dify_graph.file.enums import FileTransferMethod
+from models.model import MessageFile, UploadFile
+
+MAX_TOOL_FILE_EXTENSION_LENGTH = 10
+
+
+def prepare_file_dict(message_file: MessageFile, upload_files_map: dict[str, UploadFile]) -> dict:
+    """
+    Prepare file dictionary for message end stream response.
+
+    :param message_file: MessageFile instance
+    :param upload_files_map: Dictionary mapping upload_file_id to UploadFile
+    :return: Dictionary containing file information
+    """
+    upload_file = None
+    if message_file.transfer_method == FileTransferMethod.LOCAL_FILE and message_file.upload_file_id:
+        upload_file = upload_files_map.get(message_file.upload_file_id)
+
+    url = None
+    filename = "file"
+    mime_type = "application/octet-stream"
+    size = 0
+    extension = ""
+
+    if message_file.transfer_method == FileTransferMethod.REMOTE_URL:
+        url = message_file.url
+        if message_file.url:
+            filename = message_file.url.split("/")[-1].split("?")[0]
+            if "." in filename:
+                extension = "." + filename.rsplit(".", 1)[1]
+    elif message_file.transfer_method == FileTransferMethod.LOCAL_FILE:
+        if upload_file:
+            url = file_helpers.get_signed_file_url(upload_file_id=str(upload_file.id))
+            filename = upload_file.name
+            mime_type = upload_file.mime_type or "application/octet-stream"
+            size = upload_file.size or 0
+            extension = f".{upload_file.extension}" if upload_file.extension else ""
+        elif message_file.upload_file_id:
+            url = file_helpers.get_signed_file_url(upload_file_id=str(message_file.upload_file_id))
+    elif message_file.transfer_method == FileTransferMethod.TOOL_FILE and message_file.url:
+        if message_file.url.startswith(("http://", "https://")):
+            url = message_file.url
+            filename = message_file.url.split("/")[-1].split("?")[0]
+            if "." in filename:
+                extension = "." + filename.rsplit(".", 1)[1]
+        else:
+            url_parts = message_file.url.split("/")
+            if url_parts:
+                file_part = url_parts[-1].split("?")[0]
+                if "." in file_part:
+                    tool_file_id, ext = file_part.rsplit(".", 1)
+                    extension = f".{ext}"
+                    if len(extension) > MAX_TOOL_FILE_EXTENSION_LENGTH:
+                        extension = ".bin"
+                else:
+                    tool_file_id = file_part
+                    extension = ".bin"
+                url = sign_tool_file(tool_file_id=tool_file_id, extension=extension)
+                filename = file_part
+
+    transfer_method_value = message_file.transfer_method.value
+    remote_url = message_file.url if message_file.transfer_method == FileTransferMethod.REMOTE_URL else ""
+    return {
+        "related_id": message_file.id,
+        "extension": extension,
+        "filename": filename,
+        "size": size,
+        "mime_type": mime_type,
+        "transfer_method": transfer_method_value,
+        "type": message_file.type,
+        "url": url or "",
+        "upload_file_id": message_file.upload_file_id or message_file.id,
+        "remote_url": remote_url,
+    }

api/core/ops/opik_trace/opik_trace.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 import os
 import uuid
@@ -46,6 +47,22 @@ def wrap_metadata(metadata, **kwargs):
     return metadata
 
 
+def _seed_to_uuid4(seed: str) -> str:
+    """Derive a deterministic UUID4-formatted string from an arbitrary seed.
+
+    uuid4_to_uuid7 requires a valid UUID v4 string, but some Dify identifiers
+    are not UUIDs (e.g. a workflow_run_id with a "-root" suffix appended to
+    distinguish the root span from the trace).  This helper hashes the seed
+    with MD5 and patches the version/variant bits so the result satisfies the
+    UUID v4 contract.
+    """
+    raw = hashlib.md5(seed.encode()).digest()
+    ba = bytearray(raw)
+    ba[6] = (ba[6] & 0x0F) | 0x40  # version 4
+    ba[8] = (ba[8] & 0x3F) | 0x80  # variant 1
+    return str(uuid.UUID(bytes=bytes(ba)))
+
+
 def prepare_opik_uuid(user_datetime: datetime | None, user_uuid: str | None):
     """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most
     messages and objects. The type-hints of BaseTraceInfo indicates that
@@ -95,60 +112,52 @@ class OpikDataTrace(BaseTraceInstance):
             self.generate_name_trace(trace_info)
 
     def workflow_trace(self, trace_info: WorkflowTraceInfo):
-        dify_trace_id = trace_info.trace_id or trace_info.workflow_run_id
-        opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id)
         workflow_metadata = wrap_metadata(
             trace_info.metadata, message_id=trace_info.message_id, workflow_app_log_id=trace_info.workflow_app_log_id
         )
-        root_span_id = None
 
         if trace_info.message_id:
             dify_trace_id = trace_info.trace_id or trace_info.message_id
-            opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id)
-
-            trace_data = {
-                "id": opik_trace_id,
-                "name": TraceTaskName.MESSAGE_TRACE,
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "thread_id": trace_info.conversation_id,
-                "tags": ["message", "workflow"],
-                "project_name": self.project,
-            }
-            self.add_trace(trace_data)
-
-            root_span_id = prepare_opik_uuid(trace_info.start_time, trace_info.workflow_run_id)
-            span_data = {
-                "id": root_span_id,
-                "parent_span_id": None,
-                "trace_id": opik_trace_id,
-                "name": TraceTaskName.WORKFLOW_TRACE,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "tags": ["workflow"],
-                "project_name": self.project,
-            }
-            self.add_span(span_data)
+            trace_name = TraceTaskName.MESSAGE_TRACE
+            trace_tags = ["message", "workflow"]
+            root_span_seed = trace_info.workflow_run_id
         else:
-            trace_data = {
-                "id": opik_trace_id,
-                "name": TraceTaskName.MESSAGE_TRACE,
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "thread_id": trace_info.conversation_id,
-                "tags": ["workflow"],
-                "project_name": self.project,
-            }
-            self.add_trace(trace_data)
+            dify_trace_id = trace_info.trace_id or trace_info.workflow_run_id
+            trace_name = TraceTaskName.WORKFLOW_TRACE
+            trace_tags = ["workflow"]
+            root_span_seed = _seed_to_uuid4(trace_info.workflow_run_id + "-root")
+
+        opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id)
+
+        trace_data = {
+            "id": opik_trace_id,
+            "name": trace_name,
+            "start_time": trace_info.start_time,
+            "end_time": trace_info.end_time,
+            "metadata": workflow_metadata,
+            "input": wrap_dict("input", trace_info.workflow_run_inputs),
+            "output": wrap_dict("output", trace_info.workflow_run_outputs),
+            "thread_id": trace_info.conversation_id,
+            "tags": trace_tags,
+            "project_name": self.project,
+        }
+        self.add_trace(trace_data)
+
+        root_span_id = prepare_opik_uuid(trace_info.start_time, root_span_seed)
+        span_data = {
+            "id": root_span_id,
+            "parent_span_id": None,
+            "trace_id": opik_trace_id,
+            "name": TraceTaskName.WORKFLOW_TRACE,
+            "input": wrap_dict("input", trace_info.workflow_run_inputs),
+            "output": wrap_dict("output", trace_info.workflow_run_outputs),
+            "start_time": trace_info.start_time,
+            "end_time": trace_info.end_time,
+            "metadata": workflow_metadata,
+            "tags": ["workflow"],
+            "project_name": self.project,
+        }
+        self.add_span(span_data)
 
         # through workflow_run_id get all_nodes_execution using repository
         session_factory = sessionmaker(bind=db.engine)
@@ -231,15 +240,13 @@ class OpikDataTrace(BaseTraceInstance):
             else:
                 run_type = "tool"
 
-            parent_span_id = trace_info.workflow_app_log_id or trace_info.workflow_run_id
-
             if not total_tokens:
                 total_tokens = execution_metadata.get(WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS) or 0
 
             span_data = {
                 "trace_id": opik_trace_id,
                 "id": prepare_opik_uuid(created_at, node_execution_id),
-                "parent_span_id": prepare_opik_uuid(trace_info.start_time, parent_span_id),
+                "parent_span_id": root_span_id,
                 "name": node_name,
                 "type": run_type,
                 "start_time": created_at,

api/core/rag/extractor/pdf_extractor.py

@@ -114,7 +114,6 @@ class PdfExtractor(BaseExtractor):
         """
         image_content = []
         upload_files = []
-        base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
 
         try:
             image_objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))
@@ -164,7 +163,7 @@ class PdfExtractor(BaseExtractor):
                         used_at=naive_utc_now(),
                     )
                     upload_files.append(upload_file)
-                    image_content.append(f"![image]({base_url}/files/{upload_file.id}/file-preview)")
+                    image_content.append(f"![image](/files/{upload_file.id}/file-preview)")
                 except Exception as e:
                     logger.warning("Failed to extract image from PDF: %s", e)
                     continue

api/core/rag/extractor/word_extractor.py

@@ -87,7 +87,6 @@ class WordExtractor(BaseExtractor):
     def _extract_images_from_docx(self, doc):
         image_count = 0
         image_map = {}
-        base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
 
         for r_id, rel in doc.part.rels.items():
             if "image" in rel.target_ref:
@@ -126,7 +125,7 @@ class WordExtractor(BaseExtractor):
                             used_at=naive_utc_now(),
                         )
                         db.session.add(upload_file)
-                        image_map[r_id] = f"![image]({base_url}/files/{upload_file.id}/file-preview)"
+                        image_map[r_id] = f"![image](/files/{upload_file.id}/file-preview)"
                 else:
                     image_ext = rel.target_ref.split(".")[-1]
                     if image_ext is None:
@@ -154,7 +153,7 @@ class WordExtractor(BaseExtractor):
                         used_at=naive_utc_now(),
                     )
                     db.session.add(upload_file)
-                    image_map[rel.target_part] = f"![image]({base_url}/files/{upload_file.id}/file-preview)"
+                    image_map[rel.target_part] = f"![image](/files/{upload_file.id}/file-preview)"
         db.session.commit()
         return image_map
 

api/core/rag/retrieval/dataset_retrieval.py

@@ -57,7 +57,7 @@ from core.rag.retrieval.template_prompts import (
 from core.tools.signature import sign_upload_file
 from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
 from dify_graph.file import File, FileTransferMethod, FileType
-from dify_graph.model_runtime.entities.llm_entities import LLMResult, LLMUsage
+from dify_graph.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMUsage
 from dify_graph.model_runtime.entities.message_entities import PromptMessage, PromptMessageRole, PromptMessageTool
 from dify_graph.model_runtime.entities.model_entities import ModelFeature, ModelType
 from dify_graph.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
@@ -127,11 +127,12 @@ class DatasetRetrieval:
         metadata_filter_document_ids, metadata_condition = None, None
 
         if request.metadata_filtering_mode != "disabled":
-            # Convert workflow layer types to app_config layer types
-            if not request.metadata_model_config:
-                raise ValueError("metadata_model_config is required for this method")
+            app_metadata_model_config = ModelConfig(provider="", name="", mode=LLMMode.CHAT, completion_params={})
+            if request.metadata_filtering_mode == "automatic":
+                if not request.metadata_model_config:
+                    raise ValueError("metadata_model_config is required for this method")
 
-            app_metadata_model_config = ModelConfig.model_validate(request.metadata_model_config.model_dump())
+                app_metadata_model_config = ModelConfig.model_validate(request.metadata_model_config.model_dump())
 
             app_metadata_filtering_conditions = None
             if request.metadata_filtering_conditions is not None:

api/core/tools/tool_file_manager.py

@@ -10,28 +10,19 @@ from typing import Union
 from uuid import uuid4
 
 import httpx
-from sqlalchemy.orm import Session
 
 from configs import dify_config
+from core.db.session_factory import session_factory
 from core.helper import ssrf_proxy
-from extensions.ext_database import db as global_db
+from dify_graph.file.models import ToolFile as ToolFilePydanticModel
 from extensions.ext_storage import storage
 from models.model import MessageFile
 from models.tools import ToolFile
 
 logger = logging.getLogger(__name__)
 
-from sqlalchemy.engine import Engine
-
 
 class ToolFileManager:
-    _engine: Engine
-
-    def __init__(self, engine: Engine | None = None):
-        if engine is None:
-            engine = global_db.engine
-        self._engine = engine
-
     @staticmethod
     def sign_file(tool_file_id: str, extension: str) -> str:
         """
@@ -89,7 +80,7 @@ class ToolFileManager:
         filepath = f"tools/{tenant_id}/{unique_filename}"
         storage.save(filepath, file_binary)
 
-        with Session(self._engine, expire_on_commit=False) as session:
+        with session_factory.create_session() as session:
             tool_file = ToolFile(
                 user_id=user_id,
                 tenant_id=tenant_id,
@@ -132,7 +123,7 @@ class ToolFileManager:
         filename = f"{unique_name}{extension}"
         filepath = f"tools/{tenant_id}/{filename}"
         storage.save(filepath, blob)
-        with Session(self._engine, expire_on_commit=False) as session:
+        with session_factory.create_session() as session:
             tool_file = ToolFile(
                 user_id=user_id,
                 tenant_id=tenant_id,
@@ -157,7 +148,7 @@ class ToolFileManager:
 
         :return: the binary of the file, mime type
         """
-        with Session(self._engine, expire_on_commit=False) as session:
+        with session_factory.create_session() as session:
             tool_file: ToolFile | None = (
                 session.query(ToolFile)
                 .where(
@@ -181,7 +172,7 @@ class ToolFileManager:
 
         :return: the binary of the file, mime type
         """
-        with Session(self._engine, expire_on_commit=False) as session:
+        with session_factory.create_session() as session:
             message_file: MessageFile | None = (
                 session.query(MessageFile)
                 .where(
@@ -217,7 +208,9 @@ class ToolFileManager:
 
         return blob, tool_file.mimetype
 
-    def get_file_generator_by_tool_file_id(self, tool_file_id: str) -> tuple[Generator | None, ToolFile | None]:
+    def get_file_generator_by_tool_file_id(
+        self, tool_file_id: str
+    ) -> tuple[Generator | None, ToolFilePydanticModel | None]:
         """
         get file binary
 
@@ -225,7 +218,7 @@ class ToolFileManager:
 
         :return: the binary of the file, mime type
         """
-        with Session(self._engine, expire_on_commit=False) as session:
+        with session_factory.create_session() as session:
             tool_file: ToolFile | None = (
                 session.query(ToolFile)
                 .where(
@@ -239,7 +232,7 @@ class ToolFileManager:
 
         stream = storage.load_stream(tool_file.file_key)
 
-        return stream, tool_file
+        return stream, ToolFilePydanticModel.model_validate(tool_file)
 
 
 # init tool_file_parser

api/core/tools/workflow_as_tool/provider.py

@@ -37,6 +37,7 @@ VARIABLE_TO_PARAMETER_TYPE_MAPPING = {
     VariableEntityType.CHECKBOX: ToolParameter.ToolParameterType.BOOLEAN,
     VariableEntityType.FILE: ToolParameter.ToolParameterType.FILE,
     VariableEntityType.FILE_LIST: ToolParameter.ToolParameterType.FILES,
+    VariableEntityType.JSON_OBJECT: ToolParameter.ToolParameterType.OBJECT,
 }
 
 

api/core/workflow/node_factory.py

@@ -50,6 +50,7 @@ from dify_graph.nodes.template_transform.template_renderer import (
     CodeExecutorJinja2TemplateRenderer,
 )
 from dify_graph.nodes.template_transform.template_transform_node import TemplateTransformNode
+from dify_graph.nodes.tool.tool_node import ToolNode
 from dify_graph.variables.segments import StringSegment
 from extensions.ext_database import db
 from models.model import Conversation
@@ -250,6 +251,7 @@ class DifyNodeFactory(NodeFactory):
                 model_factory=self._llm_model_factory,
                 model_instance=model_instance,
                 memory=memory,
+                http_client=self._http_request_http_client,
             )
 
         if node_type == NodeType.DATASOURCE:
@@ -292,6 +294,7 @@ class DifyNodeFactory(NodeFactory):
                 model_factory=self._llm_model_factory,
                 model_instance=model_instance,
                 memory=memory,
+                http_client=self._http_request_http_client,
             )
 
         if node_type == NodeType.PARAMETER_EXTRACTOR:
@@ -308,6 +311,15 @@ class DifyNodeFactory(NodeFactory):
                 memory=memory,
             )
 
+        if node_type == NodeType.TOOL:
+            return ToolNode(
+                id=node_id,
+                config=node_config,
+                graph_init_params=self.graph_init_params,
+                graph_runtime_state=self.graph_runtime_state,
+                tool_file_manager_factory=self._http_request_tool_file_manager_factory(),
+            )
+
         return node_class(
             id=node_id,
             config=node_config,

api/dify_graph/file/models.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
 from typing import Any
+from uuid import UUID, uuid4
 
 from pydantic import BaseModel, Field, model_validator
 
@@ -43,6 +44,24 @@ class FileUploadConfig(BaseModel):
     number_limits: int = 0
 
 
+class ToolFile(BaseModel):
+    id: UUID = Field(default_factory=uuid4, description="Unique identifier for the file")
+    user_id: UUID = Field(..., description="ID of the user who owns this file")
+    tenant_id: UUID = Field(..., description="ID of the tenant/organization")
+    conversation_id: UUID | None = Field(None, description="ID of the associated conversation")
+    file_key: str = Field(..., max_length=255, description="Storage key for the file")
+    mimetype: str = Field(..., max_length=255, description="MIME type of the file")
+    original_url: str | None = Field(
+        None, max_length=2048, description="Original URL if file was fetched from external source"
+    )
+    name: str = Field(default="", max_length=255, description="Display name of the file")
+    size: int = Field(default=-1, ge=-1, description="File size in bytes (-1 if unknown)")
+
+    class Config:
+        from_attributes = True  # Enable ORM mode for SQLAlchemy compatibility
+        populate_by_name = True
+
+
 class File(BaseModel):
     # NOTE: dify_model_identity is a special identifier used to distinguish between
     # new and old data formats during serialization and deserialization.

api/dify_graph/nodes/document_extractor/node.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 import tempfile
+import zipfile
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
@@ -82,8 +83,18 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
 
         value = variable.value
         inputs = {"variable_selector": variable_selector}
+        if isinstance(value, list):
+            value = list(filter(lambda x: x, value))
         process_data = {"documents": value if isinstance(value, list) else [value]}
 
+        if not value:
+            return NodeRunResult(
+                status=WorkflowNodeExecutionStatus.SUCCEEDED,
+                inputs=inputs,
+                process_data=process_data,
+                outputs={"text": ArrayStringSegment(value=[])},
+            )
+
         try:
             if isinstance(value, list):
                 extracted_text_list = [
@@ -111,6 +122,7 @@ class DocumentExtractorNode(Node[DocumentExtractorNodeData]):
             else:
                 raise DocumentExtractorError(f"Unsupported variable type: {type(value)}")
         except DocumentExtractorError as e:
+            logger.warning(e, exc_info=True)
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED,
                 error=str(e),
@@ -385,6 +397,32 @@ def parser_docx_part(block, doc: Document, content_items, i):
         content_items.append((i, "table", Table(block, doc)))
 
 
+def _normalize_docx_zip(file_content: bytes) -> bytes:
+    """
+    Some DOCX files (e.g. exported by Evernote on Windows) are malformed:
+    ZIP entry names use backslash (\\) as path separator instead of the forward
+    slash (/) required by both the ZIP spec and OOXML.  On Linux/Mac the entry
+    "word\\document.xml" is never found when python-docx looks for
+    "word/document.xml", which triggers a KeyError about a missing relationship.
+
+    This function rewrites the ZIP in-memory, normalizing all entry names to
+    use forward slashes without touching any actual document content.
+    """
+    try:
+        with zipfile.ZipFile(io.BytesIO(file_content), "r") as zin:
+            out_buf = io.BytesIO()
+            with zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout:
+                for item in zin.infolist():
+                    data = zin.read(item.filename)
+                    # Normalize backslash path separators to forward slash
+                    item.filename = item.filename.replace("\\", "/")
+                    zout.writestr(item, data)
+            return out_buf.getvalue()
+    except zipfile.BadZipFile:
+        # Not a valid zip — return as-is and let python-docx report the real error
+        return file_content
+
+
 def _extract_text_from_docx(file_content: bytes) -> str:
     """
     Extract text from a DOCX file.
@@ -392,7 +430,15 @@ def _extract_text_from_docx(file_content: bytes) -> str:
     """
     try:
         doc_file = io.BytesIO(file_content)
-        doc = docx.Document(doc_file)
+        try:
+            doc = docx.Document(doc_file)
+        except Exception as e:
+            logger.warning("Failed to parse DOCX, attempting to normalize ZIP entry paths: %s", e)
+            # Some DOCX files exported by tools like Evernote on Windows use
+            # backslash path separators in ZIP entries and/or single-quoted XML
+            # attributes, both of which break python-docx on Linux. Normalize and retry.
+            file_content = _normalize_docx_zip(file_content)
+            doc = docx.Document(io.BytesIO(file_content))
         text = []
 
         # Keep track of paragraph and table positions

api/dify_graph/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -14,7 +14,6 @@ from dify_graph.model_runtime.utils.encoders import jsonable_encoder
 from dify_graph.node_events import NodeRunResult
 from dify_graph.nodes.base import LLMUsageTrackingMixin
 from dify_graph.nodes.base.node import Node
-from dify_graph.nodes.llm.file_saver import FileSaverImpl, LLMFileSaver
 from dify_graph.repositories.rag_retrieval_protocol import KnowledgeRetrievalRequest, RAGRetrievalProtocol, Source
 from dify_graph.variables import (
     ArrayFileSegment,
@@ -23,7 +22,11 @@ from dify_graph.variables import (
 )
 from dify_graph.variables.segments import ArrayObjectSegment
 
-from .entities import KnowledgeRetrievalNodeData
+from .entities import (
+    Condition,
+    KnowledgeRetrievalNodeData,
+    MetadataFilteringCondition,
+)
 from .exc import (
     KnowledgeRetrievalNodeError,
     RateLimitExceededError,
@@ -43,8 +46,6 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
     # Output variable for file
     _file_outputs: list["File"]
 
-    _llm_file_saver: LLMFileSaver
-
     def __init__(
         self,
         id: str,
@@ -52,8 +53,6 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         graph_init_params: "GraphInitParams",
         graph_runtime_state: "GraphRuntimeState",
         rag_retrieval: RAGRetrievalProtocol,
-        *,
-        llm_file_saver: LLMFileSaver | None = None,
     ):
         super().__init__(
             id=id,
@@ -65,14 +64,6 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         self._file_outputs = []
         self._rag_retrieval = rag_retrieval
 
-        if llm_file_saver is None:
-            dify_ctx = self.require_dify_context()
-            llm_file_saver = FileSaverImpl(
-                user_id=dify_ctx.user_id,
-                tenant_id=dify_ctx.tenant_id,
-            )
-        self._llm_file_saver = llm_file_saver
-
     @classmethod
     def version(cls):
         return "1"
@@ -171,6 +162,12 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         if node_data.metadata_filtering_mode is not None:
             metadata_filtering_mode = node_data.metadata_filtering_mode
 
+        resolved_metadata_conditions = (
+            self._resolve_metadata_filtering_conditions(node_data.metadata_filtering_conditions)
+            if node_data.metadata_filtering_conditions
+            else None
+        )
+
         if str(node_data.retrieval_mode) == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE and query:
             # fetch model config
             if node_data.single_retrieval_config is None:
@@ -189,7 +186,7 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
                     model_mode=model.mode,
                     model_name=model.name,
                     metadata_model_config=node_data.metadata_model_config,
-                    metadata_filtering_conditions=node_data.metadata_filtering_conditions,
+                    metadata_filtering_conditions=resolved_metadata_conditions,
                     metadata_filtering_mode=metadata_filtering_mode,
                     query=query,
                 )
@@ -247,7 +244,7 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
                     weights=weights,
                     reranking_enable=node_data.multiple_retrieval_config.reranking_enable,
                     metadata_model_config=node_data.metadata_model_config,
-                    metadata_filtering_conditions=node_data.metadata_filtering_conditions,
+                    metadata_filtering_conditions=resolved_metadata_conditions,
                     metadata_filtering_mode=metadata_filtering_mode,
                     attachment_ids=[attachment.related_id for attachment in attachments] if attachments else None,
                 )
@@ -256,6 +253,48 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         usage = self._rag_retrieval.llm_usage
         return retrieval_resource_list, usage
 
+    def _resolve_metadata_filtering_conditions(
+        self, conditions: MetadataFilteringCondition
+    ) -> MetadataFilteringCondition:
+        if conditions.conditions is None:
+            return MetadataFilteringCondition(
+                logical_operator=conditions.logical_operator,
+                conditions=None,
+            )
+
+        variable_pool = self.graph_runtime_state.variable_pool
+        resolved_conditions: list[Condition] = []
+        for cond in conditions.conditions or []:
+            value = cond.value
+            if isinstance(value, str):
+                segment_group = variable_pool.convert_template(value)
+                if len(segment_group.value) == 1:
+                    resolved_value = segment_group.value[0].to_object()
+                else:
+                    resolved_value = segment_group.text
+            elif isinstance(value, Sequence) and all(isinstance(v, str) for v in value):
+                resolved_values = []
+                for v in value:  # type: ignore
+                    segment_group = variable_pool.convert_template(v)
+                    if len(segment_group.value) == 1:
+                        resolved_values.append(segment_group.value[0].to_object())
+                    else:
+                        resolved_values.append(segment_group.text)
+                resolved_value = resolved_values
+            else:
+                resolved_value = value
+            resolved_conditions.append(
+                Condition(
+                    name=cond.name,
+                    comparison_operator=cond.comparison_operator,
+                    value=resolved_value,
+                )
+            )
+        return MetadataFilteringCondition(
+            logical_operator=conditions.logical_operator or "and",
+            conditions=resolved_conditions,
+        )
+
     @classmethod
     def _extract_variable_selector_to_variable_mapping(
         cls,

api/dify_graph/nodes/llm/file_saver.py

@@ -1,14 +1,11 @@
 import mimetypes
 import typing as tp
 
-from sqlalchemy import Engine
-
 from constants.mimetypes import DEFAULT_EXTENSION, DEFAULT_MIME_TYPE
-from core.helper import ssrf_proxy
 from core.tools.signature import sign_tool_file
 from core.tools.tool_file_manager import ToolFileManager
 from dify_graph.file import File, FileTransferMethod, FileType
-from extensions.ext_database import db as global_db
+from dify_graph.nodes.protocols import HttpClientProtocol
 
 
 class LLMFileSaver(tp.Protocol):
@@ -59,30 +56,20 @@ class LLMFileSaver(tp.Protocol):
         raise NotImplementedError()
 
 
-EngineFactory: tp.TypeAlias = tp.Callable[[], Engine]
-
-
 class FileSaverImpl(LLMFileSaver):
-    _engine_factory: EngineFactory
     _tenant_id: str
     _user_id: str
 
-    def __init__(self, user_id: str, tenant_id: str, engine_factory: EngineFactory | None = None):
-        if engine_factory is None:
-
-            def _factory():
-                return global_db.engine
-
-            engine_factory = _factory
-        self._engine_factory = engine_factory
+    def __init__(self, user_id: str, tenant_id: str, http_client: HttpClientProtocol):
         self._user_id = user_id
         self._tenant_id = tenant_id
+        self._http_client = http_client
 
     def _get_tool_file_manager(self):
-        return ToolFileManager(engine=self._engine_factory())
+        return ToolFileManager()
 
     def save_remote_url(self, url: str, file_type: FileType) -> File:
-        http_response = ssrf_proxy.get(url)
+        http_response = self._http_client.get(url)
         http_response.raise_for_status()
         data = http_response.content
         mime_type_from_header = http_response.headers.get("Content-Type")

api/dify_graph/nodes/llm/node.py

@@ -64,6 +64,7 @@ from dify_graph.nodes.base.entities import VariableSelector
 from dify_graph.nodes.base.node import Node
 from dify_graph.nodes.base.variable_template_parser import VariableTemplateParser
 from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory
+from dify_graph.nodes.protocols import HttpClientProtocol
 from dify_graph.runtime import VariablePool
 from dify_graph.variables import (
     ArrayFileSegment,
@@ -127,6 +128,7 @@ class LLMNode(Node[LLMNodeData]):
         credentials_provider: CredentialsProvider,
         model_factory: ModelFactory,
         model_instance: ModelInstance,
+        http_client: HttpClientProtocol,
         memory: PromptMessageMemory | None = None,
         llm_file_saver: LLMFileSaver | None = None,
     ):
@@ -149,6 +151,7 @@ class LLMNode(Node[LLMNodeData]):
             llm_file_saver = FileSaverImpl(
                 user_id=dify_ctx.user_id,
                 tenant_id=dify_ctx.tenant_id,
+                http_client=http_client,
             )
         self._llm_file_saver = llm_file_saver
 

api/dify_graph/nodes/protocols.py

@@ -1,8 +1,10 @@
+from collections.abc import Generator
 from typing import Any, Protocol
 
 import httpx
 
 from dify_graph.file import File
+from dify_graph.file.models import ToolFile
 
 
 class HttpClientProtocol(Protocol):
@@ -40,3 +42,5 @@ class ToolFileManagerProtocol(Protocol):
         mimetype: str,
         filename: str | None = None,
     ) -> Any: ...
+
+    def get_file_generator_by_tool_file_id(self, tool_file_id: str) -> tuple[Generator | None, ToolFile | None]: ...

api/dify_graph/nodes/question_classifier/question_classifier_node.py

@@ -28,6 +28,7 @@ from dify_graph.nodes.llm import (
 )
 from dify_graph.nodes.llm.file_saver import FileSaverImpl, LLMFileSaver
 from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory
+from dify_graph.nodes.protocols import HttpClientProtocol
 from libs.json_in_md_parser import parse_and_check_json_markdown
 
 from .entities import QuestionClassifierNodeData
@@ -68,6 +69,7 @@ class QuestionClassifierNode(Node[QuestionClassifierNodeData]):
         credentials_provider: "CredentialsProvider",
         model_factory: "ModelFactory",
         model_instance: ModelInstance,
+        http_client: HttpClientProtocol,
         memory: PromptMessageMemory | None = None,
         llm_file_saver: LLMFileSaver | None = None,
     ):
@@ -90,6 +92,7 @@ class QuestionClassifierNode(Node[QuestionClassifierNodeData]):
             llm_file_saver = FileSaverImpl(
                 user_id=dify_ctx.user_id,
                 tenant_id=dify_ctx.tenant_id,
+                http_client=http_client,
             )
         self._llm_file_saver = llm_file_saver
 

api/dify_graph/nodes/tool/tool_node.py

@@ -1,9 +1,6 @@
 from collections.abc import Generator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
-from sqlalchemy import select
-from sqlalchemy.orm import Session
-
 from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
 from core.tools.__base.tool import Tool
 from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter
@@ -21,11 +18,10 @@ from dify_graph.model_runtime.entities.llm_entities import LLMUsage
 from dify_graph.node_events import NodeEventBase, NodeRunResult, StreamChunkEvent, StreamCompletedEvent
 from dify_graph.nodes.base.node import Node
 from dify_graph.nodes.base.variable_template_parser import VariableTemplateParser
+from dify_graph.nodes.protocols import ToolFileManagerProtocol
 from dify_graph.variables.segments import ArrayAnySegment, ArrayFileSegment
 from dify_graph.variables.variables import ArrayAnyVariable
-from extensions.ext_database import db
 from factories import file_factory
-from models import ToolFile
 from services.tools.builtin_tools_manage_service import BuiltinToolManageService
 
 from .entities import ToolNodeData
@@ -36,7 +32,8 @@ from .exc import (
 )
 
 if TYPE_CHECKING:
-    from dify_graph.runtime import VariablePool
+    from dify_graph.entities import GraphInitParams
+    from dify_graph.runtime import GraphRuntimeState, VariablePool
 
 
 class ToolNode(Node[ToolNodeData]):
@@ -46,6 +43,23 @@ class ToolNode(Node[ToolNodeData]):
 
     node_type = NodeType.TOOL
 
+    def __init__(
+        self,
+        id: str,
+        config: Mapping[str, Any],
+        graph_init_params: "GraphInitParams",
+        graph_runtime_state: "GraphRuntimeState",
+        *,
+        tool_file_manager_factory: ToolFileManagerProtocol,
+    ):
+        super().__init__(
+            id=id,
+            config=config,
+            graph_init_params=graph_init_params,
+            graph_runtime_state=graph_runtime_state,
+        )
+        self._tool_file_manager_factory = tool_file_manager_factory
+
     @classmethod
     def version(cls) -> str:
         return "1"
@@ -271,11 +285,9 @@ class ToolNode(Node[ToolNodeData]):
 
                 tool_file_id = str(url).split("/")[-1].split(".")[0]
 
-                with Session(db.engine) as session:
-                    stmt = select(ToolFile).where(ToolFile.id == tool_file_id)
-                    tool_file = session.scalar(stmt)
-                    if tool_file is None:
-                        raise ToolFileError(f"Tool file {tool_file_id} does not exist")
+                _, tool_file = self._tool_file_manager_factory.get_file_generator_by_tool_file_id(tool_file_id)
+                if not tool_file:
+                    raise ToolFileError(f"tool file {tool_file_id} not found")
 
                 mapping = {
                     "tool_file_id": tool_file_id,
@@ -294,11 +306,9 @@ class ToolNode(Node[ToolNodeData]):
                 assert message.meta
 
                 tool_file_id = message.message.text.split("/")[-1].split(".")[0]
-                with Session(db.engine) as session:
-                    stmt = select(ToolFile).where(ToolFile.id == tool_file_id)
-                    tool_file = session.scalar(stmt)
-                    if tool_file is None:
-                        raise ToolFileError(f"tool file {tool_file_id} not exists")
+                _, tool_file = self._tool_file_manager_factory.get_file_generator_by_tool_file_id(tool_file_id)
+                if not tool_file:
+                    raise ToolFileError(f"tool file {tool_file_id} not exists")
 
                 mapping = {
                     "tool_file_id": tool_file_id,

api/extensions/ext_commands.py

@@ -13,6 +13,7 @@ def init_app(app: DifyApp):
         convert_to_agent_apps,
         create_tenant,
         delete_archived_workflow_runs,
+        export_app_messages,
         extract_plugins,
         extract_unique_plugins,
         file_usage,
@@ -66,6 +67,7 @@ def init_app(app: DifyApp):
         restore_workflow_runs,
         clean_workflow_runs,
         clean_expired_messages,
+        export_app_messages,
     ]
     for cmd in cmds_to_register:
         app.cli.add_command(cmd)

api/extensions/otel/celery_sqlcommenter.py

@@ -0,0 +1,114 @@
+"""
+Celery SQL comment context for OpenTelemetry SQLCommenter.
+
+Injects Celery-specific metadata (framework, task_name, traceparent, celery_retries,
+routing_key) into SQL comments for queries executed by Celery workers. This improves
+trace-to-SQL correlation and debugging in production.
+
+Uses the OpenTelemetry context key SQLCOMMENTER_ORM_TAGS_AND_VALUES, which is read
+by opentelemetry.instrumentation.sqlcommenter_utils._add_framework_tags() when the
+SQLAlchemy instrumentor appends comments to SQL statements.
+"""
+
+import logging
+from typing import Any
+
+from celery.signals import task_postrun, task_prerun
+from opentelemetry import context
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+logger = logging.getLogger(__name__)
+_TRACE_PROPAGATOR = TraceContextTextMapPropagator()
+
+_SQLCOMMENTER_CONTEXT_KEY = "SQLCOMMENTER_ORM_TAGS_AND_VALUES"
+_TOKEN_ATTR = "_dify_sqlcommenter_context_token"
+
+
+def _build_celery_sqlcommenter_tags(task: Any) -> dict[str, str | int]:
+    """Build SQL commenter tags from the current Celery task and OpenTelemetry context."""
+    tags: dict[str, str | int] = {}
+
+    try:
+        tags["framework"] = f"celery:{_get_celery_version()}"
+    except Exception:
+        tags["framework"] = "celery:unknown"
+
+    if task and getattr(task, "name", None):
+        tags["task_name"] = str(task.name)
+
+    traceparent = _get_traceparent()
+    if traceparent:
+        tags["traceparent"] = traceparent
+
+    if task and hasattr(task, "request"):
+        request = task.request
+        retries = getattr(request, "retries", None)
+        if retries is not None and retries > 0:
+            tags["celery_retries"] = int(retries)
+
+        delivery_info = getattr(request, "delivery_info", None) or {}
+        if isinstance(delivery_info, dict):
+            routing_key = delivery_info.get("routing_key")
+            if routing_key:
+                tags["routing_key"] = str(routing_key)
+
+    return tags
+
+
+def _get_celery_version() -> str:
+    import celery
+
+    return getattr(celery, "__version__", "unknown")
+
+
+def _get_traceparent() -> str | None:
+    """Extract traceparent from the current OpenTelemetry context."""
+    carrier: dict[str, str] = {}
+    _TRACE_PROPAGATOR.inject(carrier)
+    return carrier.get("traceparent")
+
+
+def _on_task_prerun(*args: object, **kwargs: object) -> None:
+    task = kwargs.get("task")
+    if not task:
+        return
+
+    tags = _build_celery_sqlcommenter_tags(task)
+    if not tags:
+        return
+
+    current = context.get_current()
+    new_ctx = context.set_value(_SQLCOMMENTER_CONTEXT_KEY, tags, current)
+    token = context.attach(new_ctx)
+    setattr(task, _TOKEN_ATTR, token)
+
+
+def _on_task_postrun(*args: object, **kwargs: object) -> None:
+    task = kwargs.get("task")
+    if not task:
+        return
+
+    token = getattr(task, _TOKEN_ATTR, None)
+    if token is None:
+        return
+
+    try:
+        context.detach(token)
+    except Exception:
+        logger.debug("Failed to detach SQL commenter context", exc_info=True)
+    finally:
+        try:
+            delattr(task, _TOKEN_ATTR)
+        except AttributeError:
+            pass
+
+
+def setup_celery_sqlcommenter() -> None:
+    """
+    Connect Celery task_prerun and task_postrun handlers to inject SQL comment
+    context for worker queries. Call this from init_celery_worker after
+    CeleryInstrumentor().instrument() so our handlers run after the OTEL
+    instrumentor's and the trace context is already attached.
+    """
+    task_prerun.connect(_on_task_prerun, weak=False)
+    task_postrun.connect(_on_task_postrun, weak=False)

api/extensions/otel/runtime.py

@@ -67,11 +67,14 @@ def init_celery_worker(*args, **kwargs):
         from opentelemetry.metrics import get_meter_provider
         from opentelemetry.trace import get_tracer_provider
 
+        from extensions.otel.celery_sqlcommenter import setup_celery_sqlcommenter
+
         tracer_provider = get_tracer_provider()
         metric_provider = get_meter_provider()
         if dify_config.DEBUG:
             logger.info("Initializing OpenTelemetry for Celery worker")
         CeleryInstrumentor(tracer_provider=tracer_provider, meter_provider=metric_provider).instrument()
+        setup_celery_sqlcommenter()
 
 
 def is_instrument_flag_enabled() -> bool:

api/models/dataset.py

@@ -10,6 +10,7 @@ import re
 import time
 from datetime import datetime
 from json import JSONDecodeError
+from operator import itemgetter
 from typing import Any, cast
 from uuid import uuid4
 
@@ -806,41 +807,53 @@ class DocumentSegment(Base):
     def sign_content(self) -> str:
         return self.get_sign_content()
 
+    @staticmethod
+    def _build_signed_query_params(*, sign_target: str, upload_file_id: str) -> str:
+        nonce = os.urandom(16).hex()
+        timestamp = str(int(time.time()))
+        data_to_sign = f"{sign_target}|{upload_file_id}|{timestamp}|{nonce}"
+        secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b""
+        sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
+        encoded_sign = base64.urlsafe_b64encode(sign).decode()
+        return f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
+
+    def _get_accessible_upload_file_ids(self, upload_file_ids: set[str]) -> set[str]:
+        if not upload_file_ids:
+            return set()
+
+        matched_upload_file_ids = db.session.scalars(
+            select(UploadFile.id).where(
+                UploadFile.tenant_id == self.tenant_id,
+                UploadFile.id.in_(list(upload_file_ids)),
+            )
+        ).all()
+        return {str(upload_file_id) for upload_file_id in matched_upload_file_ids}
+
     def get_sign_content(self) -> str:
         signed_urls: list[tuple[int, int, str]] = []
         text = self.content
 
-        # For data before v0.10.0
-        pattern = r"/files/([a-f0-9\-]+)/image-preview(?:\?.*?)?"
-        matches = re.finditer(pattern, text)
-        for match in matches:
-            upload_file_id = match.group(1)
-            nonce = os.urandom(16).hex()
-            timestamp = str(int(time.time()))
-            data_to_sign = f"image-preview|{upload_file_id}|{timestamp}|{nonce}"
-            secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b""
-            sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
-            encoded_sign = base64.urlsafe_b64encode(sign).decode()
+        upload_file_preview_patterns = {
+            "image-preview": r"(?:https?://[^\s\)\"\']+)?/files/([a-f0-9\-]+)/image-preview(?:\?[^\s\)\"\']*)?",
+            "file-preview": r"(?:https?://[^\s\)\"\']+)?/files/([a-f0-9\-]+)/file-preview(?:\?[^\s\)\"\']*)?",
+        }
+        upload_file_matches: list[tuple[re.Match[str], str, str]] = []
+        upload_file_ids: set[str] = set()
 
-            params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
-            base_url = f"/files/{upload_file_id}/image-preview"
-            signed_url = f"{base_url}?{params}"
-            signed_urls.append((match.start(), match.end(), signed_url))
+        for preview_type, pattern in upload_file_preview_patterns.items():
+            for match in re.finditer(pattern, text):
+                upload_file_id = match.group(1)
+                upload_file_matches.append((match, preview_type, upload_file_id))
+                upload_file_ids.add(upload_file_id)
 
-        # For data after v0.10.0
-        pattern = r"/files/([a-f0-9\-]+)/file-preview(?:\?.*?)?"
-        matches = re.finditer(pattern, text)
-        for match in matches:
-            upload_file_id = match.group(1)
-            nonce = os.urandom(16).hex()
-            timestamp = str(int(time.time()))
-            data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
-            secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b""
-            sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
-            encoded_sign = base64.urlsafe_b64encode(sign).decode()
+        accessible_upload_file_ids = self._get_accessible_upload_file_ids(upload_file_ids)
 
-            params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
-            base_url = f"/files/{upload_file_id}/file-preview"
+        for match, preview_type, upload_file_id in upload_file_matches:
+            if upload_file_id not in accessible_upload_file_ids:
+                continue
+
+            params = self._build_signed_query_params(sign_target=preview_type, upload_file_id=upload_file_id)
+            base_url = f"/files/{upload_file_id}/{preview_type}"
             signed_url = f"{base_url}?{params}"
             signed_urls.append((match.start(), match.end(), signed_url))
 
@@ -851,19 +864,13 @@ class DocumentSegment(Base):
         for match in matches:
             upload_file_id = match.group(1)
             file_extension = match.group(2)
-            nonce = os.urandom(16).hex()
-            timestamp = str(int(time.time()))
-            data_to_sign = f"file-preview|{upload_file_id}|{timestamp}|{nonce}"
-            secret_key = dify_config.SECRET_KEY.encode() if dify_config.SECRET_KEY else b""
-            sign = hmac.new(secret_key, data_to_sign.encode(), hashlib.sha256).digest()
-            encoded_sign = base64.urlsafe_b64encode(sign).decode()
-
-            params = f"timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}"
+            params = self._build_signed_query_params(sign_target="file-preview", upload_file_id=upload_file_id)
             base_url = f"/files/tools/{upload_file_id}.{file_extension}"
             signed_url = f"{base_url}?{params}"
             signed_urls.append((match.start(), match.end(), signed_url))
 
         # Reconstruct the text with signed URLs
+        signed_urls.sort(key=itemgetter(0))
         offset = 0
         for start, end, signed_url in signed_urls:
             text = text[: start + offset] + signed_url + text[end + offset :]

api/schedule/queue_monitor_task.py

@@ -21,6 +21,10 @@ celery_redis = Redis(
     ssl_cert_reqs=getattr(dify_config, "REDIS_SSL_CERT_REQS", None) if dify_config.BROKER_USE_SSL else None,
     ssl_certfile=getattr(dify_config, "REDIS_SSL_CERTFILE", None) if dify_config.BROKER_USE_SSL else None,
     ssl_keyfile=getattr(dify_config, "REDIS_SSL_KEYFILE", None) if dify_config.BROKER_USE_SSL else None,
+    # Add conservative socket timeouts and health checks to avoid long-lived half-open sockets
+    socket_timeout=5,
+    socket_connect_timeout=5,
+    health_check_interval=30,
 )
 
 logger = logging.getLogger(__name__)

api/schedule/trigger_provider_refresh_task.py

@@ -3,6 +3,7 @@ import math
 import time
 from collections.abc import Iterable, Sequence
 
+from celery import group
 from sqlalchemy import ColumnElement, and_, func, or_, select
 from sqlalchemy.engine.row import Row
 from sqlalchemy.orm import Session
@@ -85,20 +86,25 @@ def trigger_provider_refresh() -> None:
             lock_keys: list[str] = build_trigger_refresh_lock_keys(subscriptions)
             acquired: list[bool] = _acquire_locks(keys=lock_keys, ttl_seconds=lock_ttl)
 
-            enqueued: int = 0
-            for (tenant_id, subscription_id), is_locked in zip(subscriptions, acquired):
-                if not is_locked:
-                    continue
-                trigger_subscription_refresh.delay(tenant_id=tenant_id, subscription_id=subscription_id)
-                enqueued += 1
+            if not any(acquired):
+                continue
+
+            jobs = [
+                trigger_subscription_refresh.s(tenant_id=tenant_id, subscription_id=subscription_id)
+                for (tenant_id, subscription_id), is_locked in zip(subscriptions, acquired)
+                if is_locked
+            ]
+            result = group(jobs).apply_async()
+            enqueued = len(jobs)
 
             logger.info(
-                "Trigger refresh page %d/%d: scanned=%d locks_acquired=%d enqueued=%d",
+                "Trigger refresh page %d/%d: scanned=%d locks_acquired=%d enqueued=%d result=%s",
                 page + 1,
                 pages,
                 len(subscriptions),
                 sum(1 for x in acquired if x),
                 enqueued,
+                result,
             )
 
     logger.info("Trigger refresh scan done: due=%d", total_due)

api/schedule/workflow_schedule_task.py

@@ -1,6 +1,6 @@
 import logging
 
-from celery import group, shared_task
+from celery import current_app, group, shared_task
 from sqlalchemy import and_, select
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -29,31 +29,27 @@ def poll_workflow_schedules() -> None:
     with session_factory() as session:
         total_dispatched = 0
 
-        # Process in batches until we've handled all due schedules or hit the limit
         while True:
             due_schedules = _fetch_due_schedules(session)
 
             if not due_schedules:
                 break
 
-            dispatched_count = _process_schedules(session, due_schedules)
-            total_dispatched += dispatched_count
+            with current_app.producer_or_acquire() as producer:  # type: ignore
+                dispatched_count = _process_schedules(session, due_schedules, producer)
+                total_dispatched += dispatched_count
 
-            logger.debug("Batch processed: %d dispatched", dispatched_count)
-
-            # Circuit breaker: check if we've hit the per-tick limit (if enabled)
-            if (
-                dify_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK > 0
-                and total_dispatched >= dify_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK
-            ):
-                logger.warning(
-                    "Circuit breaker activated: reached dispatch limit (%d), will continue next tick",
-                    dify_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK,
-                )
-                break
+                logger.debug("Batch processed: %d dispatched", dispatched_count)
 
+                # Circuit breaker: check if we've hit the per-tick limit (if enabled)
+                if 0 < dify_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK <= total_dispatched:
+                    logger.warning(
+                        "Circuit breaker activated: reached dispatch limit (%d), will continue next tick",
+                        dify_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK,
+                    )
+                    break
         if total_dispatched > 0:
-            logger.info("Total processed: %d dispatched", total_dispatched)
+            logger.info("Total processed: %d workflow schedule(s) dispatched", total_dispatched)
 
 
 def _fetch_due_schedules(session: Session) -> list[WorkflowSchedulePlan]:
@@ -90,7 +86,7 @@ def _fetch_due_schedules(session: Session) -> list[WorkflowSchedulePlan]:
     return list(due_schedules)
 
 
-def _process_schedules(session: Session, schedules: list[WorkflowSchedulePlan]) -> int:
+def _process_schedules(session: Session, schedules: list[WorkflowSchedulePlan], producer=None) -> int:
     """Process schedules: check quota, update next run time and dispatch to Celery in parallel."""
     if not schedules:
         return 0
@@ -107,7 +103,7 @@ def _process_schedules(session: Session, schedules: list[WorkflowSchedulePlan])
 
     if tasks_to_dispatch:
         job = group(run_schedule_trigger.s(schedule_id) for schedule_id in tasks_to_dispatch)
-        job.apply_async()
+        job.apply_async(producer=producer)
 
         logger.debug("Dispatched %d tasks in parallel", len(tasks_to_dispatch))
 

api/services/datasource_provider_service.py

@@ -824,6 +824,7 @@ class DatasourceProviderService:
                 "langgenius/firecrawl_datasource",
                 "langgenius/notion_datasource",
                 "langgenius/jina_datasource",
+                "watercrawl/watercrawl_datasource",
             ]:
                 datasource_provider_id = DatasourceProviderID(f"{datasource.plugin_id}/{datasource.provider}")
                 credentials = self.list_datasource_credentials(

api/services/enterprise/plugin_manager_service.py

@@ -3,6 +3,7 @@ import logging
 
 from pydantic import BaseModel
 
+from configs import dify_config
 from services.enterprise.base import EnterprisePluginManagerRequest
 from services.errors.base import BaseServiceError
 
@@ -28,6 +29,11 @@ class CheckCredentialPolicyComplianceRequest(BaseModel):
         return data
 
 
+class PreUninstallPluginRequest(BaseModel):
+    tenant_id: str
+    plugin_unique_identifier: str
+
+
 class CredentialPolicyViolationError(BaseServiceError):
     pass
 
@@ -55,3 +61,21 @@ class PluginManagerService:
             body.dify_credential_id,
             ret.get("result", False),
         )
+
+    @classmethod
+    def try_pre_uninstall_plugin(cls, body: PreUninstallPluginRequest):
+        try:
+            # the invocation must be synchronous.
+            EnterprisePluginManagerRequest.send_request(
+                "POST",
+                "/pre-uninstall-plugin",
+                json=body.model_dump(),
+                raise_for_status=True,
+                timeout=dify_config.ENTERPRISE_REQUEST_TIMEOUT,
+            )
+        except Exception:
+            logger.exception(
+                "failed to perform pre uninstall plugin hook. tenant_id: %s, plugin_unique_identifier: %s",
+                body.tenant_id,
+                body.plugin_unique_identifier,
+            )

api/services/plugin/plugin_service.py

@@ -32,6 +32,10 @@ from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.provider import Provider, ProviderCredential
 from models.provider_ids import GenericProviderID
+from services.enterprise.plugin_manager_service import (
+    PluginManagerService,
+    PreUninstallPluginRequest,
+)
 from services.errors.plugin import PluginInstallationForbiddenError
 from services.feature_service import FeatureService, PluginInstallationScope
 
@@ -519,6 +523,13 @@ class PluginService:
         if not plugin:
             return manager.uninstall(tenant_id, plugin_installation_id)
 
+        if dify_config.ENTERPRISE_ENABLED:
+            PluginManagerService.try_pre_uninstall_plugin(
+                PreUninstallPluginRequest(
+                    tenant_id=tenant_id,
+                    plugin_unique_identifier=plugin.plugin_unique_identifier,
+                )
+            )
         with Session(db.engine) as session, session.begin():
             plugin_id = plugin.plugin_id
             logger.info("Deleting credentials for plugin: %s", plugin_id)

api/services/rag_pipeline/rag_pipeline_transform_service.py

@@ -63,7 +63,12 @@ class RagPipelineTransformService:
             ):
                 node = self._deal_file_extensions(node)
             if node.get("data", {}).get("type") == "knowledge-index":
-                node = self._deal_knowledge_index(dataset, doc_form, indexing_technique, retrieval_model, node)
+                knowledge_configuration = KnowledgeConfiguration.model_validate(node.get("data", {}))
+                if dataset.tenant_id != current_user.current_tenant_id:
+                    raise ValueError("Unauthorized")
+                node = self._deal_knowledge_index(
+                    knowledge_configuration, dataset, indexing_technique, retrieval_model, node
+                )
             new_nodes.append(node)
         if new_nodes:
             graph["nodes"] = new_nodes
@@ -155,14 +160,13 @@ class RagPipelineTransformService:
 
     def _deal_knowledge_index(
         self,
+        knowledge_configuration: KnowledgeConfiguration,
         dataset: Dataset,
-        doc_form: str,
         indexing_technique: str | None,
         retrieval_model: RetrievalSetting | None,
         node: dict,
     ):
         knowledge_configuration_dict = node.get("data", {})
-        knowledge_configuration = KnowledgeConfiguration.model_validate(knowledge_configuration_dict)
 
         if indexing_technique == "high_quality":
             knowledge_configuration.embedding_model = dataset.embedding_model

api/services/retention/conversation/message_export_service.py

@@ -0,0 +1,304 @@
+"""
+Export app messages to JSONL.GZ format.
+
+Outputs: conversation_id, message_id, query, answer, inputs (raw JSON),
+retriever_resources (from message_metadata), feedback (user feedbacks array).
+
+Uses (created_at, id) cursor pagination and batch-loads feedbacks to avoid N+1.
+Does NOT touch Message.inputs / Message.user_feedback properties.
+"""
+
+import datetime
+import gzip
+import json
+import logging
+import tempfile
+from collections import defaultdict
+from collections.abc import Generator, Iterable
+from pathlib import Path, PurePosixPath
+from typing import Any, BinaryIO, cast
+
+import orjson
+import sqlalchemy as sa
+from pydantic import BaseModel, ConfigDict, Field
+from sqlalchemy import select, tuple_
+from sqlalchemy.orm import Session
+
+from extensions.ext_database import db
+from extensions.ext_storage import storage
+from models.model import Message, MessageFeedback
+
+logger = logging.getLogger(__name__)
+
+MAX_FILENAME_BASE_LENGTH = 1024
+FORBIDDEN_FILENAME_SUFFIXES = (".jsonl.gz", ".jsonl", ".gz")
+
+
+class AppMessageExportFeedback(BaseModel):
+    id: str
+    app_id: str
+    conversation_id: str
+    message_id: str
+    rating: str
+    content: str | None = None
+    from_source: str
+    from_end_user_id: str | None = None
+    from_account_id: str | None = None
+    created_at: str
+    updated_at: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AppMessageExportRecord(BaseModel):
+    conversation_id: str
+    message_id: str
+    query: str
+    answer: str
+    inputs: dict[str, Any]
+    retriever_resources: list[Any] = Field(default_factory=list)
+    feedback: list[AppMessageExportFeedback] = Field(default_factory=list)
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AppMessageExportStats(BaseModel):
+    batches: int = 0
+    total_messages: int = 0
+    messages_with_feedback: int = 0
+    total_feedbacks: int = 0
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class AppMessageExportService:
+    @staticmethod
+    def validate_export_filename(filename: str) -> str:
+        normalized = filename.strip()
+        if not normalized:
+            raise ValueError("--filename must not be empty.")
+
+        normalized_lower = normalized.lower()
+        if normalized_lower.endswith(FORBIDDEN_FILENAME_SUFFIXES):
+            raise ValueError("--filename must not include .jsonl.gz/.jsonl/.gz suffix; pass base filename only.")
+
+        if normalized.startswith("/"):
+            raise ValueError("--filename must be a relative path; absolute paths are not allowed.")
+
+        if "\\" in normalized:
+            raise ValueError("--filename must use '/' as path separator; '\\' is not allowed.")
+
+        if "//" in normalized:
+            raise ValueError("--filename must not contain empty path segments ('//').")
+
+        if len(normalized) > MAX_FILENAME_BASE_LENGTH:
+            raise ValueError(f"--filename is too long; max length is {MAX_FILENAME_BASE_LENGTH}.")
+
+        for ch in normalized:
+            if ch == "\x00" or ord(ch) < 32 or ord(ch) == 127:
+                raise ValueError("--filename must not contain control characters or NUL.")
+
+        parts = PurePosixPath(normalized).parts
+        if not parts:
+            raise ValueError("--filename must include a file name.")
+
+        if any(part in (".", "..") for part in parts):
+            raise ValueError("--filename must not contain '.' or '..' path segments.")
+
+        return normalized
+
+    @property
+    def output_gz_name(self) -> str:
+        return f"{self._filename_base}.jsonl.gz"
+
+    @property
+    def output_jsonl_name(self) -> str:
+        return f"{self._filename_base}.jsonl"
+
+    def __init__(
+        self,
+        app_id: str,
+        end_before: datetime.datetime,
+        filename: str,
+        *,
+        start_from: datetime.datetime | None = None,
+        batch_size: int = 1000,
+        use_cloud_storage: bool = False,
+        dry_run: bool = False,
+    ) -> None:
+        if start_from and start_from >= end_before:
+            raise ValueError(f"start_from ({start_from}) must be before end_before ({end_before})")
+
+        self._app_id = app_id
+        self._end_before = end_before
+        self._start_from = start_from
+        self._filename_base = self.validate_export_filename(filename)
+        self._batch_size = batch_size
+        self._use_cloud_storage = use_cloud_storage
+        self._dry_run = dry_run
+
+    def run(self) -> AppMessageExportStats:
+        stats = AppMessageExportStats()
+
+        logger.info(
+            "export_app_messages: app_id=%s, start_from=%s, end_before=%s, dry_run=%s, cloud=%s, output_gz=%s",
+            self._app_id,
+            self._start_from,
+            self._end_before,
+            self._dry_run,
+            self._use_cloud_storage,
+            self.output_gz_name,
+        )
+
+        if self._dry_run:
+            for _ in self._iter_records_with_stats(stats):
+                pass
+            self._finalize_stats(stats)
+            return stats
+
+        if self._use_cloud_storage:
+            self._export_to_cloud(stats)
+        else:
+            self._export_to_local(stats)
+
+        self._finalize_stats(stats)
+        return stats
+
+    def iter_records(self) -> Generator[AppMessageExportRecord, None, None]:
+        for batch in self._iter_record_batches():
+            yield from batch
+
+    @staticmethod
+    def write_jsonl_gz(records: Iterable[AppMessageExportRecord], fileobj: BinaryIO) -> None:
+        with gzip.GzipFile(fileobj=fileobj, mode="wb") as gz:
+            for record in records:
+                gz.write(orjson.dumps(record.model_dump(mode="json")) + b"\n")
+
+    def _export_to_local(self, stats: AppMessageExportStats) -> None:
+        output_path = Path.cwd() / self.output_gz_name
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        with output_path.open("wb") as output_file:
+            self.write_jsonl_gz(self._iter_records_with_stats(stats), output_file)
+
+    def _export_to_cloud(self, stats: AppMessageExportStats) -> None:
+        with tempfile.SpooledTemporaryFile(max_size=64 * 1024 * 1024) as tmp:
+            self.write_jsonl_gz(self._iter_records_with_stats(stats), cast(BinaryIO, tmp))
+            tmp.seek(0)
+            data = tmp.read()
+
+        storage.save(self.output_gz_name, data)
+        logger.info("export_app_messages: uploaded %d bytes to cloud key=%s", len(data), self.output_gz_name)
+
+    def _iter_records_with_stats(self, stats: AppMessageExportStats) -> Generator[AppMessageExportRecord, None, None]:
+        for record in self.iter_records():
+            self._update_stats(stats, record)
+            yield record
+
+    @staticmethod
+    def _update_stats(stats: AppMessageExportStats, record: AppMessageExportRecord) -> None:
+        stats.total_messages += 1
+        if record.feedback:
+            stats.messages_with_feedback += 1
+            stats.total_feedbacks += len(record.feedback)
+
+    def _finalize_stats(self, stats: AppMessageExportStats) -> None:
+        if stats.total_messages == 0:
+            stats.batches = 0
+            return
+        stats.batches = (stats.total_messages + self._batch_size - 1) // self._batch_size
+
+    def _iter_record_batches(self) -> Generator[list[AppMessageExportRecord], None, None]:
+        cursor: tuple[datetime.datetime, str] | None = None
+        while True:
+            rows, cursor = self._fetch_batch(cursor)
+            if not rows:
+                break
+
+            message_ids = [str(row.id) for row in rows]
+            feedbacks_map = self._fetch_feedbacks(message_ids)
+            yield [self._build_record(row, feedbacks_map) for row in rows]
+
+    def _fetch_batch(
+        self, cursor: tuple[datetime.datetime, str] | None
+    ) -> tuple[list[Any], tuple[datetime.datetime, str] | None]:
+        with Session(db.engine, expire_on_commit=False) as session:
+            stmt = (
+                select(
+                    Message.id,
+                    Message.conversation_id,
+                    Message.query,
+                    Message.answer,
+                    Message._inputs,  # pyright: ignore[reportPrivateUsage]
+                    Message.message_metadata,
+                    Message.created_at,
+                )
+                .where(
+                    Message.app_id == self._app_id,
+                    Message.created_at < self._end_before,
+                )
+                .order_by(Message.created_at, Message.id)
+                .limit(self._batch_size)
+            )
+
+            if self._start_from:
+                stmt = stmt.where(Message.created_at >= self._start_from)
+
+            if cursor:
+                stmt = stmt.where(
+                    tuple_(Message.created_at, Message.id)
+                    > tuple_(
+                        sa.literal(cursor[0], type_=sa.DateTime()),
+                        sa.literal(cursor[1], type_=Message.id.type),
+                    )
+                )
+
+            rows = list(session.execute(stmt).all())
+
+        if not rows:
+            return [], cursor
+
+        last = rows[-1]
+        return rows, (last.created_at, last.id)
+
+    def _fetch_feedbacks(self, message_ids: list[str]) -> dict[str, list[AppMessageExportFeedback]]:
+        if not message_ids:
+            return {}
+
+        with Session(db.engine, expire_on_commit=False) as session:
+            stmt = (
+                select(MessageFeedback)
+                .where(
+                    MessageFeedback.message_id.in_(message_ids),
+                    MessageFeedback.from_source == "user",
+                )
+                .order_by(MessageFeedback.message_id, MessageFeedback.created_at)
+            )
+            feedbacks = list(session.scalars(stmt).all())
+
+        result: dict[str, list[AppMessageExportFeedback]] = defaultdict(list)
+        for feedback in feedbacks:
+            result[str(feedback.message_id)].append(AppMessageExportFeedback.model_validate(feedback.to_dict()))
+        return result
+
+    @staticmethod
+    def _build_record(row: Any, feedbacks_map: dict[str, list[AppMessageExportFeedback]]) -> AppMessageExportRecord:
+        retriever_resources: list[Any] = []
+        if row.message_metadata:
+            try:
+                metadata = json.loads(row.message_metadata)
+                value = metadata.get("retriever_resources", [])
+                if isinstance(value, list):
+                    retriever_resources = value
+            except (json.JSONDecodeError, TypeError):
+                pass
+
+        message_id = str(row.id)
+        return AppMessageExportRecord(
+            conversation_id=str(row.conversation_id),
+            message_id=message_id,
+            query=row.query,
+            answer=row.answer,
+            inputs=row._inputs if isinstance(row._inputs, dict) else {},
+            retriever_resources=retriever_resources,
+            feedback=feedbacks_map.get(message_id, []),
+        )

api/services/retention/conversation/messages_clean_service.py

@@ -12,6 +12,7 @@ from sqlalchemy.engine import CursorResult
 from sqlalchemy.orm import Session
 
 from extensions.ext_database import db
+from libs.datetime_utils import naive_utc_now
 from models.model import (
     App,
     AppAnnotationHitHistory,
@@ -142,7 +143,7 @@ class MessagesCleanService:
         if batch_size <= 0:
             raise ValueError(f"batch_size ({batch_size}) must be greater than 0")
 
-        end_before = datetime.datetime.now() - datetime.timedelta(days=days)
+        end_before = naive_utc_now() - datetime.timedelta(days=days)
 
         logger.info(
             "clean_messages: days=%s, end_before=%s, batch_size=%s, policy=%s",

api/services/website_service.py

@@ -124,7 +124,7 @@ class WebsiteService:
         if provider == "firecrawl":
             plugin_id = "langgenius/firecrawl_datasource"
         elif provider == "watercrawl":
-            plugin_id = "langgenius/watercrawl_datasource"
+            plugin_id = "watercrawl/watercrawl_datasource"
         elif provider == "jinareader":
             plugin_id = "langgenius/jina_datasource"
         else:

api/tasks/document_indexing_task.py

@@ -1,9 +1,10 @@
 import logging
 import time
-from collections.abc import Callable, Sequence
+from collections.abc import Sequence
+from typing import Any, Protocol
 
 import click
-from celery import shared_task
+from celery import current_app, shared_task
 
 from configs import dify_config
 from core.db.session_factory import session_factory
@@ -19,6 +20,12 @@ from tasks.generate_summary_index_task import generate_summary_index_task
 logger = logging.getLogger(__name__)
 
 
+class CeleryTaskLike(Protocol):
+    def delay(self, *args: Any, **kwargs: Any) -> Any: ...
+
+    def apply_async(self, *args: Any, **kwargs: Any) -> Any: ...
+
+
 @shared_task(queue="dataset")
 def document_indexing_task(dataset_id: str, document_ids: list):
     """
@@ -179,8 +186,8 @@ def _document_indexing(dataset_id: str, document_ids: Sequence[str]):
 
 
 def _document_indexing_with_tenant_queue(
-    tenant_id: str, dataset_id: str, document_ids: Sequence[str], task_func: Callable[[str, str, Sequence[str]], None]
-):
+    tenant_id: str, dataset_id: str, document_ids: Sequence[str], task_func: CeleryTaskLike
+) -> None:
     try:
         _document_indexing(dataset_id, document_ids)
     except Exception:
@@ -201,16 +208,20 @@ def _document_indexing_with_tenant_queue(
         logger.info("document indexing tenant isolation queue %s next tasks: %s", tenant_id, next_tasks)
 
         if next_tasks:
-            for next_task in next_tasks:
-                document_task = DocumentTask(**next_task)
-                # Process the next waiting task
-                # Keep the flag set to indicate a task is running
-                tenant_isolated_task_queue.set_task_waiting_time()
-                task_func.delay(  # type: ignore
-                    tenant_id=document_task.tenant_id,
-                    dataset_id=document_task.dataset_id,
-                    document_ids=document_task.document_ids,
-                )
+            with current_app.producer_or_acquire() as producer:  # type: ignore
+                for next_task in next_tasks:
+                    document_task = DocumentTask(**next_task)
+                    # Keep the flag set to indicate a task is running
+                    tenant_isolated_task_queue.set_task_waiting_time()
+                    task_func.apply_async(
+                        kwargs={
+                            "tenant_id": document_task.tenant_id,
+                            "dataset_id": document_task.dataset_id,
+                            "document_ids": document_task.document_ids,
+                        },
+                        producer=producer,
+                    )
+
         else:
             # No more waiting tasks, clear the flag
             tenant_isolated_task_queue.delete_task_key()

api/tasks/rag_pipeline/rag_pipeline_run_task.py

@@ -3,12 +3,13 @@ import json
 import logging
 import time
 import uuid
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from concurrent.futures import ThreadPoolExecutor
+from itertools import islice
 from typing import Any
 
 import click
-from celery import shared_task  # type: ignore
+from celery import group, shared_task
 from flask import current_app, g
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -27,6 +28,11 @@ from services.file_service import FileService
 logger = logging.getLogger(__name__)
 
 
+def chunked(iterable: Sequence, size: int):
+    it = iter(iterable)
+    return iter(lambda: list(islice(it, size)), [])
+
+
 @shared_task(queue="pipeline")
 def rag_pipeline_run_task(
     rag_pipeline_invoke_entities_file_id: str,
@@ -83,16 +89,24 @@ def rag_pipeline_run_task(
         logger.info("rag pipeline tenant isolation queue %s next files: %s", tenant_id, next_file_ids)
 
         if next_file_ids:
-            for next_file_id in next_file_ids:
-                # Process the next waiting task
-                # Keep the flag set to indicate a task is running
-                tenant_isolated_task_queue.set_task_waiting_time()
-                rag_pipeline_run_task.delay(  # type: ignore
-                    rag_pipeline_invoke_entities_file_id=next_file_id.decode("utf-8")
-                    if isinstance(next_file_id, bytes)
-                    else next_file_id,
-                    tenant_id=tenant_id,
-                )
+            for batch in chunked(next_file_ids, 100):
+                jobs = []
+                for next_file_id in batch:
+                    tenant_isolated_task_queue.set_task_waiting_time()
+
+                    file_id = (
+                        next_file_id.decode("utf-8") if isinstance(next_file_id, (bytes, bytearray)) else next_file_id
+                    )
+
+                    jobs.append(
+                        rag_pipeline_run_task.s(
+                            rag_pipeline_invoke_entities_file_id=file_id,
+                            tenant_id=tenant_id,
+                        )
+                    )
+
+                if jobs:
+                    group(jobs).apply_async()
         else:
             # No more waiting tasks, clear the flag
             tenant_isolated_task_queue.delete_task_key()

api/tests/integration_tests/workflow/nodes/test_llm.py

@@ -11,6 +11,7 @@ from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.node_events import StreamCompletedEvent
 from dify_graph.nodes.llm.node import LLMNode
 from dify_graph.nodes.llm.protocols import CredentialsProvider, ModelFactory
+from dify_graph.nodes.protocols import HttpClientProtocol
 from dify_graph.runtime import GraphRuntimeState, VariablePool
 from dify_graph.system_variable import SystemVariable
 from extensions.ext_database import db
@@ -74,6 +75,7 @@ def init_llm_node(config: dict) -> LLMNode:
         credentials_provider=MagicMock(spec=CredentialsProvider),
         model_factory=MagicMock(spec=ModelFactory),
         model_instance=MagicMock(spec=ModelInstance),
+        http_client=MagicMock(spec=HttpClientProtocol),
     )
 
     return node

api/tests/integration_tests/workflow/nodes/test_tool.py

@@ -8,6 +8,7 @@ from core.workflow.node_factory import DifyNodeFactory
 from dify_graph.enums import WorkflowNodeExecutionStatus
 from dify_graph.graph import Graph
 from dify_graph.node_events import StreamCompletedEvent
+from dify_graph.nodes.protocols import ToolFileManagerProtocol
 from dify_graph.nodes.tool.tool_node import ToolNode
 from dify_graph.runtime import GraphRuntimeState, VariablePool
 from dify_graph.system_variable import SystemVariable
@@ -55,11 +56,14 @@ def init_tool_node(config: dict):
 
     graph = Graph.init(graph_config=graph_config, node_factory=node_factory)
 
+    tool_file_manager_factory = MagicMock(spec=ToolFileManagerProtocol)
+
     node = ToolNode(
         id=str(uuid.uuid4()),
         config=config,
         graph_init_params=init_params,
         graph_runtime_state=graph_runtime_state,
+        tool_file_manager_factory=tool_file_manager_factory,
     )
     return node
 

api/tests/test_containers_integration_tests/core/rag/retrieval/test_dataset_retrieval_integration.py

@@ -8,6 +8,7 @@ from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
 from dify_graph.repositories.rag_retrieval_protocol import KnowledgeRetrievalRequest
 from models.dataset import Dataset, Document
 from services.account_service import AccountService, TenantService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestGetAvailableDatasetsIntegration:
@@ -22,7 +23,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -83,7 +84,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -136,7 +137,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -189,7 +190,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -252,7 +253,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -286,7 +287,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account1, name=fake.company())
         tenant1 = account1.current_tenant
@@ -295,7 +296,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account2, name=fake.company())
         tenant2 = account2.current_tenant
@@ -362,7 +363,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -384,7 +385,7 @@ class TestGetAvailableDatasetsIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -445,7 +446,7 @@ class TestKnowledgeRetrievalIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -513,7 +514,7 @@ class TestKnowledgeRetrievalIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -561,7 +562,7 @@ class TestKnowledgeRetrievalIntegration:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/helpers/__init__.py

@@ -1 +1,24 @@
 """Helper utilities for integration tests."""
+
+import re
+
+
+def generate_valid_password(fake, length: int = 12) -> str:
+    """Generate a password that always satisfies the project's password validation rules.
+
+    The password validation rule in ``api/libs/password.py`` requires passwords to
+    contain **both letters and digits** with a minimum length of 8:
+
+        ``^(?=.*[a-zA-Z])(?=.*\\d).{8,}$``
+
+    ``Faker.password()`` does **not** guarantee that the generated password will
+    contain both character types, which can cause intermittent test failures.
+
+    This helper re-generates until the result is valid (typically first attempt).
+    """
+    for _ in range(100):
+        pwd = fake.password(length=length)
+        if re.search(r"[a-zA-Z]", pwd) and re.search(r"\d", pwd):
+            return pwd
+    # Fallback: should never be reached in practice
+    return fake.password(length=max(length - 2, 6)) + "a1"

api/tests/test_containers_integration_tests/services/test_account_service.py

@@ -20,6 +20,7 @@ from services.errors.account import (
     TenantNotFoundError,
 )
 from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkspacesLimitExceededError
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAccountService:
@@ -53,7 +54,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -133,7 +134,7 @@ class TestAccountService:
                 email=email,
                 name=name,
                 interface_language="en-US",
-                password=fake.password(length=12),
+                password=generate_valid_password(fake),
             )
 
     def test_create_account_email_in_freeze(
@@ -145,7 +146,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = True
@@ -169,7 +170,7 @@ class TestAccountService:
         """
         fake = Faker()
         email = fake.email()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         with pytest.raises(AccountPasswordError):
             AccountService.authenticate(email, password)
 
@@ -180,7 +181,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -208,8 +209,8 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        correct_password = fake.password(length=12)
-        wrong_password = fake.password(length=12)
+        correct_password = generate_valid_password(fake)
+        wrong_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -234,7 +235,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        new_password = fake.password(length=12)
+        new_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -267,7 +268,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -297,8 +298,8 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        old_password = fake.password(length=12)
-        new_password = fake.password(length=12)
+        old_password = generate_valid_password(fake)
+        new_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -327,9 +328,9 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        old_password = fake.password(length=12)
-        wrong_password = fake.password(length=12)
-        new_password = fake.password(length=12)
+        old_password = generate_valid_password(fake)
+        wrong_password = generate_valid_password(fake)
+        new_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -354,7 +355,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        old_password = fake.password(length=12)
+        old_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -378,7 +379,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies[
@@ -412,7 +413,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies[
@@ -437,7 +438,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies[
@@ -535,7 +536,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -563,7 +564,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         updated_name = fake.name()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -592,7 +593,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -615,7 +616,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         ip_address = fake.ipv4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -645,7 +646,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         ip_address = fake.ipv4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -684,7 +685,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -714,7 +715,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -747,7 +748,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -792,7 +793,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -825,7 +826,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -864,7 +865,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -892,7 +893,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -926,7 +927,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -957,7 +958,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -997,7 +998,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -1043,7 +1044,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -1080,7 +1081,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -1110,7 +1111,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -1139,7 +1140,7 @@ class TestAccountService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         wrong_code = fake.numerify(text="######")
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -1259,7 +1260,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1291,10 +1292,10 @@ class TestTenantService:
         tenant_name = fake.company()
         email1 = fake.email()
         name1 = fake.name()
-        password1 = fake.password(length=12)
+        password1 = generate_valid_password(fake)
         email2 = fake.email()
         name2 = fake.name()
-        password2 = fake.password(length=12)
+        password2 = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1332,7 +1333,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1364,7 +1365,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant1_name = fake.company()
         tenant2_name = fake.company()
         # Setup mocks
@@ -1403,7 +1404,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies[
@@ -1441,7 +1442,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1466,7 +1467,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant1_name = fake.company()
         tenant2_name = fake.company()
         # Setup mocks
@@ -1507,7 +1508,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1534,7 +1535,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         tenant_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies[
@@ -1562,10 +1563,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         admin_email = fake.email()
         admin_name = fake.name()
-        admin_password = fake.password(length=12)
+        admin_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1631,7 +1632,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1664,10 +1665,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         member_email = fake.email()
         member_name = fake.name()
-        member_password = fake.password(length=12)
+        member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1705,7 +1706,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         invalid_action = "invalid_action_that_doesnt_exist"
         # Setup mocks
         mock_external_service_dependencies[
@@ -1738,7 +1739,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1770,10 +1771,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         member_email = fake.email()
         member_name = fake.name()
-        member_password = fake.password(length=12)
+        member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1829,7 +1830,7 @@ class TestTenantService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1861,10 +1862,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         non_member_email = fake.email()
         non_member_name = fake.name()
-        non_member_password = fake.password(length=12)
+        non_member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1900,10 +1901,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         member_email = fake.email()
         member_name = fake.name()
-        member_password = fake.password(length=12)
+        member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -1949,10 +1950,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         member_email = fake.email()
         member_name = fake.name()
-        member_password = fake.password(length=12)
+        member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -2006,10 +2007,10 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         member_email = fake.email()
         member_name = fake.name()
-        member_password = fake.password(length=12)
+        member_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -2071,7 +2072,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         workspace_name = fake.company()
         # Setup mocks
         mock_external_service_dependencies[
@@ -2110,7 +2111,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         existing_tenant_name = fake.company()
         new_workspace_name = fake.company()
         # Setup mocks
@@ -2151,7 +2152,7 @@ class TestTenantService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         workspace_name = fake.company()
         # Setup mocks to disable workspace creation
         mock_external_service_dependencies[
@@ -2178,13 +2179,13 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         admin_email = fake.email()
         admin_name = fake.name()
-        admin_password = fake.password(length=12)
+        admin_password = generate_valid_password(fake)
         normal_email = fake.email()
         normal_name = fake.name()
-        normal_password = fake.password(length=12)
+        normal_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -2244,13 +2245,13 @@ class TestTenantService:
         tenant_name = fake.company()
         owner_email = fake.email()
         owner_name = fake.name()
-        owner_password = fake.password(length=12)
+        owner_password = generate_valid_password(fake)
         operator_email = fake.email()
         operator_name = fake.name()
-        operator_password = fake.password(length=12)
+        operator_password = generate_valid_password(fake)
         normal_email = fake.email()
         normal_name = fake.name()
-        normal_password = fake.password(length=12)
+        normal_password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies[
             "feature_service"
@@ -2351,7 +2352,7 @@ class TestRegisterService:
         fake = Faker()
         admin_email = fake.email()
         admin_name = fake.name()
-        admin_password = fake.password(length=12)
+        admin_password = generate_valid_password(fake)
         ip_address = fake.ipv4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2399,7 +2400,7 @@ class TestRegisterService:
         fake = Faker()
         admin_email = fake.email()
         admin_name = fake.name()
-        admin_password = fake.password(length=12)
+        admin_password = generate_valid_password(fake)
         ip_address = fake.ipv4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2440,7 +2441,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2531,7 +2532,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2576,7 +2577,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2614,7 +2615,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2653,7 +2654,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2690,7 +2691,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         inviter_email = fake.email()
         inviter_name = fake.name()
-        inviter_password = fake.password(length=12)
+        inviter_password = generate_valid_password(fake)
         new_member_email = fake.email()
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
@@ -2760,10 +2761,10 @@ class TestRegisterService:
         tenant_name = fake.company()
         inviter_email = fake.email()
         inviter_name = fake.name()
-        inviter_password = fake.password(length=12)
+        inviter_password = generate_valid_password(fake)
         existing_member_email = fake.email()
         existing_member_name = fake.name()
-        existing_member_password = fake.password(length=12)
+        existing_member_password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2824,10 +2825,10 @@ class TestRegisterService:
         tenant_name = fake.company()
         inviter_email = fake.email()
         inviter_name = fake.name()
-        inviter_password = fake.password(length=12)
+        inviter_password = generate_valid_password(fake)
         existing_pending_member_email = fake.email()
         existing_pending_member_name = fake.name()
-        existing_pending_member_password = fake.password(length=12)
+        existing_pending_member_password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2914,10 +2915,10 @@ class TestRegisterService:
         tenant_name = fake.company()
         inviter_email = fake.email()
         inviter_name = fake.name()
-        inviter_password = fake.password(length=12)
+        inviter_password = generate_valid_password(fake)
         already_in_tenant_email = fake.email()
         already_in_tenant_name = fake.name()
-        already_in_tenant_password = fake.password(length=12)
+        already_in_tenant_password = generate_valid_password(fake)
         language = fake.random_element(elements=("en-US", "zh-CN"))
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -2967,7 +2968,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -3011,7 +3012,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -3058,7 +3059,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -3101,7 +3102,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -3144,7 +3145,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
         mock_external_service_dependencies["billing_service"].is_email_in_freeze.return_value = False
@@ -3212,7 +3213,7 @@ class TestRegisterService:
         fake = Faker()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         invalid_tenant_id = fake.uuid4()
         token = fake.uuid4()
         # Setup mocks
@@ -3263,7 +3264,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         token = fake.uuid4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True
@@ -3313,7 +3314,7 @@ class TestRegisterService:
         tenant_name = fake.company()
         email = fake.email()
         name = fake.name()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         token = fake.uuid4()
         # Setup mocks
         mock_external_service_dependencies["feature_service"].get_system_features.return_value.is_allow_register = True

api/tests/test_containers_integration_tests/services/test_agent_service.py

@@ -11,6 +11,7 @@ from models.model import AppModelConfig, Conversation, EndUser, Message, Message
 from services.account_service import AccountService, TenantService
 from services.agent_service import AgentService
 from services.app_service import AppService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAgentService:
@@ -111,7 +112,7 @@ class TestAgentService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_annotation_service.py

@@ -9,6 +9,7 @@ from models import Account
 from models.model import MessageAnnotation
 from services.annotation_service import AppAnnotationService
 from services.app_service import AppService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAnnotationService:
@@ -78,7 +79,7 @@ class TestAnnotationService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_api_based_extension_service.py

@@ -7,6 +7,7 @@ from sqlalchemy.orm import Session
 from models.api_based_extension import APIBasedExtension
 from services.account_service import AccountService, TenantService
 from services.api_based_extension_service import APIBasedExtensionService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAPIBasedExtensionService:
@@ -55,7 +56,7 @@ class TestAPIBasedExtensionService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_app_dsl_service.py

@@ -9,6 +9,7 @@ from models.model import App, AppModelConfig
 from services.account_service import AccountService, TenantService
 from services.app_dsl_service import AppDslService, ImportMode, ImportStatus
 from services.app_service import AppService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAppDslService:
@@ -89,7 +90,7 @@ class TestAppDslService:
                 email=fake.email(),
                 name=fake.name(),
                 interface_language="en-US",
-                password=fake.password(length=12),
+                password=generate_valid_password(fake),
             )
             TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
             tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_app_generate_service.py

@@ -10,6 +10,7 @@ from models.model import EndUser
 from models.workflow import Workflow
 from services.app_generate_service import AppGenerateService
 from services.errors.app import WorkflowIdFormatError, WorkflowNotFoundError
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestAppGenerateService:
@@ -147,7 +148,7 @@ class TestAppGenerateService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_app_service.py

@@ -8,6 +8,7 @@ from constants.model_template import default_app_templates
 from models import Account
 from models.model import App, Site
 from services.account_service import AccountService, TenantService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 # Delay import of AppService to avoid circular dependency
 # from services.app_service import AppService
@@ -56,7 +57,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -112,7 +113,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -155,7 +156,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -203,7 +204,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -259,7 +260,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -334,7 +335,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -404,7 +405,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -473,7 +474,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -526,7 +527,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -585,7 +586,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -645,7 +646,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -705,7 +706,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -756,7 +757,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -808,7 +809,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -868,7 +869,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -907,7 +908,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -947,7 +948,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -997,7 +998,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -1039,7 +1040,7 @@ class TestAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_message_export_service.py

@@ -0,0 +1,233 @@
+import datetime
+import json
+import uuid
+from decimal import Decimal
+
+import pytest
+from sqlalchemy.orm import Session
+
+from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
+from models.model import (
+    App,
+    AppAnnotationHitHistory,
+    Conversation,
+    DatasetRetrieverResource,
+    Message,
+    MessageAgentThought,
+    MessageAnnotation,
+    MessageChain,
+    MessageFeedback,
+    MessageFile,
+)
+from models.web import SavedMessage
+from services.retention.conversation.message_export_service import AppMessageExportService, AppMessageExportStats
+
+
+class TestAppMessageExportServiceIntegration:
+    @pytest.fixture(autouse=True)
+    def cleanup_database(self, db_session_with_containers: Session):
+        yield
+        db_session_with_containers.query(DatasetRetrieverResource).delete()
+        db_session_with_containers.query(AppAnnotationHitHistory).delete()
+        db_session_with_containers.query(SavedMessage).delete()
+        db_session_with_containers.query(MessageFile).delete()
+        db_session_with_containers.query(MessageAgentThought).delete()
+        db_session_with_containers.query(MessageChain).delete()
+        db_session_with_containers.query(MessageAnnotation).delete()
+        db_session_with_containers.query(MessageFeedback).delete()
+        db_session_with_containers.query(Message).delete()
+        db_session_with_containers.query(Conversation).delete()
+        db_session_with_containers.query(App).delete()
+        db_session_with_containers.query(TenantAccountJoin).delete()
+        db_session_with_containers.query(Tenant).delete()
+        db_session_with_containers.query(Account).delete()
+        db_session_with_containers.commit()
+
+    @staticmethod
+    def _create_app_context(session: Session) -> tuple[App, Conversation]:
+        account = Account(
+            email=f"test-{uuid.uuid4()}@example.com",
+            name="tester",
+            interface_language="en-US",
+            status="active",
+        )
+        session.add(account)
+        session.flush()
+
+        tenant = Tenant(name=f"tenant-{uuid.uuid4()}", status="normal")
+        session.add(tenant)
+        session.flush()
+
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER,
+            current=True,
+        )
+        session.add(join)
+        session.flush()
+
+        app = App(
+            tenant_id=tenant.id,
+            name="export-app",
+            description="integration test app",
+            mode="chat",
+            enable_site=True,
+            enable_api=True,
+            api_rpm=60,
+            api_rph=3600,
+            is_demo=False,
+            is_public=False,
+            created_by=account.id,
+            updated_by=account.id,
+        )
+        session.add(app)
+        session.flush()
+
+        conversation = Conversation(
+            app_id=app.id,
+            app_model_config_id=str(uuid.uuid4()),
+            model_provider="openai",
+            model_id="gpt-4o-mini",
+            mode="chat",
+            name="conv",
+            inputs={"seed": 1},
+            status="normal",
+            from_source="api",
+            from_end_user_id=str(uuid.uuid4()),
+        )
+        session.add(conversation)
+        session.commit()
+        return app, conversation
+
+    @staticmethod
+    def _create_message(
+        session: Session,
+        app: App,
+        conversation: Conversation,
+        created_at: datetime.datetime,
+        *,
+        query: str,
+        answer: str,
+        inputs: dict,
+        message_metadata: str | None,
+    ) -> Message:
+        message = Message(
+            app_id=app.id,
+            conversation_id=conversation.id,
+            model_provider="openai",
+            model_id="gpt-4o-mini",
+            inputs=inputs,
+            query=query,
+            answer=answer,
+            message=[{"role": "assistant", "content": answer}],
+            message_tokens=10,
+            message_unit_price=Decimal("0.001"),
+            answer_tokens=20,
+            answer_unit_price=Decimal("0.002"),
+            total_price=Decimal("0.003"),
+            currency="USD",
+            message_metadata=message_metadata,
+            from_source="api",
+            from_end_user_id=conversation.from_end_user_id,
+            created_at=created_at,
+        )
+        session.add(message)
+        session.flush()
+        return message
+
+    def test_iter_records_with_stats(self, db_session_with_containers: Session):
+        app, conversation = self._create_app_context(db_session_with_containers)
+
+        first_inputs = {
+            "plain": "v1",
+            "nested": {"a": 1, "b": [1, {"x": True}]},
+            "list": ["x", 2, {"y": "z"}],
+        }
+        second_inputs = {"other": "value", "items": [1, 2, 3]}
+
+        base_time = datetime.datetime(2026, 2, 25, 10, 0, 0)
+        first_message = self._create_message(
+            db_session_with_containers,
+            app,
+            conversation,
+            created_at=base_time,
+            query="q1",
+            answer="a1",
+            inputs=first_inputs,
+            message_metadata=json.dumps({"retriever_resources": [{"dataset_id": "ds-1"}]}),
+        )
+        second_message = self._create_message(
+            db_session_with_containers,
+            app,
+            conversation,
+            created_at=base_time + datetime.timedelta(minutes=1),
+            query="q2",
+            answer="a2",
+            inputs=second_inputs,
+            message_metadata=None,
+        )
+
+        user_feedback_1 = MessageFeedback(
+            app_id=app.id,
+            conversation_id=conversation.id,
+            message_id=first_message.id,
+            rating="like",
+            from_source="user",
+            content="first",
+            from_end_user_id=conversation.from_end_user_id,
+        )
+        user_feedback_2 = MessageFeedback(
+            app_id=app.id,
+            conversation_id=conversation.id,
+            message_id=first_message.id,
+            rating="dislike",
+            from_source="user",
+            content="second",
+            from_end_user_id=conversation.from_end_user_id,
+        )
+        admin_feedback = MessageFeedback(
+            app_id=app.id,
+            conversation_id=conversation.id,
+            message_id=first_message.id,
+            rating="like",
+            from_source="admin",
+            content="should-be-filtered",
+            from_account_id=str(uuid.uuid4()),
+        )
+        db_session_with_containers.add_all([user_feedback_1, user_feedback_2, admin_feedback])
+        user_feedback_1.created_at = base_time + datetime.timedelta(minutes=2)
+        user_feedback_2.created_at = base_time + datetime.timedelta(minutes=3)
+        admin_feedback.created_at = base_time + datetime.timedelta(minutes=4)
+        db_session_with_containers.commit()
+
+        service = AppMessageExportService(
+            app_id=app.id,
+            start_from=base_time - datetime.timedelta(minutes=1),
+            end_before=base_time + datetime.timedelta(minutes=10),
+            filename="unused",
+            batch_size=1,
+            dry_run=True,
+        )
+        stats = AppMessageExportStats()
+        records = list(service._iter_records_with_stats(stats))
+        service._finalize_stats(stats)
+
+        assert len(records) == 2
+        assert records[0].message_id == first_message.id
+        assert records[1].message_id == second_message.id
+
+        assert records[0].inputs == first_inputs
+        assert records[1].inputs == second_inputs
+
+        assert records[0].retriever_resources == [{"dataset_id": "ds-1"}]
+        assert records[1].retriever_resources == []
+
+        assert [feedback.rating for feedback in records[0].feedback] == ["like", "dislike"]
+        assert [feedback.content for feedback in records[0].feedback] == ["first", "second"]
+        assert records[1].feedback == []
+
+        assert stats.batches == 2
+        assert stats.total_messages == 2
+        assert stats.messages_with_feedback == 1
+        assert stats.total_feedbacks == 2

api/tests/test_containers_integration_tests/services/test_message_service.py

@@ -13,6 +13,7 @@ from services.errors.message import (
     SuggestedQuestionsAfterAnswerDisabledError,
 )
 from services.message_service import MessageService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestMessageService:
@@ -95,7 +96,7 @@ class TestMessageService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -633,7 +634,7 @@ class TestMessageService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(other_account, name=fake.company())
 

api/tests/test_containers_integration_tests/services/test_saved_message_service.py

@@ -8,6 +8,7 @@ from models.model import EndUser, Message
 from models.web import SavedMessage
 from services.app_service import AppService
 from services.saved_message_service import SavedMessageService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestSavedMessageService:
@@ -64,7 +65,7 @@ class TestSavedMessageService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_trigger_provider_service.py

@@ -10,6 +10,7 @@ from core.trigger.entities.entities import Subscription as TriggerSubscriptionEn
 from models.provider_ids import TriggerProviderID
 from models.trigger import TriggerSubscription
 from services.trigger.trigger_provider_service import TriggerProviderService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestTriggerProviderService:
@@ -75,7 +76,7 @@ class TestTriggerProviderService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_web_conversation_service.py

@@ -12,6 +12,7 @@ from models.web import PinnedConversation
 from services.account_service import AccountService, TenantService
 from services.app_service import AppService
 from services.web_conversation_service import WebConversationService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWebConversationService:
@@ -69,7 +70,7 @@ class TestWebConversationService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_webapp_auth_service.py

@@ -12,6 +12,7 @@ from models import Account, AccountStatus, Tenant, TenantAccountJoin, TenantAcco
 from models.model import App, Site
 from services.errors.account import AccountLoginError, AccountNotFoundError, AccountPasswordError
 from services.webapp_auth_service import WebAppAuthService, WebAppAuthType
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWebAppAuthService:
@@ -109,7 +110,7 @@ class TestWebAppAuthService:
             tuple: (account, tenant, password) - Created account, tenant and password
         """
         fake = Faker()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
 
         # Create account with password
         import uuid
@@ -272,7 +273,7 @@ class TestWebAppAuthService:
         """
         # Arrange: Create banned account
         fake = Faker()
-        password = fake.password(length=12)
+        password = generate_valid_password(fake)
         unique_email = f"test_{uuid.uuid4().hex[:8]}@example.com"
 
         account = Account(

api/tests/test_containers_integration_tests/services/test_webhook_service.py

@@ -13,6 +13,7 @@ from models.trigger import AppTrigger, WorkflowWebhookTrigger
 from models.workflow import Workflow
 from services.account_service import AccountService, TenantService
 from services.trigger.webhook_service import WebhookService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWebhookService:
@@ -60,7 +61,7 @@ class TestWebhookService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_workflow_app_service.py

@@ -15,6 +15,7 @@ from services.account_service import AccountService, TenantService
 # Delay import of AppService to avoid circular dependency
 # from services.app_service import AppService
 from services.workflow_app_service import WorkflowAppService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWorkflowAppService:
@@ -72,7 +73,7 @@ class TestWorkflowAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -120,7 +121,7 @@ class TestWorkflowAppService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/test_workflow_run_service.py

@@ -15,6 +15,7 @@ from models.workflow import WorkflowRun
 from services.account_service import AccountService, TenantService
 from services.app_service import AppService
 from services.workflow_run_service import WorkflowRunService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWorkflowRunService:
@@ -72,7 +73,7 @@ class TestWorkflowRunService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/services/tools/test_workflow_tools_manage_service.py

@@ -13,6 +13,7 @@ from models.workflow import Workflow as WorkflowModel
 from services.account_service import AccountService, TenantService
 from services.app_service import AppService
 from services.tools.workflow_tools_manage_service import WorkflowToolManageService
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestWorkflowToolManageService:
@@ -87,7 +88,7 @@ class TestWorkflowToolManageService:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/tasks/test_clean_notion_document_task.py

@@ -15,6 +15,7 @@ from faker import Faker
 from models.dataset import Dataset, Document, DocumentSegment
 from services.account_service import AccountService, TenantService
 from tasks.clean_notion_document_task import clean_notion_document_task
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestCleanNotionDocumentTask:
@@ -76,7 +77,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -208,7 +209,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -252,7 +253,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -345,7 +346,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -431,7 +432,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -546,7 +547,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -642,7 +643,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -724,7 +725,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -834,7 +835,7 @@ class TestCleanNotionDocumentTask:
                 email=fake.email(),
                 name=fake.name(),
                 interface_language="en-US",
-                password=fake.password(length=12),
+                password=generate_valid_password(fake),
             )
             TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
             tenant = account.current_tenant
@@ -951,7 +952,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant
@@ -1054,7 +1055,7 @@ class TestCleanNotionDocumentTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/tasks/test_dataset_indexing_task.py

@@ -322,11 +322,14 @@ class TestDatasetIndexingTaskIntegration:
             _document_indexing_with_tenant_queue(dataset.tenant_id, dataset.id, document_ids, task_dispatch_spy)
 
         # Assert
-        task_dispatch_spy.delay.assert_called_once_with(
-            tenant_id=next_task["tenant_id"],
-            dataset_id=next_task["dataset_id"],
-            document_ids=next_task["document_ids"],
-        )
+        # apply_async is used by implementation; assert it was called once with expected kwargs
+        assert task_dispatch_spy.apply_async.call_count == 1
+        call_kwargs = task_dispatch_spy.apply_async.call_args.kwargs.get("kwargs", {})
+        assert call_kwargs == {
+            "tenant_id": next_task["tenant_id"],
+            "dataset_id": next_task["dataset_id"],
+            "document_ids": next_task["document_ids"],
+        }
         set_waiting_spy.assert_called_once()
         delete_key_spy.assert_not_called()
 
@@ -352,7 +355,7 @@ class TestDatasetIndexingTaskIntegration:
             _document_indexing_with_tenant_queue(dataset.tenant_id, dataset.id, document_ids, task_dispatch_spy)
 
         # Assert
-        task_dispatch_spy.delay.assert_not_called()
+        task_dispatch_spy.apply_async.assert_not_called()
         delete_key_spy.assert_called_once()
 
     def test_validation_failure_sets_error_status_when_vector_space_at_limit(
@@ -447,7 +450,7 @@ class TestDatasetIndexingTaskIntegration:
             _document_indexing_with_tenant_queue(dataset.tenant_id, dataset.id, document_ids, task_dispatch_spy)
 
         # Assert
-        task_dispatch_spy.delay.assert_called_once()
+        task_dispatch_spy.apply_async.assert_called_once()
 
     def test_sessions_close_on_successful_indexing(
         self,
@@ -534,7 +537,7 @@ class TestDatasetIndexingTaskIntegration:
             _document_indexing_with_tenant_queue(dataset.tenant_id, dataset.id, document_ids, task_dispatch_spy)
 
         # Assert
-        assert task_dispatch_spy.delay.call_count == concurrency_limit
+        assert task_dispatch_spy.apply_async.call_count == concurrency_limit
         assert set_waiting_spy.call_count == concurrency_limit
 
     def test_task_queue_fifo_ordering(self, db_session_with_containers, patched_external_dependencies):
@@ -565,9 +568,10 @@ class TestDatasetIndexingTaskIntegration:
             _document_indexing_with_tenant_queue(dataset.tenant_id, dataset.id, document_ids, task_dispatch_spy)
 
         # Assert
-        assert task_dispatch_spy.delay.call_count == 3
+        assert task_dispatch_spy.apply_async.call_count == 3
         for index, expected_task in enumerate(ordered_tasks):
-            assert task_dispatch_spy.delay.call_args_list[index].kwargs["document_ids"] == expected_task["document_ids"]
+            call_kwargs = task_dispatch_spy.apply_async.call_args_list[index].kwargs.get("kwargs", {})
+            assert call_kwargs.get("document_ids") == expected_task["document_ids"]
 
     def test_billing_disabled_skips_limit_checks(self, db_session_with_containers, patched_external_dependencies):
         """Skip limit checks when billing feature is disabled."""

api/tests/test_containers_integration_tests/tasks/test_deal_dataset_vector_index_task.py

@@ -15,6 +15,7 @@ from faker import Faker
 from models.dataset import Dataset, Document, DocumentSegment
 from services.account_service import AccountService, TenantService
 from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task
+from tests.test_containers_integration_tests.helpers import generate_valid_password
 
 
 class TestDealDatasetVectorIndexTask:
@@ -61,7 +62,7 @@ class TestDealDatasetVectorIndexTask:
             email=fake.email(),
             name=fake.name(),
             interface_language="en-US",
-            password=fake.password(length=12),
+            password=generate_valid_password(fake),
         )
         TenantService.create_owner_tenant_if_not_exist(account, name=fake.company())
         tenant = account.current_tenant

api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py

@@ -762,11 +762,12 @@ class TestDocumentIndexingTasks:
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify task function was called for each waiting task
-        assert mock_task_func.delay.call_count == 1
+        assert mock_task_func.apply_async.call_count == 1
 
         # Verify correct parameters for each call
-        calls = mock_task_func.delay.call_args_list
-        assert calls[0][1] == {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": ["waiting-doc-1"]}
+        calls = mock_task_func.apply_async.call_args_list
+        sent_kwargs = calls[0][1]["kwargs"]
+        assert sent_kwargs == {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": ["waiting-doc-1"]}
 
         # Verify queue is empty after processing (tasks were pulled)
         remaining_tasks = queue.pull_tasks(count=10)  # Pull more than we added
@@ -830,11 +831,15 @@ class TestDocumentIndexingTasks:
             assert updated_document.processing_started_at is not None
 
         # Verify waiting task was still processed despite core processing error
-        mock_task_func.delay.assert_called_once()
+        mock_task_func.apply_async.assert_called_once()
 
         # Verify correct parameters for the call
-        call = mock_task_func.delay.call_args
-        assert call[1] == {"tenant_id": tenant_id, "dataset_id": dataset_id, "document_ids": ["waiting-doc-1"]}
+        call = mock_task_func.apply_async.call_args
+        assert call[1]["kwargs"] == {
+            "tenant_id": tenant_id,
+            "dataset_id": dataset_id,
+            "document_ids": ["waiting-doc-1"],
+        }
 
         # Verify queue is empty after processing (task was pulled)
         remaining_tasks = queue.pull_tasks(count=10)
@@ -896,9 +901,13 @@ class TestDocumentIndexingTasks:
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify only tenant1's waiting task was processed
-        mock_task_func.delay.assert_called_once()
-        call = mock_task_func.delay.call_args
-        assert call[1] == {"tenant_id": tenant1_id, "dataset_id": dataset1_id, "document_ids": ["tenant1-doc-1"]}
+        mock_task_func.apply_async.assert_called_once()
+        call = mock_task_func.apply_async.call_args
+        assert call[1]["kwargs"] == {
+            "tenant_id": tenant1_id,
+            "dataset_id": dataset1_id,
+            "document_ids": ["tenant1-doc-1"],
+        }
 
         # Verify tenant1's queue is empty
         remaining_tasks1 = queue1.pull_tasks(count=10)

api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py

@@ -1,6 +1,6 @@
 import json
 import uuid
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import pytest
 from faker import Faker
@@ -388,8 +388,10 @@ class TestRagPipelineRunTasks:
         # Set the task key to indicate there are waiting tasks (legacy behavior)
         redis_client.set(legacy_task_key, 1, ex=60 * 60)
 
-        # Mock the task function calls
-        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+        # Mock the Celery group scheduling used by the implementation
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
+            mock_group.return_value.apply_async = MagicMock()
+
             # Act: Execute the priority task with new code but legacy queue data
             rag_pipeline_run_task(file_id, tenant.id)
 
@@ -398,13 +400,14 @@ class TestRagPipelineRunTasks:
             mock_file_service["delete_file"].assert_called_once_with(file_id)
             assert mock_pipeline_generator.call_count == 1
 
-            # Verify waiting tasks were processed, pull 1 task a time by default
-            assert mock_delay.call_count == 1
+            # Verify waiting tasks were processed via group, pull 1 task a time by default
+            assert mock_group.return_value.apply_async.called
 
-            # Verify correct parameters for the call
-            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
-            assert call_kwargs.get("rag_pipeline_invoke_entities_file_id") == legacy_file_ids[0]
-            assert call_kwargs.get("tenant_id") == tenant.id
+            # Verify correct parameters for the first scheduled job signature
+            jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+            first_kwargs = jobs[0].kwargs if jobs else {}
+            assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == legacy_file_ids[0]
+            assert first_kwargs.get("tenant_id") == tenant.id
 
             # Verify that new code can process legacy queue entries
             # The new TenantIsolatedTaskQueue should be able to read from the legacy format
@@ -446,8 +449,10 @@ class TestRagPipelineRunTasks:
         waiting_file_ids = [str(uuid.uuid4()) for _ in range(3)]
         queue.push_tasks(waiting_file_ids)
 
-        # Mock the task function calls
-        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+        # Mock the Celery group scheduling used by the implementation
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
+            mock_group.return_value.apply_async = MagicMock()
+
             # Act: Execute the regular task
             rag_pipeline_run_task(file_id, tenant.id)
 
@@ -456,13 +461,14 @@ class TestRagPipelineRunTasks:
             mock_file_service["delete_file"].assert_called_once_with(file_id)
             assert mock_pipeline_generator.call_count == 1
 
-            # Verify waiting tasks were processed, pull 1 task a time by default
-            assert mock_delay.call_count == 1
+            # Verify waiting tasks were processed via group.apply_async
+            assert mock_group.return_value.apply_async.called
 
-            # Verify correct parameters for the call
-            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
-            assert call_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_ids[0]
-            assert call_kwargs.get("tenant_id") == tenant.id
+            # Verify correct parameters for the first scheduled job signature
+            jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+            first_kwargs = jobs[0].kwargs if jobs else {}
+            assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_ids[0]
+            assert first_kwargs.get("tenant_id") == tenant.id
 
             # Verify queue still has remaining tasks (only 1 was pulled)
             remaining_tasks = queue.pull_tasks(count=10)
@@ -557,8 +563,10 @@ class TestRagPipelineRunTasks:
         waiting_file_id = str(uuid.uuid4())
         queue.push_tasks([waiting_file_id])
 
-        # Mock the task function calls
-        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+        # Mock the Celery group scheduling used by the implementation
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
+            mock_group.return_value.apply_async = MagicMock()
+
             # Act: Execute the regular task (should not raise exception)
             rag_pipeline_run_task(file_id, tenant.id)
 
@@ -569,12 +577,13 @@ class TestRagPipelineRunTasks:
             assert mock_pipeline_generator.call_count == 1
 
             # Verify waiting task was still processed despite core processing error
-            mock_delay.assert_called_once()
+            assert mock_group.return_value.apply_async.called
 
-            # Verify correct parameters for the call
-            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
-            assert call_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
-            assert call_kwargs.get("tenant_id") == tenant.id
+            # Verify correct parameters for the first scheduled job signature
+            jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+            first_kwargs = jobs[0].kwargs if jobs else {}
+            assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
+            assert first_kwargs.get("tenant_id") == tenant.id
 
             # Verify queue is empty after processing (task was pulled)
             remaining_tasks = queue.pull_tasks(count=10)
@@ -684,8 +693,10 @@ class TestRagPipelineRunTasks:
         queue1.push_tasks([waiting_file_id1])
         queue2.push_tasks([waiting_file_id2])
 
-        # Mock the task function calls
-        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+        # Mock the Celery group scheduling used by the implementation
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
+            mock_group.return_value.apply_async = MagicMock()
+
             # Act: Execute the regular task for tenant1 only
             rag_pipeline_run_task(file_id1, tenant1.id)
 
@@ -694,11 +705,12 @@ class TestRagPipelineRunTasks:
             assert mock_file_service["delete_file"].call_count == 1
             assert mock_pipeline_generator.call_count == 1
 
-            # Verify only tenant1's waiting task was processed
-            mock_delay.assert_called_once()
-            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
-            assert call_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id1
-            assert call_kwargs.get("tenant_id") == tenant1.id
+            # Verify only tenant1's waiting task was processed (via group)
+            assert mock_group.return_value.apply_async.called
+            jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+            first_kwargs = jobs[0].kwargs if jobs else {}
+            assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id1
+            assert first_kwargs.get("tenant_id") == tenant1.id
 
             # Verify tenant1's queue is empty
             remaining_tasks1 = queue1.pull_tasks(count=10)
@@ -913,8 +925,10 @@ class TestRagPipelineRunTasks:
         waiting_file_id = str(uuid.uuid4())
         queue.push_tasks([waiting_file_id])
 
-        # Mock the task function calls
-        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+        # Mock the Celery group scheduling used by the implementation
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
+            mock_group.return_value.apply_async = MagicMock()
+
             # Act & Assert: Execute the regular task (should raise Exception)
             with pytest.raises(Exception, match="File not found"):
                 rag_pipeline_run_task(file_id, tenant.id)
@@ -924,12 +938,13 @@ class TestRagPipelineRunTasks:
             mock_pipeline_generator.assert_not_called()
 
             # Verify waiting task was still processed despite file error
-            mock_delay.assert_called_once()
+            assert mock_group.return_value.apply_async.called
 
-            # Verify correct parameters for the call
-            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
-            assert call_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
-            assert call_kwargs.get("tenant_id") == tenant.id
+            # Verify correct parameters for the first scheduled job signature
+            jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+            first_kwargs = jobs[0].kwargs if jobs else {}
+            assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
+            assert first_kwargs.get("tenant_id") == tenant.id
 
             # Verify queue is empty after processing (task was pulled)
             remaining_tasks = queue.pull_tasks(count=10)

api/tests/test_containers_integration_tests/trigger/conftest.py

@@ -105,18 +105,26 @@ def app_model(
 
 
 class MockCeleryGroup:
-    """Mock for celery group() function that collects dispatched tasks."""
+    """Mock for celery group() function that collects dispatched tasks.
+
+    Matches the Celery group API loosely, accepting arbitrary kwargs on apply_async
+    (e.g. producer) so production code can pass broker-related options without
+    breaking tests.
+    """
 
     def __init__(self) -> None:
         self.collected: list[dict[str, Any]] = []
         self._applied = False
+        self.last_apply_async_kwargs: dict[str, Any] | None = None
 
     def __call__(self, items: Any) -> MockCeleryGroup:
         self.collected = list(items)
         return self
 
-    def apply_async(self) -> None:
+    def apply_async(self, **kwargs: Any) -> None:
+        # Accept arbitrary kwargs like producer to be compatible with Celery
         self._applied = True
+        self.last_apply_async_kwargs = kwargs
 
     @property
     def applied(self) -> bool:

api/tests/unit_tests/commands/test_clean_expired_messages.py

@@ -0,0 +1,181 @@
+import datetime
+import re
+from unittest.mock import MagicMock, patch
+
+import click
+import pytest
+
+from commands import clean_expired_messages
+
+
+def _mock_service() -> MagicMock:
+    service = MagicMock()
+    service.run.return_value = {
+        "batches": 1,
+        "total_messages": 10,
+        "filtered_messages": 5,
+        "total_deleted": 5,
+    }
+    return service
+
+
+def test_absolute_mode_calls_from_time_range():
+    policy = object()
+    service = _mock_service()
+    start_from = datetime.datetime(2024, 1, 1, 0, 0, 0)
+    end_before = datetime.datetime(2024, 2, 1, 0, 0, 0)
+
+    with (
+        patch("commands.create_message_clean_policy", return_value=policy),
+        patch("commands.MessagesCleanService.from_time_range", return_value=service) as mock_from_time_range,
+        patch("commands.MessagesCleanService.from_days") as mock_from_days,
+    ):
+        clean_expired_messages.callback(
+            batch_size=200,
+            graceful_period=21,
+            start_from=start_from,
+            end_before=end_before,
+            from_days_ago=None,
+            before_days=None,
+            dry_run=True,
+        )
+
+    mock_from_time_range.assert_called_once_with(
+        policy=policy,
+        start_from=start_from,
+        end_before=end_before,
+        batch_size=200,
+        dry_run=True,
+    )
+    mock_from_days.assert_not_called()
+
+
+def test_relative_mode_before_days_only_calls_from_days():
+    policy = object()
+    service = _mock_service()
+
+    with (
+        patch("commands.create_message_clean_policy", return_value=policy),
+        patch("commands.MessagesCleanService.from_days", return_value=service) as mock_from_days,
+        patch("commands.MessagesCleanService.from_time_range") as mock_from_time_range,
+    ):
+        clean_expired_messages.callback(
+            batch_size=500,
+            graceful_period=14,
+            start_from=None,
+            end_before=None,
+            from_days_ago=None,
+            before_days=30,
+            dry_run=False,
+        )
+
+    mock_from_days.assert_called_once_with(
+        policy=policy,
+        days=30,
+        batch_size=500,
+        dry_run=False,
+    )
+    mock_from_time_range.assert_not_called()
+
+
+def test_relative_mode_with_from_days_ago_calls_from_time_range():
+    policy = object()
+    service = _mock_service()
+    fixed_now = datetime.datetime(2024, 8, 20, 12, 0, 0)
+
+    with (
+        patch("commands.create_message_clean_policy", return_value=policy),
+        patch("commands.MessagesCleanService.from_time_range", return_value=service) as mock_from_time_range,
+        patch("commands.MessagesCleanService.from_days") as mock_from_days,
+        patch("commands.naive_utc_now", return_value=fixed_now),
+    ):
+        clean_expired_messages.callback(
+            batch_size=1000,
+            graceful_period=21,
+            start_from=None,
+            end_before=None,
+            from_days_ago=60,
+            before_days=30,
+            dry_run=False,
+        )
+
+    mock_from_time_range.assert_called_once_with(
+        policy=policy,
+        start_from=fixed_now - datetime.timedelta(days=60),
+        end_before=fixed_now - datetime.timedelta(days=30),
+        batch_size=1000,
+        dry_run=False,
+    )
+    mock_from_days.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "message"),
+    [
+        (
+            {
+                "start_from": datetime.datetime(2024, 1, 1),
+                "end_before": datetime.datetime(2024, 2, 1),
+                "from_days_ago": None,
+                "before_days": 30,
+            },
+            "mutually exclusive",
+        ),
+        (
+            {
+                "start_from": datetime.datetime(2024, 1, 1),
+                "end_before": None,
+                "from_days_ago": None,
+                "before_days": None,
+            },
+            "Both --start-from and --end-before are required",
+        ),
+        (
+            {
+                "start_from": None,
+                "end_before": None,
+                "from_days_ago": 10,
+                "before_days": None,
+            },
+            "--from-days-ago must be used together with --before-days",
+        ),
+        (
+            {
+                "start_from": None,
+                "end_before": None,
+                "from_days_ago": None,
+                "before_days": -1,
+            },
+            "--before-days must be >= 0",
+        ),
+        (
+            {
+                "start_from": None,
+                "end_before": None,
+                "from_days_ago": 30,
+                "before_days": 30,
+            },
+            "--from-days-ago must be greater than --before-days",
+        ),
+        (
+            {
+                "start_from": None,
+                "end_before": None,
+                "from_days_ago": None,
+                "before_days": None,
+            },
+            "You must provide either (--start-from,--end-before) or (--before-days [--from-days-ago])",
+        ),
+    ],
+)
+def test_invalid_inputs_raise_usage_error(kwargs: dict, message: str):
+    with pytest.raises(click.UsageError, match=re.escape(message)):
+        clean_expired_messages.callback(
+            batch_size=1000,
+            graceful_period=21,
+            start_from=kwargs["start_from"],
+            end_before=kwargs["end_before"],
+            from_days_ago=kwargs["from_days_ago"],
+            before_days=kwargs["before_days"],
+            dry_run=False,
+        )

api/tests/unit_tests/controllers/common/test_errors.py

@@ -0,0 +1,70 @@
+from controllers.common.errors import (
+    BlockedFileExtensionError,
+    FilenameNotExistsError,
+    FileTooLargeError,
+    NoFileUploadedError,
+    RemoteFileUploadError,
+    TooManyFilesError,
+    UnsupportedFileTypeError,
+)
+
+
+class TestFilenameNotExistsError:
+    def test_defaults(self):
+        error = FilenameNotExistsError()
+
+        assert error.code == 400
+        assert error.description == "The specified filename does not exist."
+
+
+class TestRemoteFileUploadError:
+    def test_defaults(self):
+        error = RemoteFileUploadError()
+
+        assert error.code == 400
+        assert error.description == "Error uploading remote file."
+
+
+class TestFileTooLargeError:
+    def test_defaults(self):
+        error = FileTooLargeError()
+
+        assert error.code == 413
+        assert error.error_code == "file_too_large"
+        assert error.description == "File size exceeded. {message}"
+
+
+class TestUnsupportedFileTypeError:
+    def test_defaults(self):
+        error = UnsupportedFileTypeError()
+
+        assert error.code == 415
+        assert error.error_code == "unsupported_file_type"
+        assert error.description == "File type not allowed."
+
+
+class TestBlockedFileExtensionError:
+    def test_defaults(self):
+        error = BlockedFileExtensionError()
+
+        assert error.code == 400
+        assert error.error_code == "file_extension_blocked"
+        assert error.description == "The file extension is blocked for security reasons."
+
+
+class TestTooManyFilesError:
+    def test_defaults(self):
+        error = TooManyFilesError()
+
+        assert error.code == 400
+        assert error.error_code == "too_many_files"
+        assert error.description == "Only one file is allowed."
+
+
+class TestNoFileUploadedError:
+    def test_defaults(self):
+        error = NoFileUploadedError()
+
+        assert error.code == 400
+        assert error.error_code == "no_file_uploaded"
+        assert error.description == "Please upload your file."

api/tests/unit_tests/controllers/common/test_file_response.py

@@ -1,22 +1,95 @@
 from flask import Response
 
-from controllers.common.file_response import enforce_download_for_html, is_html_content
+from controllers.common.file_response import (
+    _normalize_mime_type,
+    enforce_download_for_html,
+    is_html_content,
+)
 
 
-class TestFileResponseHelpers:
-    def test_is_html_content_detects_mime_type(self):
+class TestNormalizeMimeType:
+    def test_returns_empty_string_for_none(self):
+        assert _normalize_mime_type(None) == ""
+
+    def test_returns_empty_string_for_empty_string(self):
+        assert _normalize_mime_type("") == ""
+
+    def test_normalizes_mime_type(self):
+        assert _normalize_mime_type("Text/HTML; Charset=UTF-8") == "text/html"
+
+
+class TestIsHtmlContent:
+    def test_detects_html_via_mime_type(self):
         mime_type = "text/html; charset=UTF-8"
 
-        result = is_html_content(mime_type, filename="file.txt", extension="txt")
+        result = is_html_content(
+            mime_type=mime_type,
+            filename="file.txt",
+            extension="txt",
+        )
 
         assert result is True
 
-    def test_is_html_content_detects_extension(self):
-        result = is_html_content("text/plain", filename="report.html", extension=None)
+    def test_detects_html_via_extension_argument(self):
+        result = is_html_content(
+            mime_type="text/plain",
+            filename=None,
+            extension="html",
+        )
 
         assert result is True
 
-    def test_enforce_download_for_html_sets_headers(self):
+    def test_detects_html_via_filename_extension(self):
+        result = is_html_content(
+            mime_type="text/plain",
+            filename="report.html",
+            extension=None,
+        )
+
+        assert result is True
+
+    def test_returns_false_when_no_html_detected_anywhere(self):
+        """
+        Missing negative test:
+        - MIME type is not HTML
+        - filename has no HTML extension
+        - extension argument is not HTML
+        """
+        result = is_html_content(
+            mime_type="application/json",
+            filename="data.json",
+            extension="json",
+        )
+
+        assert result is False
+
+    def test_returns_false_when_all_inputs_are_none(self):
+        result = is_html_content(
+            mime_type=None,
+            filename=None,
+            extension=None,
+        )
+
+        assert result is False
+
+
+class TestEnforceDownloadForHtml:
+    def test_sets_attachment_when_filename_missing(self):
+        response = Response("payload", mimetype="text/html")
+
+        updated = enforce_download_for_html(
+            response,
+            mime_type="text/html",
+            filename=None,
+            extension="html",
+        )
+
+        assert updated is True
+        assert response.headers["Content-Disposition"] == "attachment"
+        assert response.headers["Content-Type"] == "application/octet-stream"
+        assert response.headers["X-Content-Type-Options"] == "nosniff"
+
+    def test_sets_headers_when_filename_present(self):
         response = Response("payload", mimetype="text/html")
 
         updated = enforce_download_for_html(
@@ -27,11 +100,12 @@ class TestFileResponseHelpers:
         )
 
         assert updated is True
-        assert "attachment" in response.headers["Content-Disposition"]
+        assert response.headers["Content-Disposition"].startswith("attachment")
+        assert "unsafe.html" in response.headers["Content-Disposition"]
         assert response.headers["Content-Type"] == "application/octet-stream"
         assert response.headers["X-Content-Type-Options"] == "nosniff"
 
-    def test_enforce_download_for_html_no_change_for_non_html(self):
+    def test_does_not_modify_response_for_non_html_content(self):
         response = Response("payload", mimetype="text/plain")
 
         updated = enforce_download_for_html(

api/tests/unit_tests/controllers/common/test_helpers.py

@@ -0,0 +1,188 @@
+from uuid import UUID
+
+import httpx
+import pytest
+
+from controllers.common import helpers
+from controllers.common.helpers import FileInfo, guess_file_info_from_response
+
+
+def make_response(
+    url="https://example.com/file.txt",
+    headers=None,
+    content=None,
+):
+    return httpx.Response(
+        200,
+        request=httpx.Request("GET", url),
+        headers=headers or {},
+        content=content or b"",
+    )
+
+
+class TestGuessFileInfoFromResponse:
+    def test_filename_from_url(self):
+        response = make_response(
+            url="https://example.com/test.pdf",
+            content=b"Hello World",
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.filename == "test.pdf"
+        assert info.extension == ".pdf"
+        assert info.mimetype == "application/pdf"
+
+    def test_filename_from_content_disposition(self):
+        headers = {
+            "Content-Disposition": "attachment; filename=myfile.csv",
+            "Content-Type": "text/csv",
+        }
+        response = make_response(
+            url="https://example.com/",
+            headers=headers,
+            content=b"Hello World",
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.filename == "myfile.csv"
+        assert info.extension == ".csv"
+        assert info.mimetype == "text/csv"
+
+    @pytest.mark.parametrize(
+        ("magic_available", "expected_ext"),
+        [
+            (True, "txt"),
+            (False, "bin"),
+        ],
+    )
+    def test_generated_filename_when_missing(self, monkeypatch, magic_available, expected_ext):
+        if magic_available:
+            if helpers.magic is None:
+                pytest.skip("python-magic is not installed, cannot run 'magic_available=True' test variant")
+        else:
+            monkeypatch.setattr(helpers, "magic", None)
+
+        response = make_response(
+            url="https://example.com/",
+            content=b"Hello World",
+        )
+
+        info = guess_file_info_from_response(response)
+
+        name, ext = info.filename.split(".")
+        UUID(name)
+        assert ext == expected_ext
+
+    def test_mimetype_from_header_when_unknown(self):
+        headers = {"Content-Type": "application/json"}
+        response = make_response(
+            url="https://example.com/file.unknown",
+            headers=headers,
+            content=b'{"a": 1}',
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.mimetype == "application/json"
+
+    def test_extension_added_when_missing(self):
+        headers = {"Content-Type": "image/png"}
+        response = make_response(
+            url="https://example.com/image",
+            headers=headers,
+            content=b"fakepngdata",
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.extension == ".png"
+        assert info.filename.endswith(".png")
+
+    def test_content_length_used_as_size(self):
+        headers = {
+            "Content-Length": "1234",
+            "Content-Type": "text/plain",
+        }
+        response = make_response(
+            url="https://example.com/a.txt",
+            headers=headers,
+            content=b"a" * 1234,
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.size == 1234
+
+    def test_size_minus_one_when_header_missing(self):
+        response = make_response(url="https://example.com/a.txt")
+
+        info = guess_file_info_from_response(response)
+
+        assert info.size == -1
+
+    def test_fallback_to_bin_extension(self):
+        headers = {"Content-Type": "application/octet-stream"}
+        response = make_response(
+            url="https://example.com/download",
+            headers=headers,
+            content=b"\x00\x01\x02\x03",
+        )
+
+        info = guess_file_info_from_response(response)
+
+        assert info.extension == ".bin"
+        assert info.filename.endswith(".bin")
+
+    def test_return_type(self):
+        response = make_response()
+
+        info = guess_file_info_from_response(response)
+
+        assert isinstance(info, FileInfo)
+
+
+class TestMagicImportWarnings:
+    @pytest.mark.parametrize(
+        ("platform_name", "expected_message"),
+        [
+            ("Windows", "pip install python-magic-bin"),
+            ("Darwin", "brew install libmagic"),
+            ("Linux", "sudo apt-get install libmagic1"),
+            ("Other", "install `libmagic`"),
+        ],
+    )
+    def test_magic_import_warning_per_platform(
+        self,
+        monkeypatch,
+        platform_name,
+        expected_message,
+    ):
+        import builtins
+        import importlib
+
+        # Force ImportError when "magic" is imported
+        real_import = builtins.__import__
+
+        def fake_import(name, *args, **kwargs):
+            if name == "magic":
+                raise ImportError("No module named magic")
+            return real_import(name, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "__import__", fake_import)
+        monkeypatch.setattr("platform.system", lambda: platform_name)
+
+        # Remove helpers so it imports fresh
+        import sys
+
+        original_helpers = sys.modules.get(helpers.__name__)
+        sys.modules.pop(helpers.__name__, None)
+
+        try:
+            with pytest.warns(UserWarning, match="To use python-magic") as warning:
+                imported_helpers = importlib.import_module(helpers.__name__)
+            assert expected_message in str(warning[0].message)
+        finally:
+            if original_helpers is not None:
+                sys.modules[helpers.__name__] = original_helpers

api/tests/unit_tests/controllers/common/test_schema.py

@@ -0,0 +1,189 @@
+import sys
+from enum import StrEnum
+from unittest.mock import MagicMock, patch
+
+import pytest
+from flask_restx import Namespace
+from pydantic import BaseModel
+
+
+class UserModel(BaseModel):
+    id: int
+    name: str
+
+
+class ProductModel(BaseModel):
+    id: int
+    price: float
+
+
+@pytest.fixture(autouse=True)
+def mock_console_ns():
+    """Mock the console_ns to avoid circular imports during test collection."""
+    mock_ns = MagicMock(spec=Namespace)
+    mock_ns.models = {}
+
+    # Inject mock before importing schema module
+    with patch.dict(sys.modules, {"controllers.console": MagicMock(console_ns=mock_ns)}):
+        yield mock_ns
+
+
+def test_default_ref_template_value():
+    from controllers.common.schema import DEFAULT_REF_TEMPLATE_SWAGGER_2_0
+
+    assert DEFAULT_REF_TEMPLATE_SWAGGER_2_0 == "#/definitions/{model}"
+
+
+def test_register_schema_model_calls_namespace_schema_model():
+    from controllers.common.schema import register_schema_model
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_schema_model(namespace, UserModel)
+
+    namespace.schema_model.assert_called_once()
+
+    model_name, schema = namespace.schema_model.call_args.args
+
+    assert model_name == "UserModel"
+    assert isinstance(schema, dict)
+    assert "properties" in schema
+
+
+def test_register_schema_model_passes_schema_from_pydantic():
+    from controllers.common.schema import DEFAULT_REF_TEMPLATE_SWAGGER_2_0, register_schema_model
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_schema_model(namespace, UserModel)
+
+    schema = namespace.schema_model.call_args.args[1]
+
+    expected_schema = UserModel.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+
+    assert schema == expected_schema
+
+
+def test_register_schema_models_registers_multiple_models():
+    from controllers.common.schema import register_schema_models
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_schema_models(namespace, UserModel, ProductModel)
+
+    assert namespace.schema_model.call_count == 2
+
+    called_names = [call.args[0] for call in namespace.schema_model.call_args_list]
+    assert called_names == ["UserModel", "ProductModel"]
+
+
+def test_register_schema_models_calls_register_schema_model(monkeypatch):
+    from controllers.common.schema import register_schema_models
+
+    namespace = MagicMock(spec=Namespace)
+
+    calls = []
+
+    def fake_register(ns, model):
+        calls.append((ns, model))
+
+    monkeypatch.setattr(
+        "controllers.common.schema.register_schema_model",
+        fake_register,
+    )
+
+    register_schema_models(namespace, UserModel, ProductModel)
+
+    assert calls == [
+        (namespace, UserModel),
+        (namespace, ProductModel),
+    ]
+
+
+class StatusEnum(StrEnum):
+    ACTIVE = "active"
+    INACTIVE = "inactive"
+
+
+class PriorityEnum(StrEnum):
+    HIGH = "high"
+    LOW = "low"
+
+
+def test_get_or_create_model_returns_existing_model(mock_console_ns):
+    from controllers.common.schema import get_or_create_model
+
+    existing_model = MagicMock()
+    mock_console_ns.models = {"TestModel": existing_model}
+
+    result = get_or_create_model("TestModel", {"key": "value"})
+
+    assert result == existing_model
+    mock_console_ns.model.assert_not_called()
+
+
+def test_get_or_create_model_creates_new_model_when_not_exists(mock_console_ns):
+    from controllers.common.schema import get_or_create_model
+
+    mock_console_ns.models = {}
+    new_model = MagicMock()
+    mock_console_ns.model.return_value = new_model
+    field_def = {"name": {"type": "string"}}
+
+    result = get_or_create_model("NewModel", field_def)
+
+    assert result == new_model
+    mock_console_ns.model.assert_called_once_with("NewModel", field_def)
+
+
+def test_get_or_create_model_does_not_call_model_if_exists(mock_console_ns):
+    from controllers.common.schema import get_or_create_model
+
+    existing_model = MagicMock()
+    mock_console_ns.models = {"ExistingModel": existing_model}
+
+    result = get_or_create_model("ExistingModel", {"key": "value"})
+
+    assert result == existing_model
+    mock_console_ns.model.assert_not_called()
+
+
+def test_register_enum_models_registers_single_enum():
+    from controllers.common.schema import register_enum_models
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_enum_models(namespace, StatusEnum)
+
+    namespace.schema_model.assert_called_once()
+
+    model_name, schema = namespace.schema_model.call_args.args
+
+    assert model_name == "StatusEnum"
+    assert isinstance(schema, dict)
+
+
+def test_register_enum_models_registers_multiple_enums():
+    from controllers.common.schema import register_enum_models
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_enum_models(namespace, StatusEnum, PriorityEnum)
+
+    assert namespace.schema_model.call_count == 2
+
+    called_names = [call.args[0] for call in namespace.schema_model.call_args_list]
+    assert called_names == ["StatusEnum", "PriorityEnum"]
+
+
+def test_register_enum_models_uses_correct_ref_template():
+    from controllers.common.schema import register_enum_models
+
+    namespace = MagicMock(spec=Namespace)
+
+    register_enum_models(namespace, StatusEnum)
+
+    schema = namespace.schema_model.call_args.args[1]
+
+    # Verify the schema contains enum values
+    assert "enum" in schema or "anyOf" in schema

api/tests/unit_tests/controllers/console/app/test_annotation_api.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from controllers.console.app import annotation as annotation_module
+
+
+def test_annotation_reply_payload_valid():
+    """Test AnnotationReplyPayload with valid data."""
+    payload = annotation_module.AnnotationReplyPayload(
+        score_threshold=0.5,
+        embedding_provider_name="openai",
+        embedding_model_name="text-embedding-3-small",
+    )
+    assert payload.score_threshold == 0.5
+    assert payload.embedding_provider_name == "openai"
+    assert payload.embedding_model_name == "text-embedding-3-small"
+
+
+def test_annotation_setting_update_payload_valid():
+    """Test AnnotationSettingUpdatePayload with valid data."""
+    payload = annotation_module.AnnotationSettingUpdatePayload(
+        score_threshold=0.75,
+    )
+    assert payload.score_threshold == 0.75
+
+
+def test_annotation_list_query_defaults():
+    """Test AnnotationListQuery with default parameters."""
+    query = annotation_module.AnnotationListQuery()
+    assert query.page == 1
+    assert query.limit == 20
+    assert query.keyword == ""
+
+
+def test_annotation_list_query_custom_page():
+    """Test AnnotationListQuery with custom page."""
+    query = annotation_module.AnnotationListQuery(page=3, limit=50)
+    assert query.page == 3
+    assert query.limit == 50
+
+
+def test_annotation_list_query_with_keyword():
+    """Test AnnotationListQuery with keyword."""
+    query = annotation_module.AnnotationListQuery(keyword="test")
+    assert query.keyword == "test"
+
+
+def test_create_annotation_payload_with_message_id():
+    """Test CreateAnnotationPayload with message ID."""
+    payload = annotation_module.CreateAnnotationPayload(
+        message_id="550e8400-e29b-41d4-a716-446655440000",
+        question="What is AI?",
+    )
+    assert payload.message_id == "550e8400-e29b-41d4-a716-446655440000"
+    assert payload.question == "What is AI?"
+
+
+def test_create_annotation_payload_with_text():
+    """Test CreateAnnotationPayload with text content."""
+    payload = annotation_module.CreateAnnotationPayload(
+        question="What is ML?",
+        answer="Machine learning is...",
+    )
+    assert payload.question == "What is ML?"
+    assert payload.answer == "Machine learning is..."
+
+
+def test_update_annotation_payload():
+    """Test UpdateAnnotationPayload."""
+    payload = annotation_module.UpdateAnnotationPayload(
+        question="Updated question",
+        answer="Updated answer",
+    )
+    assert payload.question == "Updated question"
+    assert payload.answer == "Updated answer"
+
+
+def test_annotation_reply_status_query_enable():
+    """Test AnnotationReplyStatusQuery with enable action."""
+    query = annotation_module.AnnotationReplyStatusQuery(action="enable")
+    assert query.action == "enable"
+
+
+def test_annotation_reply_status_query_disable():
+    """Test AnnotationReplyStatusQuery with disable action."""
+    query = annotation_module.AnnotationReplyStatusQuery(action="disable")
+    assert query.action == "disable"
+
+
+def test_annotation_file_payload_valid():
+    """Test AnnotationFilePayload with valid message ID."""
+    payload = annotation_module.AnnotationFilePayload(message_id="550e8400-e29b-41d4-a716-446655440000")
+    assert payload.message_id == "550e8400-e29b-41d4-a716-446655440000"