Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
77e41a37ee | ||
|
|
79229b7ede | ||
|
|
36ae648a63 | ||
|
|
c409d531da | ||
|
|
511ca69e4e | ||
|
|
4c7e60d7e7 | ||
|
|
a9e163403d | ||
|
|
bc136b89f9 | ||
|
|
b1bd4f9a8b | ||
|
|
06d3927f05 | ||
|
|
5c3516cae8 | ||
|
|
1c18d8ddbd | ||
|
|
253dfe9351 | ||
|
|
2d3999e984 | ||
|
|
a431cc726d | ||
|
|
08493d2429 | ||
|
|
8dd0969006 | ||
|
|
2d80d3c35c | ||
|
|
4da764904c |
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx -y block-no-verify@1.1.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,6 @@
|
||||
**/*.pyc
|
||||
**/.mypy_cache
|
||||
**/.ruff_cache
|
||||
knowledge-fs/
|
||||
.git
|
||||
.github
|
||||
*.md
|
||||
|
||||
@@ -1,15 +1,6 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/knowledge-fs"
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
groups:
|
||||
knowledge-fs-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
- package-ecosystem: "uv"
|
||||
directory: "/api"
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
@@ -2,70 +2,23 @@ name: Deploy Dev
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["KnowledgeFS CI"]
|
||||
workflows: ["Build and Push API & Web"]
|
||||
branches:
|
||||
- "deploy/konwledge"
|
||||
- "deploy/dev"
|
||||
types:
|
||||
- completed
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/konwledge'
|
||||
github.event.workflow_run.head_branch == 'deploy/dev'
|
||||
steps:
|
||||
- name: Wait for API and Web image build
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
timeout-minutes: 35
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const workflowId = "build-push.yml";
|
||||
const headBranch = context.payload.workflow_run.head_branch;
|
||||
const headSha = context.payload.workflow_run.head_sha;
|
||||
const deadline = Date.now() + 30 * 60 * 1000;
|
||||
const pollIntervalMs = 15 * 1000;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: workflowId,
|
||||
branch: headBranch,
|
||||
event: "push",
|
||||
head_sha: headSha,
|
||||
per_page: 10,
|
||||
});
|
||||
const run = data.workflow_runs[0];
|
||||
|
||||
if (!run) {
|
||||
core.info(`Waiting for ${workflowId} to start for ${headSha}.`);
|
||||
} else if (run.status !== "completed") {
|
||||
core.info(`Waiting for ${run.html_url}; current status is ${run.status}.`);
|
||||
} else if (run.conclusion !== "success") {
|
||||
throw new Error(
|
||||
`${workflowId} did not succeed for ${headSha}: ${run.conclusion} (${run.html_url})`,
|
||||
);
|
||||
} else {
|
||||
core.info(`Both image workflows succeeded for ${headSha}: ${run.html_url}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Timed out waiting for ${workflowId} to succeed for ${headSha}.`);
|
||||
|
||||
- name: Deploy to server
|
||||
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
|
||||
with:
|
||||
host: ${{ secrets.SSH_NEW_RAG_HOST }}
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
script: |
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
name: Deploy Knowledge
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
on:
|
||||
@@ -19,48 +18,6 @@ jobs:
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/konwledge'
|
||||
steps:
|
||||
- name: Wait for KnowledgeFS CI
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
timeout-minutes: 35
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const workflowId = "knowledge-fs-ci.yml";
|
||||
const headBranch = context.payload.workflow_run.head_branch;
|
||||
const headSha = context.payload.workflow_run.head_sha;
|
||||
const deadline = Date.now() + 30 * 60 * 1000;
|
||||
const pollIntervalMs = 15 * 1000;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: workflowId,
|
||||
branch: headBranch,
|
||||
event: "push",
|
||||
head_sha: headSha,
|
||||
per_page: 10,
|
||||
});
|
||||
const run = data.workflow_runs[0];
|
||||
|
||||
if (!run) {
|
||||
core.info(`Waiting for ${workflowId} to start for ${headSha}.`);
|
||||
} else if (run.status !== "completed") {
|
||||
core.info(`Waiting for ${run.html_url}; current status is ${run.status}.`);
|
||||
} else if (run.conclusion !== "success") {
|
||||
throw new Error(
|
||||
`${workflowId} did not succeed for ${headSha}: ${run.conclusion} (${run.html_url})`,
|
||||
);
|
||||
} else {
|
||||
core.info(`KnowledgeFS CI succeeded for ${headSha}: ${run.html_url}`);
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Timed out waiting for ${workflowId} to succeed for ${headSha}.`);
|
||||
|
||||
- name: Deploy to server
|
||||
uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
|
||||
with:
|
||||
|
||||
@@ -1,514 +0,0 @@
|
||||
name: KnowledgeFS CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
merge_group:
|
||||
branches: ["main"]
|
||||
types: [checks_requested]
|
||||
push:
|
||||
branches: ["main", "deploy/konwledge"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CI: true
|
||||
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
|
||||
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
name: Check KnowledgeFS changes
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect KnowledgeFS changes
|
||||
id: changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
filters: |
|
||||
knowledge-fs:
|
||||
- 'knowledge-fs/**'
|
||||
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
|
||||
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
|
||||
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
|
||||
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
|
||||
- 'knowledge-fs/scripts/export-openapi.mjs'
|
||||
- 'api/dev/generate_knowledge_fs_contract.py'
|
||||
- 'api/dev/knowledge_fs_product_contract.py'
|
||||
- 'api/knowledge-fs-contract.lock.json'
|
||||
- 'api/knowledge-fs-product-operation-gaps.json'
|
||||
- 'api/knowledge-fs-product-operations.json'
|
||||
- 'api/**/knowledge_fs/**'
|
||||
- 'api/**/*knowledge_fs*'
|
||||
- 'api/**/*knowledge-fs*'
|
||||
- 'api/.env.example'
|
||||
- 'api/app_factory.py'
|
||||
- 'api/commands/__init__.py'
|
||||
- 'api/controllers/console/__init__.py'
|
||||
- 'api/controllers/console/workspace/rbac.py'
|
||||
- 'api/controllers/service_api/__init__.py'
|
||||
- 'api/core/agent/base_agent_runner.py'
|
||||
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
|
||||
- 'api/core/rbac/entities.py'
|
||||
- 'api/core/tools/__base/tool_runtime.py'
|
||||
- 'api/core/tools/builtin_tool/_position.yaml'
|
||||
- 'api/core/workflow/node_runtime.py'
|
||||
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
|
||||
- 'api/extensions/ext_celery.py'
|
||||
- 'api/extensions/ext_commands.py'
|
||||
- 'api/models/__init__.py'
|
||||
- 'api/services/account_service.py'
|
||||
- 'api/services/agent_tool_inner_service.py'
|
||||
- 'api/services/enterprise/rbac_service.py'
|
||||
- 'api/services/entities/agent_tool_inner.py'
|
||||
- 'api/services/knowledge_fs/**'
|
||||
- 'api/services/knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
|
||||
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
|
||||
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
|
||||
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
|
||||
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
|
||||
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
|
||||
- 'api/tests/unit_tests/services/test_account_service.py'
|
||||
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
|
||||
- 'api/pyproject.toml'
|
||||
- 'api/uv.lock'
|
||||
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
|
||||
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
|
||||
- 'packages/contracts/generated/api/console/**'
|
||||
- 'packages/contracts/generated/api/service/**'
|
||||
- 'docker/.env.example'
|
||||
- 'docker/README.md'
|
||||
- 'docker/dify-env-sync.py'
|
||||
- 'docker/dify-env-sync.sh'
|
||||
- 'docker/docker-compose-template.yaml'
|
||||
- 'docker/docker-compose.yaml'
|
||||
- 'docker/envs/core-services/api.env.example'
|
||||
- 'docker/envs/core-services/knowledge-fs.env.example'
|
||||
- 'docker/generate_docker_compose'
|
||||
- 'docs/design/knowledge-fs*'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/workflows/knowledge-fs-ci.yml'
|
||||
|
||||
build:
|
||||
name: Build KnowledgeFS API production image
|
||||
needs:
|
||||
- check-changes
|
||||
- quality
|
||||
if: >-
|
||||
(needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch') &&
|
||||
needs.quality.result == 'success'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/deploy/konwledge'))
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract KnowledgeFS image metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
type=ref,event=branch
|
||||
type=sha,format=long
|
||||
|
||||
- name: Build KnowledgeFS API image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: ./knowledge-fs
|
||||
file: ./knowledge-fs/apps/api/Dockerfile
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64
|
||||
push: >-
|
||||
${{ github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'push' &&
|
||||
(github.ref == 'refs/heads/main' || github.ref == 'refs/heads/deploy/konwledge')) }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
|
||||
quality:
|
||||
name: Run KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: ./knowledge-fs
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
package_json_file: knowledge-fs/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
|
||||
|
||||
- name: Install KnowledgeFS dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Scan KnowledgeFS secrets
|
||||
run: pnpm security:secrets
|
||||
|
||||
- name: Audit KnowledgeFS production dependencies
|
||||
run: pnpm security:dependencies
|
||||
|
||||
- name: Run KnowledgeFS checks
|
||||
run: pnpm check
|
||||
|
||||
- name: Build KnowledgeFS
|
||||
run: pnpm build
|
||||
|
||||
- name: Lint KnowledgeFS
|
||||
run: pnpm lint:backend
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.12"
|
||||
cache-dependency-glob: |
|
||||
api/uv.lock
|
||||
dify-agent/uv.lock
|
||||
|
||||
- name: Verify Dify dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install Dify contract dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project api --locked --dev
|
||||
|
||||
- name: Collect Dify KnowledgeFS gate targets
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
production_targets=()
|
||||
add_production_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS production target is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
production_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_production_target "$path"
|
||||
fi
|
||||
done < <(
|
||||
find api \
|
||||
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
|
||||
-o -type f -name '*.py' -print0
|
||||
)
|
||||
|
||||
production_touchpoints=(
|
||||
api/app_factory.py
|
||||
api/commands/__init__.py
|
||||
api/controllers/console/__init__.py
|
||||
api/controllers/console/workspace/rbac.py
|
||||
api/controllers/service_api/__init__.py
|
||||
api/core/agent/base_agent_runner.py
|
||||
api/core/app/apps/agent_app/runtime_request_builder.py
|
||||
api/core/rbac/entities.py
|
||||
api/core/tools/__base/tool_runtime.py
|
||||
api/core/workflow/node_runtime.py
|
||||
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
|
||||
api/extensions/ext_celery.py
|
||||
api/extensions/ext_commands.py
|
||||
api/models/__init__.py
|
||||
api/services/account_service.py
|
||||
api/services/agent_tool_inner_service.py
|
||||
api/services/enterprise/rbac_service.py
|
||||
api/services/entities/agent_tool_inner.py
|
||||
)
|
||||
for path in "${production_touchpoints[@]}"; do
|
||||
add_production_target "$path"
|
||||
done
|
||||
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
|
||||
|
||||
if ((${#production_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
|
||||
|
||||
test_targets=()
|
||||
add_test_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
test_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_test_target "$path"
|
||||
fi
|
||||
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
|
||||
|
||||
test_touchpoints=(
|
||||
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
|
||||
api/tests/unit_tests/core/agent/test_base_agent_runner.py
|
||||
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
|
||||
api/tests/unit_tests/core/workflow/test_node_runtime.py
|
||||
api/tests/unit_tests/services/enterprise/test_rbac_service.py
|
||||
api/tests/unit_tests/services/test_account_service.py
|
||||
api/tests/unit_tests/services/test_agent_tool_inner_service.py
|
||||
)
|
||||
for path in "${test_touchpoints[@]}"; do
|
||||
add_test_target "$path"
|
||||
done
|
||||
|
||||
required_test_scopes=(
|
||||
/commands/
|
||||
/configs/
|
||||
/controllers/
|
||||
/core/agent/
|
||||
/core/app/apps/agent_app/
|
||||
/core/tools/builtin_tool/providers/knowledge_fs/
|
||||
/core/workflow/
|
||||
/dev/
|
||||
/extensions/
|
||||
/migrations/
|
||||
/models/
|
||||
/repositories/
|
||||
/services/
|
||||
/tasks/
|
||||
)
|
||||
for required_scope in "${required_test_scopes[@]}"; do
|
||||
scope_found=false
|
||||
for path in "${test_targets[@]}"; do
|
||||
if [[ "$path" == *"$required_scope"* ]]; then
|
||||
scope_found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$scope_found" != true ]]; then
|
||||
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ((${#test_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
|
||||
|
||||
- name: Lint Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev ruff format --check "${targets[@]}"
|
||||
uv run --project api --dev ruff check "${targets[@]}"
|
||||
|
||||
- name: Type-check Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
|
||||
|
||||
mypy_targets=()
|
||||
for path in "${targets[@]}"; do
|
||||
if [[ "$path" != api/migrations/* ]]; then
|
||||
mypy_targets+=("${path#api/}")
|
||||
fi
|
||||
done
|
||||
if ((${#mypy_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS Mypy target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --directory api --dev mypy \
|
||||
--explicit-package-bases \
|
||||
--exclude-gitignore \
|
||||
--exclude '(^|/)conftest\.py$' \
|
||||
--exclude 'tests/' \
|
||||
--exclude 'migrations/' \
|
||||
--check-untyped-defs \
|
||||
--disable-error-code=import-untyped \
|
||||
"${mypy_targets[@]}"
|
||||
|
||||
- name: Test Dify KnowledgeFS unit surface
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
|
||||
|
||||
- name: Enforce Dify KnowledgeFS focused coverage
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
KNOWLEDGE_FS_COVERAGE_BASE: >-
|
||||
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
|
||||
uv run --project api --dev coverage json --show-contexts -o "$report"
|
||||
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
|
||||
--coverage-json "$report" \
|
||||
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
|
||||
--base "$KNOWLEDGE_FS_COVERAGE_BASE" \
|
||||
--minimum 90 \
|
||||
--glue-minimum 90
|
||||
|
||||
- name: Verify Dify KnowledgeFS contract
|
||||
working-directory: .
|
||||
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
|
||||
|
||||
- name: Verify Dify Agent dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project dify-agent --check
|
||||
|
||||
- name: Install Dify Agent gate dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project dify-agent --locked --dev
|
||||
|
||||
- name: Lint Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: |
|
||||
uv run --project . --dev ruff format --check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
uv run --project . --dev ruff check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Type-check Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev basedpyright --level error
|
||||
src/dify_agent/layers/dify_core_tools/client.py
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Test Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev python -m pytest
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
-q
|
||||
|
||||
skip:
|
||||
name: Skip KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped KnowledgeFS checks
|
||||
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
|
||||
|
||||
final:
|
||||
name: KnowledgeFS CI
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
- check-changes
|
||||
- build
|
||||
- quality
|
||||
- skip
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize KnowledgeFS CI status
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BUILD_RESULT: ${{ needs.build.result }}
|
||||
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
|
||||
QUALITY_RESULT: ${{ needs.quality.result }}
|
||||
SKIP_RESULT: ${{ needs.skip.result }}
|
||||
run: |
|
||||
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
|
||||
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS build and checks ran successfully."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS checks were skipped because no related files changed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
|
||||
exit 1
|
||||
@@ -30,11 +30,6 @@ share/python-wheels/
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
|
||||
# directory contains source files rather than Python build output.
|
||||
!/knowledge-fs/apps/admin/lib/
|
||||
!/knowledge-fs/apps/admin/lib/**
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
|
||||
+3
-17
@@ -686,25 +686,11 @@ AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
|
||||
|
||||
# KnowledgeFS (Dataset 2.0)
|
||||
KNOWLEDGE_FS_ENABLED=false
|
||||
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
|
||||
KNOWLEDGE_FS_BASE_URL=
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN=
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
|
||||
# Shared with KnowledgeFS; use at least 32 random characters.
|
||||
KNOWLEDGE_FS_JWT_SECRET=
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
|
||||
|
||||
# Marketplace configuration
|
||||
MARKETPLACE_ENABLED=true
|
||||
|
||||
+1
-3
@@ -50,9 +50,7 @@ The scripts resolve paths relative to their location, so you can run them from a
|
||||
./dev/start-worker
|
||||
```
|
||||
|
||||
1. Start Celery Beat when scheduled tasks are needed. This is required when
|
||||
`KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=true` so provisioning and deletion
|
||||
outbox commands are dispatched to the lifecycle worker.
|
||||
1. Optional: start Celery Beat (scheduled tasks).
|
||||
|
||||
```bash
|
||||
./dev/start-beat
|
||||
|
||||
@@ -183,7 +183,6 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_forward_refs,
|
||||
ext_hosting_provider,
|
||||
ext_import_modules,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_logging,
|
||||
ext_login,
|
||||
ext_logstore,
|
||||
@@ -234,7 +233,6 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_enterprise_telemetry,
|
||||
ext_request_logging,
|
||||
ext_session_factory,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_oauth_bearer,
|
||||
]
|
||||
for ext in extensions:
|
||||
|
||||
@@ -10,7 +10,6 @@ from .data_migration import (
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
)
|
||||
from .knowledge_fs import knowledge_fs_control_space
|
||||
from .plugin import (
|
||||
backfill_plugin_auto_upgrade,
|
||||
extract_plugins,
|
||||
@@ -76,7 +75,6 @@ __all__ = [
|
||||
"import_migration_data",
|
||||
"install_plugins",
|
||||
"install_rag_pipeline_plugins",
|
||||
"knowledge_fs_control_space",
|
||||
"legacy_model_types",
|
||||
"migrate_annotation_vector_database",
|
||||
"migrate_data_for_plugin",
|
||||
|
||||
@@ -1,621 +0,0 @@
|
||||
"""Operator commands for the independent KnowledgeFS control-plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from services.knowledge_fs.cleanup import (
|
||||
CleanupApprovalInput,
|
||||
CleanupCompletionEvidenceInput,
|
||||
CleanupReadinessEvidenceInput,
|
||||
CleanupStartInput,
|
||||
KnowledgeFSCleanupError,
|
||||
KnowledgeFSCleanupService,
|
||||
)
|
||||
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
|
||||
from services.knowledge_fs.control_space_management import (
|
||||
KnowledgeFSControlSpaceManagementService,
|
||||
KnowledgeFSControlSpaceRegistration,
|
||||
)
|
||||
from services.knowledge_fs.cutover import (
|
||||
CutoverSmokeResultsInput,
|
||||
FinalDeltaInput,
|
||||
KnowledgeFSCutoverError,
|
||||
KnowledgeFSWorkspaceCutoverService,
|
||||
LegacyDependencyInput,
|
||||
QuarantineResolutionInput,
|
||||
ShadowAuthorizationObservationInput,
|
||||
ShadowCompletionInput,
|
||||
WorkspaceInventoryInput,
|
||||
)
|
||||
from services.knowledge_fs.greenfield_initializer import KnowledgeFSWorkspaceGreenfieldInitializer
|
||||
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
|
||||
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
|
||||
|
||||
|
||||
@click.group("knowledge-fs-control-space")
|
||||
def knowledge_fs_control_space() -> None:
|
||||
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("dry-run")
|
||||
@click.option("--tenant-id", default=None)
|
||||
def dry_run(tenant_id: str | None) -> None:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("inventory")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
|
||||
def inventory(input_path: Path, apply: bool) -> None:
|
||||
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.inventory, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("register")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--owner-account-id", required=True)
|
||||
@click.option("--provisioning-key", required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def register(
|
||||
tenant_id: str,
|
||||
owner_account_id: str,
|
||||
provisioning_key: str,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space, replayed = _management_service().register(
|
||||
KnowledgeFSControlSpaceRegistration(
|
||||
tenant_id,
|
||||
owner_account_id,
|
||||
provisioning_key,
|
||||
knowledge_space_id,
|
||||
knowledge_space_revision,
|
||||
)
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("backfill")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
|
||||
def backfill(input_path: Path, apply: bool) -> None:
|
||||
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.backfill, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("quarantine-resolve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
|
||||
def quarantine_resolve(input_path: Path, apply: bool) -> None:
|
||||
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
|
||||
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-start")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
|
||||
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.begin_shadow(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at) if started_at is not None else None,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-report")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
|
||||
def shadow_report(input_path: Path, apply: bool) -> None:
|
||||
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
|
||||
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
|
||||
def shadow_complete(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
|
||||
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_issue_fail_closed(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_issue(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-dashboard")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--checked-at", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
|
||||
def legacy_dashboard(
|
||||
tenant_id: str,
|
||||
input_path: Path,
|
||||
checked_at: str,
|
||||
expected_cas_version: int | None,
|
||||
apply: bool,
|
||||
) -> None:
|
||||
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
|
||||
report = _operator_call(
|
||||
lambda: _cutover_service().legacy_dependency_dashboard(
|
||||
tenant_id=tenant_id,
|
||||
dependencies=dependencies,
|
||||
expected_cas_version=expected_cas_version,
|
||||
checked_at=_parse_timestamp(checked_at),
|
||||
apply=apply,
|
||||
)
|
||||
)
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-check")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def legacy_check(tenant_id: str) -> None:
|
||||
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
|
||||
passed = (
|
||||
bool(status_report["legacy_dependency_ready"])
|
||||
and status_report["open_issues"] == 0
|
||||
and status_report["unresolved_cutover_quarantine"] == 0
|
||||
)
|
||||
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
|
||||
if not passed:
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("freeze")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "freeze_at", required=True)
|
||||
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.freeze(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
freeze_at=_parse_timestamp(freeze_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("final-delta")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def final_delta(input_path: Path) -> None:
|
||||
payload = _read_one_jsonl(input_path, FinalDeltaInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(lambda: service.apply_final_delta(payload))
|
||||
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cutover")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "cutover_at", required=True)
|
||||
@click.option("--rollback-cutoff-at", required=True)
|
||||
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.cutover(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
cutover_at=_parse_timestamp(cutover_at),
|
||||
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("smoke")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
|
||||
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.record_smoke_results(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
results=results,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("observe")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--started-at", default=None)
|
||||
@click.option("--window-ends-at", default=None)
|
||||
@click.option("--maximum-task-expires-at", default=None)
|
||||
@click.option("--observed-at", default=None)
|
||||
def observe(
|
||||
tenant_id: str,
|
||||
expected_cas_version: int,
|
||||
started_at: str | None,
|
||||
window_ends_at: str | None,
|
||||
maximum_task_expires_at: str | None,
|
||||
observed_at: str | None,
|
||||
) -> None:
|
||||
service = _cutover_service()
|
||||
if observed_at is not None:
|
||||
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
|
||||
raise click.UsageError("--observed-at cannot be combined with observation start options")
|
||||
_operator_call(
|
||||
lambda: service.complete_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
observed_at=_parse_timestamp(observed_at),
|
||||
)
|
||||
)
|
||||
else:
|
||||
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
|
||||
raise click.UsageError(
|
||||
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
|
||||
)
|
||||
_operator_call(
|
||||
lambda: service.begin_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at),
|
||||
window_ends_at=_parse_timestamp(window_ends_at),
|
||||
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("rollback")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "rolled_back_at", required=True)
|
||||
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.rollback(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
rolled_back_at=_parse_timestamp(rolled_back_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def status(tenant_id: str) -> None:
|
||||
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("greenfield-initialize")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def greenfield_initialize(tenant_id: str) -> None:
|
||||
"""Idempotently initialize one Workspace that has no KnowledgeFS state."""
|
||||
|
||||
_operator_call(lambda: _greenfield_initializer().ensure_initialized(tenant_id=tenant_id))
|
||||
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-request")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
|
||||
def cleanup_request(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-approve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
|
||||
def cleanup_approve(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-start")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
|
||||
@click.option(
|
||||
"--acknowledge-irreversible",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms rollback will be permanently closed.",
|
||||
)
|
||||
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupStartInput)
|
||||
if apply and not acknowledge_irreversible:
|
||||
raise click.UsageError("--apply requires --acknowledge-irreversible")
|
||||
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
|
||||
@click.option(
|
||||
"--acknowledge-executed",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
|
||||
)
|
||||
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
|
||||
if apply and not acknowledge_executed:
|
||||
raise click.UsageError("--apply requires --acknowledge-executed")
|
||||
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--request-id", required=True)
|
||||
def cleanup_status(tenant_id: str, request_id: str) -> None:
|
||||
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("repair")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--control-space-id", required=True)
|
||||
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def repair(
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
expected_resource_version: int,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space = _management_service().repair_registration(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=expected_resource_version,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("orphan-report")
|
||||
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
|
||||
def orphan_report(limit: int) -> None:
|
||||
report = KnowledgeFSOrphanReconciler(
|
||||
session_factory.get_session_maker(),
|
||||
get_knowledge_fs_lifecycle_remote(),
|
||||
).reconcile(limit=limit, apply_repairs=False)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-request")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
|
||||
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
|
||||
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
|
||||
|
||||
if not apply:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
_echo_json(
|
||||
{
|
||||
"apply": False,
|
||||
"by_state": report.by_state,
|
||||
"tenant_id": tenant_id,
|
||||
"total": report.total,
|
||||
}
|
||||
)
|
||||
return
|
||||
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
|
||||
_echo_json(
|
||||
{
|
||||
"apply": True,
|
||||
"control_space_ids": [result.control_space.id for result in results],
|
||||
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-finalize")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
|
||||
@click.option(
|
||||
"--acknowledge-control-plane-purge",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply after every remote Space has reached deleted.",
|
||||
)
|
||||
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
|
||||
"""Release the Workspace FK only after all remote deletions are terminal."""
|
||||
|
||||
service = _lifecycle_service()
|
||||
if not apply:
|
||||
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
|
||||
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
|
||||
return
|
||||
if not acknowledge_control_plane_purge:
|
||||
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
|
||||
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
|
||||
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
|
||||
|
||||
|
||||
def _read_jsonl[InputT: BaseModel](
|
||||
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
|
||||
) -> tuple[InputT, ...]:
|
||||
records: list[InputT] = []
|
||||
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
records.append(input_type.model_validate_json(line))
|
||||
except ValidationError as exc:
|
||||
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
|
||||
if not records and not allow_empty:
|
||||
raise click.ClickException("strict JSONL input must contain at least one record")
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
|
||||
records = _read_jsonl(input_path, input_type)
|
||||
if len(records) != 1:
|
||||
raise click.ClickException("this command requires exactly one JSONL record")
|
||||
return records[0]
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise click.ClickException("operator timestamps must include an explicit timezone")
|
||||
return parsed
|
||||
|
||||
|
||||
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCutoverError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCleanupError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSControlSpaceLifecycleError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _echo_json(payload: object) -> None:
|
||||
click.echo(json.dumps(payload, default=str, sort_keys=True))
|
||||
|
||||
|
||||
def _management_service() -> KnowledgeFSControlSpaceManagementService:
|
||||
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
|
||||
return KnowledgeFSWorkspaceCutoverService(
|
||||
session_factory.get_session_maker(),
|
||||
remote_factory=get_knowledge_fs_lifecycle_remote,
|
||||
)
|
||||
|
||||
|
||||
def _greenfield_initializer() -> KnowledgeFSWorkspaceGreenfieldInitializer:
|
||||
return KnowledgeFSWorkspaceGreenfieldInitializer(
|
||||
session_factory.get_session_maker(),
|
||||
cutover=_cutover_service(),
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_service() -> KnowledgeFSCleanupService:
|
||||
return KnowledgeFSCleanupService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
|
||||
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
__all__ = ["knowledge_fs_control_space"]
|
||||
@@ -6,6 +6,7 @@ from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from configs import dify_config
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from events.app_event import app_was_created
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
@@ -42,7 +43,7 @@ def reset_encrypt_key_pair():
|
||||
After the reset, all LLM credentials will become invalid, requiring re-entry.
|
||||
Only support SELF_HOSTED mode.
|
||||
"""
|
||||
if dify_config.EDITION != "SELF_HOSTED":
|
||||
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
|
||||
click.echo(click.style("This command is only for SELF_HOSTED installations.", fg="red"))
|
||||
return
|
||||
with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
|
||||
|
||||
@@ -1,72 +1,30 @@
|
||||
"""Configuration for the optional KnowledgeFS control-plane integration."""
|
||||
"""Configuration for the optional KnowledgeFS Console bridge."""
|
||||
|
||||
from ipaddress import ip_address
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
|
||||
from pydantic import Field, PositiveFloat, SecretStr, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class KnowledgeFSConfig(BaseSettings):
|
||||
"""Server-only KnowledgeFS connection and rollout settings."""
|
||||
"""Server-only settings for the KnowledgeFS production connection."""
|
||||
|
||||
KNOWLEDGE_FS_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable the KnowledgeFS control-plane product routes.",
|
||||
description="Enable the private KnowledgeFS Console bridge.",
|
||||
)
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
|
||||
)
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
|
||||
)
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
|
||||
)
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
|
||||
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN: str | None = Field(
|
||||
KNOWLEDGE_FS_JWT_SECRET: SecretStr | None = Field(
|
||||
default=None,
|
||||
description="Public KnowledgeFS origin returned with direct upload capabilities.",
|
||||
min_length=32,
|
||||
description="Shared secret used to sign short-lived KnowledgeFS service JWTs.",
|
||||
)
|
||||
KNOWLEDGE_FS_DIRECT_UPLOAD_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that KnowledgeFS direct upload and its browser origin policy are deployed and verified.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
|
||||
default=None,
|
||||
description="Identifier for the current asymmetric Capability v2 signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
|
||||
default=None,
|
||||
description="Server-only PEM for the current Capability v2 RSA signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
|
||||
default=None,
|
||||
description="Optional public-only JWKS JSON retained during key rotation overlap.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS: PositiveFloat = Field(default=300.0, le=3600.0, allow_inf_nan=False)
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
|
||||
|
||||
@field_validator(
|
||||
"KNOWLEDGE_FS_BASE_URL",
|
||||
"KNOWLEDGE_FS_DIRECT_ORIGIN",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
|
||||
"KNOWLEDGE_FS_JWT_SECRET",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
@@ -82,57 +40,25 @@ class KnowledgeFSConfig(BaseSettings):
|
||||
@field_validator("KNOWLEDGE_FS_BASE_URL")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
|
||||
|
||||
@field_validator("KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
@classmethod
|
||||
def validate_direct_origin(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
|
||||
@classmethod
|
||||
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must be an absolute HTTP(S) URL")
|
||||
try:
|
||||
_ = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{name} must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
|
||||
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must not include credentials, query, or fragment")
|
||||
return value.rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
|
||||
if str(getattr(self, "DEPLOY_ENV", "")).strip().upper() == "PRODUCTION":
|
||||
for name, value in (
|
||||
("KNOWLEDGE_FS_BASE_URL", self.KNOWLEDGE_FS_BASE_URL),
|
||||
("KNOWLEDGE_FS_DIRECT_ORIGIN", self.KNOWLEDGE_FS_DIRECT_ORIGIN),
|
||||
):
|
||||
if value and not self._is_secure_or_loopback_origin(value):
|
||||
raise ValueError(f"{name} must use HTTPS in production unless it targets loopback")
|
||||
if self.KNOWLEDGE_FS_ENABLED:
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
|
||||
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
|
||||
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
|
||||
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
|
||||
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
|
||||
):
|
||||
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
|
||||
if not self.KNOWLEDGE_FS_ENABLED:
|
||||
return self
|
||||
if bool(self.KNOWLEDGE_FS_BASE_URL) != bool(self.KNOWLEDGE_FS_JWT_SECRET):
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL and KNOWLEDGE_FS_JWT_SECRET must be configured together")
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS connection settings are required when the integration is enabled")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _is_secure_or_loopback_origin(value: str) -> bool:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme == "https":
|
||||
return True
|
||||
hostname = (parsed.hostname or "").rstrip(".").lower()
|
||||
if hostname == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ip_address(hostname).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@@ -77,5 +77,3 @@ COOKIE_NAME_PASSPORT = "passport"
|
||||
HEADER_NAME_CSRF_TOKEN = "X-CSRF-Token"
|
||||
HEADER_NAME_APP_CODE = "X-App-Code"
|
||||
HEADER_NAME_PASSPORT = "X-App-Passport"
|
||||
HEADER_NAME_IDEMPOTENCY_KEY = "Idempotency-Key"
|
||||
HEADER_NAME_REQUEST_ID = "X-Request-ID"
|
||||
|
||||
@@ -10,6 +10,7 @@ from extensions.ext_database import db
|
||||
from libs.login import current_account_with_tenant
|
||||
from models.dataset import Dataset
|
||||
from models.model import App
|
||||
from services.agent.roster_service import AgentRosterService
|
||||
from services.enterprise.rbac_service import RBACService
|
||||
|
||||
__all__ = ["RBACPermission", "RBACResourceScope", "enforce_rbac_access", "rbac_permission_required"]
|
||||
@@ -51,7 +52,7 @@ def enforce_rbac_access(
|
||||
check_resource_type = None if resource_type == RBACResourceScope.WORKSPACE else resource_type
|
||||
resource_id = None
|
||||
if resource_required and check_resource_type:
|
||||
resource_id = _extract_resource_id(resource_type, path_args)
|
||||
resource_id = _extract_resource_id(resource_type, tenant_id, path_args)
|
||||
if _is_resource_owned_by_current_user(tenant_id, account_id, resource_type, resource_id):
|
||||
return
|
||||
allowed = RBACService.CheckAccess.check(
|
||||
@@ -131,11 +132,14 @@ def _is_resource_owned_by_current_user(
|
||||
return False
|
||||
|
||||
|
||||
def _extract_resource_id(resource_type: RBACResourceScope, path_args: dict[str, object] | None = None) -> str:
|
||||
def _extract_resource_id(
|
||||
resource_type: RBACResourceScope, tenant_id: str, path_args: dict[str, object] | None = None
|
||||
) -> str:
|
||||
"""Extract the resource ID from matched path arguments.
|
||||
|
||||
Some legacy route classes use neutral names such as ``resource_id`` for
|
||||
app/dataset resources, and Agent App routes use ``agent_id`` as the app id.
|
||||
app/dataset resources, and Agent routes carry ``agent_id``, which is
|
||||
resolved to the App backing that Agent.
|
||||
Dataset endpoints behind a rag-pipeline route contain ``pipeline_id``
|
||||
instead of ``dataset_id``. In that case we look up the associated
|
||||
``Dataset`` row via ``Dataset.pipeline_id``.
|
||||
@@ -146,10 +150,19 @@ def _extract_resource_id(resource_type: RBACResourceScope, path_args: dict[str,
|
||||
matched_args = {**view_args, **(path_args or {})}
|
||||
|
||||
if resource_type == RBACResourceScope.APP:
|
||||
app_id = matched_args.get("app_id") or matched_args.get("agent_id") or matched_args.get("resource_id")
|
||||
if not app_id:
|
||||
raise ValueError("Missing app_id in request path")
|
||||
return str(app_id)
|
||||
app_id = matched_args.get("app_id")
|
||||
if app_id:
|
||||
return str(app_id)
|
||||
|
||||
agent_id = matched_args.get("agent_id")
|
||||
if agent_id:
|
||||
authz_app_id = AgentRosterService(db.session).peek_authz_app_id(tenant_id=tenant_id, agent_id=str(agent_id))
|
||||
return authz_app_id or str(agent_id)
|
||||
|
||||
resource_id = matched_args.get("resource_id")
|
||||
if resource_id:
|
||||
return str(resource_id)
|
||||
raise ValueError("Missing app_id in request path")
|
||||
|
||||
if resource_type == RBACResourceScope.DATASET:
|
||||
dataset_id = matched_args.get("dataset_id") or matched_args.get("resource_id")
|
||||
|
||||
@@ -38,6 +38,7 @@ from . import (
|
||||
feature,
|
||||
human_input_form,
|
||||
init_validate,
|
||||
knowledge_fs_proxy,
|
||||
notification,
|
||||
onboarding,
|
||||
ping,
|
||||
@@ -126,7 +127,6 @@ from .explore import (
|
||||
saved_message,
|
||||
trial,
|
||||
)
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .snippets import snippet_workflow, snippet_workflow_draft_variable
|
||||
from .socketio import workflow as socketio_workflow
|
||||
|
||||
@@ -144,6 +144,7 @@ from .workspace import (
|
||||
models,
|
||||
plugin,
|
||||
rbac,
|
||||
skills,
|
||||
snippets,
|
||||
tool_providers,
|
||||
trigger_providers,
|
||||
@@ -197,7 +198,7 @@ __all__ = [
|
||||
"human_input_form",
|
||||
"init_validate",
|
||||
"installed_app",
|
||||
"knowledge_fs_resources",
|
||||
"knowledge_fs_proxy",
|
||||
"load_balancing_config",
|
||||
"login",
|
||||
"mcp_server",
|
||||
@@ -225,6 +226,7 @@ __all__ = [
|
||||
"saved_message",
|
||||
"setup",
|
||||
"site",
|
||||
"skills",
|
||||
"snippet_workflow",
|
||||
"snippet_workflow_draft_variable",
|
||||
"snippets",
|
||||
|
||||
@@ -230,6 +230,7 @@ class WorkflowAgentComposerSaveToRosterApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -439,6 +440,7 @@ class SnippetAgentComposerSaveToRosterApi(Resource):
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
|
||||
)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -478,6 +480,7 @@ class AgentComposerApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
|
||||
@@ -534,9 +534,23 @@ def _parse_observability_time_range(start: str | None, end: str | None, account:
|
||||
|
||||
|
||||
def _query_values(name: str, alias_name: str | None = None) -> list[str]:
|
||||
values = request.args.getlist(name)
|
||||
def _get_values(field_name: str) -> list[str]:
|
||||
values = request.args.getlist(field_name)
|
||||
indexed_values: list[tuple[int, list[str]]] = []
|
||||
prefix = f"{field_name}["
|
||||
for key in request.args:
|
||||
if not key.startswith(prefix) or not key.endswith("]"):
|
||||
continue
|
||||
index = key[len(prefix) : -1]
|
||||
if index.isdigit():
|
||||
indexed_values.append((int(index), request.args.getlist(key)))
|
||||
for _, items in sorted(indexed_values):
|
||||
values.extend(items)
|
||||
return values
|
||||
|
||||
values = _get_values(name)
|
||||
if alias_name:
|
||||
values.extend(request.args.getlist(alias_name))
|
||||
values.extend(_get_values(alias_name))
|
||||
return [value.strip() for value in values if value.strip()]
|
||||
|
||||
|
||||
@@ -547,6 +561,7 @@ class AgentAppListApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -584,6 +599,7 @@ class AgentAppListApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -625,6 +641,7 @@ class AgentAppApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -650,6 +667,7 @@ class AgentAppApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
def delete(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -707,6 +725,7 @@ class AgentPublishApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -729,6 +748,7 @@ class AgentBuildDraftCheckoutApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -805,6 +825,7 @@ class AgentBuildDraftApplyApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -827,6 +848,7 @@ class AgentAppCopyApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -852,6 +874,7 @@ class AgentApiAccessApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -868,6 +891,7 @@ class AgentApiStatusApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -886,6 +910,7 @@ class AgentApiKeyListApi(BaseApiKeyListResource):
|
||||
token_prefix = "app-"
|
||||
|
||||
@console_ns.response(200, "Agent service API keys", console_ns.models[ApiKeyList.__name__])
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID) -> dict[str, object]:
|
||||
@@ -896,6 +921,7 @@ class AgentApiKeyListApi(BaseApiKeyListResource):
|
||||
@console_ns.response(400, "Maximum keys exceeded")
|
||||
@with_current_tenant_id
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, agent_id: UUID) -> tuple[dict[str, object], int]:
|
||||
@@ -915,6 +941,7 @@ class AgentApiKeyApi(BaseApiKeyResource):
|
||||
@console_ns.response(204, "Agent service API key deleted")
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_session
|
||||
def delete(
|
||||
@@ -960,6 +987,7 @@ class AgentLogsApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -998,6 +1026,7 @@ class AgentLogMessagesApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1036,6 +1065,7 @@ class AgentLogSourcesApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1056,6 +1086,7 @@ class AgentStatisticsSummaryApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1081,6 +1112,7 @@ class AgentRosterVersionsApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -1096,6 +1128,7 @@ class AgentRosterVersionDetailApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID, version_id: UUID):
|
||||
@@ -1116,6 +1149,7 @@ class AgentRosterVersionRestoreApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
|
||||
@@ -12,6 +12,7 @@ from werkzeug.exceptions import Forbidden
|
||||
from configs import dify_config
|
||||
from controllers.common.schema import register_response_schema_models
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import dump_response, to_timestamp
|
||||
from libs.login import login_required
|
||||
@@ -194,6 +195,7 @@ class AppApiKeyListResource(BaseApiKeyListResource):
|
||||
@console_ns.doc(params={"resource_id": "App ID"})
|
||||
@console_ns.response(200, "API keys retrieved successfully", console_ns.models[ApiKeyList.__name__])
|
||||
@with_current_tenant_id
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, current_tenant_id: str, resource_id: UUID) -> dict[str, object]:
|
||||
"""Get all API keys for an app"""
|
||||
@@ -210,6 +212,7 @@ class AppApiKeyListResource(BaseApiKeyListResource):
|
||||
@with_current_tenant_id
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
def post(self, session: Session, current_tenant_id: str, resource_id: UUID) -> tuple[dict[str, object], int]:
|
||||
"""Create a new API key for an app"""
|
||||
@@ -233,6 +236,7 @@ class AppApiKeyResource(BaseApiKeyResource):
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
def delete(
|
||||
self,
|
||||
|
||||
@@ -9,7 +9,7 @@ from flask_restx import Resource
|
||||
from pydantic import AliasChoices, BaseModel, Field, ValidationInfo, computed_field, field_validator, model_validator
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import BadRequest, NotFound
|
||||
from werkzeug.exceptions import BadRequest, Forbidden, NotFound
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.app_access import resolve_app_access_filter
|
||||
@@ -23,7 +23,7 @@ from controllers.common.schema import (
|
||||
register_schema_models,
|
||||
)
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.wraps import get_app_model, with_session
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app, get_app_model, with_session
|
||||
from controllers.console.workspace.models import LoadBalancingPayload
|
||||
from controllers.console.wraps import (
|
||||
RBACPermission,
|
||||
@@ -76,6 +76,7 @@ from services.entities.knowledge_entities.knowledge_entities import (
|
||||
WeightModel,
|
||||
WeightVectorSetting,
|
||||
)
|
||||
from services.errors.account import NoPermissionError
|
||||
from services.feature_service import FeatureService
|
||||
from tasks.initialize_created_app_rbac_access_task import initialize_created_app_rbac_access_task
|
||||
|
||||
@@ -904,6 +905,7 @@ class AppApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def put(self, session: Session, app_model: App):
|
||||
@@ -938,6 +940,7 @@ class AppApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_DELETE)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model
|
||||
def delete(self, session: Session, app_model: App):
|
||||
@@ -962,6 +965,7 @@ class AppCopyApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_CREATE_AND_MANAGEMENT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@get_app_model(mode=None)
|
||||
@@ -973,16 +977,19 @@ class AppCopyApi(Resource):
|
||||
with Session(db.engine, expire_on_commit=False) as session:
|
||||
import_service = AppDslService(session)
|
||||
yaml_content = import_service.export_dsl(app_model=app_model, session=session, include_secret=True)
|
||||
result = import_service.import_app(
|
||||
account=current_user,
|
||||
import_mode=ImportMode.YAML_CONTENT,
|
||||
yaml_content=yaml_content,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
)
|
||||
try:
|
||||
result = import_service.import_app(
|
||||
account=current_user,
|
||||
import_mode=ImportMode.YAML_CONTENT,
|
||||
yaml_content=yaml_content,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
return dump_response(AppImportResponse, result), 400
|
||||
@@ -1036,6 +1043,7 @@ class AppExportApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_IMPORT_EXPORT_DSL)
|
||||
@agent_manage_required_for_agent_app
|
||||
@get_app_model
|
||||
def get(self, app_model: App):
|
||||
"""Export app"""
|
||||
@@ -1060,6 +1068,7 @@ class AppPublishToCreatorsPlatformApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_IMPORT_EXPORT_DSL)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_current_user_id
|
||||
@get_app_model(mode=None)
|
||||
def post(self, current_user_id: str, app_model: App):
|
||||
@@ -1090,6 +1099,7 @@ class AppNameApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1117,6 +1127,7 @@ class AppIconApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1150,6 +1161,7 @@ class AppSiteStatus(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1177,6 +1189,7 @@ class AppApiStatus(Resource):
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.schema import register_enum_models, register_schema_models
|
||||
@@ -28,6 +29,7 @@ from services.app_dsl_service import (
|
||||
)
|
||||
from services.enterprise.enterprise_service import EnterpriseService
|
||||
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
|
||||
from services.errors.account import NoPermissionError
|
||||
from services.feature_service import FeatureService
|
||||
|
||||
from .. import console_ns
|
||||
@@ -91,18 +93,21 @@ class AppImportApi(Resource):
|
||||
import_service = AppDslService(session)
|
||||
# Import app
|
||||
account = current_user
|
||||
result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode=args.mode,
|
||||
yaml_content=args.yaml_content,
|
||||
yaml_url=args.yaml_url,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
app_id=args.app_id,
|
||||
)
|
||||
try:
|
||||
result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode=args.mode,
|
||||
yaml_content=args.yaml_content,
|
||||
yaml_url=args.yaml_url,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
app_id=args.app_id,
|
||||
)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
else:
|
||||
@@ -157,7 +162,10 @@ class AppImportConfirmApi(Resource):
|
||||
import_service = AppDslService(session)
|
||||
# Confirm import
|
||||
account = current_user
|
||||
result = import_service.confirm_import(import_id=import_id, account=account)
|
||||
try:
|
||||
result = import_service.confirm_import(import_id=import_id, account=account)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
else:
|
||||
|
||||
@@ -351,24 +351,31 @@ def _resolve_current_user_agent_debug_conversation_id(
|
||||
app_model: App,
|
||||
agent_id: str | None,
|
||||
draft_type: AgentConfigDraftType,
|
||||
start_new: bool = False,
|
||||
) -> str:
|
||||
"""Resolve the current editor's conversation without crossing draft surfaces."""
|
||||
"""Resolve or rotate the current editor's conversation within one draft surface.
|
||||
|
||||
``start_new`` rotates the scoped mapping through ``AgentRosterService`` so
|
||||
the old runtime session is retired before the new conversation is used.
|
||||
Continuations and Build chat keep resolving the existing mapping.
|
||||
"""
|
||||
|
||||
roster_service = AgentRosterService(session)
|
||||
if agent_id:
|
||||
return roster_service.get_or_create_agent_app_debug_conversation_id(
|
||||
tenant_id=current_tenant_id,
|
||||
agent_id=agent_id,
|
||||
account_id=current_user.id,
|
||||
draft_type=draft_type,
|
||||
)
|
||||
resolved_agent_id = agent_id
|
||||
if not resolved_agent_id:
|
||||
agent = roster_service.get_app_backing_agent(tenant_id=current_tenant_id, app_id=str(app_model.id))
|
||||
if agent is None:
|
||||
raise AgentNotFoundError()
|
||||
resolved_agent_id = agent.id
|
||||
|
||||
agent = roster_service.get_app_backing_agent(tenant_id=current_tenant_id, app_id=str(app_model.id))
|
||||
if agent is None:
|
||||
raise AgentNotFoundError()
|
||||
return roster_service.get_or_create_agent_app_debug_conversation_id(
|
||||
resolve_conversation = (
|
||||
roster_service.refresh_agent_app_debug_conversation_id
|
||||
if start_new
|
||||
else roster_service.get_or_create_agent_app_debug_conversation_id
|
||||
)
|
||||
return resolve_conversation(
|
||||
tenant_id=current_tenant_id,
|
||||
agent_id=agent.id,
|
||||
agent_id=resolved_agent_id,
|
||||
account_id=current_user.id,
|
||||
draft_type=draft_type,
|
||||
)
|
||||
@@ -387,13 +394,17 @@ def _create_chat_message(
|
||||
args = args_model.model_dump(exclude_none=True, by_alias=True)
|
||||
|
||||
if AppMode.value_of(app_model.mode) == AppMode.AGENT:
|
||||
draft_type = AgentConfigDraftType(args_model.draft_type)
|
||||
# Preview follows the normal chat contract: an omitted/empty conversation ID starts a new
|
||||
# conversation. Build chat keeps its stable mapping so build drafts and finalization stay continuous.
|
||||
debug_conversation_id = _resolve_current_user_agent_debug_conversation_id(
|
||||
session=session,
|
||||
current_tenant_id=current_tenant_id or app_model.tenant_id,
|
||||
current_user=current_user,
|
||||
app_model=app_model,
|
||||
agent_id=agent_id,
|
||||
draft_type=AgentConfigDraftType(args_model.draft_type),
|
||||
draft_type=draft_type,
|
||||
start_new=draft_type == AgentConfigDraftType.DRAFT and not args_model.conversation_id,
|
||||
)
|
||||
if args_model.conversation_id and args_model.conversation_id != debug_conversation_id:
|
||||
raise NotFound("Conversation Not Exists.")
|
||||
|
||||
@@ -10,7 +10,7 @@ from constants.languages import supported_language
|
||||
from controllers.common.schema import register_schema_models
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.wraps import get_app_model
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app, get_app_model
|
||||
from controllers.console.wraps import (
|
||||
RBACPermission,
|
||||
RBACResourceScope,
|
||||
@@ -93,6 +93,7 @@ class AppSite(Resource):
|
||||
@login_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_session
|
||||
@@ -145,6 +146,7 @@ class AppSiteAccessTokenReset(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_session
|
||||
|
||||
@@ -12,14 +12,22 @@ from typing import cast, overload
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.session import with_session
|
||||
from controllers.common.wraps import RBACPermission, RBACResourceScope, enforce_rbac_access
|
||||
from controllers.console.app.error import AppNotFoundError
|
||||
from extensions.ext_database import db
|
||||
from libs.login import current_account_with_tenant
|
||||
from models import App, AppMode, TrialApp
|
||||
from models.agent import AgentScope
|
||||
from services.recommended_app_service import RecommendedAppService
|
||||
|
||||
__all__ = ["get_app_model", "get_app_model_with_trial", "with_session"]
|
||||
__all__ = [
|
||||
"agent_manage_required_for_agent_app",
|
||||
"get_app_model",
|
||||
"get_app_model_with_trial",
|
||||
"with_session",
|
||||
]
|
||||
|
||||
|
||||
def _load_app_model(session: Session, app_id: str) -> App | None:
|
||||
@@ -48,6 +56,45 @@ def _load_app_model_with_trial(session: Session, app_id: str) -> App | None:
|
||||
return app_model
|
||||
|
||||
|
||||
def agent_manage_required_for_agent_app[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
"""Gate generic app management routes that target an Agent App.
|
||||
|
||||
A hidden workflow-only backing App only reuses the App runtime and is not
|
||||
part of the general app management plane, so generic routes reject it
|
||||
outright. Managing a roster Agent App mutates the roster Agent behind it
|
||||
(rename/icon sync, archive, API enablement), so it additionally requires
|
||||
workspace ``agent.manage`` on top of the route's existing App permission
|
||||
checks when RBAC is enabled. A no-op for non-agent Apps. Must be placed
|
||||
above ``get_app_model`` so the ``app_id`` path parameter is still present.
|
||||
"""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
raw_app_id = kwargs.get("app_id") or kwargs.get("resource_id")
|
||||
if raw_app_id is not None:
|
||||
app_model = _load_app_model_from_scoped_session(str(raw_app_id))
|
||||
binding = (
|
||||
app_model.agent_app_binding_with_session(session=db.session(), include_archived=True)
|
||||
if app_model is not None
|
||||
else None
|
||||
)
|
||||
if binding is not None:
|
||||
if binding.scope == AgentScope.WORKFLOW_ONLY:
|
||||
raise AppNotFoundError()
|
||||
if dify_config.RBAC_ENABLED:
|
||||
current_user, current_tenant_id = current_account_with_tenant()
|
||||
enforce_rbac_access(
|
||||
tenant_id=current_tenant_id,
|
||||
account_id=current_user.id,
|
||||
resource_type=RBACResourceScope.WORKSPACE,
|
||||
scene=RBACPermission.AGENT_MANAGE,
|
||||
resource_required=False,
|
||||
)
|
||||
return view(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _get_injected_session(args: tuple[object, ...]) -> Session | None:
|
||||
"""Return the request session inserted by `with_session`, if this handler has been migrated."""
|
||||
if len(args) < 2:
|
||||
|
||||
@@ -7,10 +7,13 @@ from configs import dify_config
|
||||
from constants.languages import supported_language
|
||||
from controllers.common.schema import query_params_from_model, register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.auth.error import InvitationAccountMismatchError
|
||||
from controllers.console.error import AccountInFreezeError, AlreadyActivateError
|
||||
from extensions.ext_database import db
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from libs.helper import EmailStr, timezone
|
||||
from libs.login import current_account_with_tenant
|
||||
from libs.token import extract_access_token
|
||||
from models import AccountStatus
|
||||
from models.account import TenantAccountJoin, TenantAccountRole
|
||||
from services.account_service import RegisterService, TenantService
|
||||
@@ -136,6 +139,12 @@ class ActivateApi(Resource):
|
||||
)
|
||||
@console_ns.response(400, "Already activated or invalid token")
|
||||
def post(self):
|
||||
"""Accept an invitation without letting an existing session act for another account.
|
||||
|
||||
Token-only activation remains available for legacy clients. When the request already
|
||||
carries a console session, that session must belong to the account encoded in the
|
||||
invitation before the token is consumed or tenant membership is changed.
|
||||
"""
|
||||
args = ActivatePayload.model_validate(console_ns.payload)
|
||||
|
||||
normalized_request_email = args.email.lower() if args.email else None
|
||||
@@ -146,6 +155,11 @@ class ActivateApi(Resource):
|
||||
raise AlreadyActivateError()
|
||||
|
||||
account = invitation["account"]
|
||||
if extract_access_token(request):
|
||||
current_account, _ = current_account_with_tenant()
|
||||
if current_account.id != account.id:
|
||||
raise InvitationAccountMismatchError()
|
||||
|
||||
if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(account.email):
|
||||
raise AccountInFreezeError()
|
||||
|
||||
|
||||
@@ -13,6 +13,12 @@ class InvalidEmailError(BaseHTTPException):
|
||||
code = 400
|
||||
|
||||
|
||||
class InvitationAccountMismatchError(BaseHTTPException):
|
||||
error_code = "invitation_account_mismatch"
|
||||
description = "This invitation was sent to another account. Please sign in with the invited account."
|
||||
code = 403
|
||||
|
||||
|
||||
class PasswordMismatchError(BaseHTTPException):
|
||||
error_code = "password_mismatch"
|
||||
description = "The passwords do not match."
|
||||
|
||||
@@ -56,6 +56,7 @@ class OAuthProviderTokenResponse(BaseModel):
|
||||
|
||||
|
||||
class OAuthProviderAccountResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
email: str
|
||||
avatar: str | None = None
|
||||
@@ -251,6 +252,7 @@ class OAuthServerUserAccountApi(Resource):
|
||||
def post(self, oauth_provider_app: OAuthProviderApp, account: Account):
|
||||
return jsonable_encoder(
|
||||
{
|
||||
"id": account.id,
|
||||
"name": account.name,
|
||||
"email": account.email,
|
||||
"avatar": account.avatar,
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.fastopenapi import console_router
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from extensions.ext_database import db
|
||||
from models.model import DifySetup
|
||||
from services.account_service import TenantService
|
||||
@@ -63,7 +64,7 @@ def validate_init_password(payload: InitValidatePayload) -> InitValidateResponse
|
||||
|
||||
|
||||
def get_init_validate_status() -> bool:
|
||||
if dify_config.EDITION == "SELF_HOSTED":
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
|
||||
if os.environ.get("INIT_PASSWORD"):
|
||||
if session.get("is_init_validated"):
|
||||
return True
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Typed Dify-owned KnowledgeFS Console product API."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Stable, non-enumerating Console error contract for KnowledgeFS."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSSpaceNotFoundHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_space_not_found"
|
||||
description = "KnowledgeFS space was not found."
|
||||
code = 404
|
||||
|
||||
|
||||
class KnowledgeFSOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSAccessDeniedHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_access_denied"
|
||||
description = "KnowledgeFS operation is not allowed."
|
||||
code = 403
|
||||
|
||||
|
||||
class KnowledgeFSRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSAccessDeniedHTTPError",
|
||||
"KnowledgeFSInvalidRequestHTTPError",
|
||||
"KnowledgeFSOperationUnavailableHTTPError",
|
||||
"KnowledgeFSQuotaExceededHTTPError",
|
||||
"KnowledgeFSRateLimitHTTPError",
|
||||
"KnowledgeFSSpaceNotFoundHTTPError",
|
||||
"KnowledgeFSUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,419 @@
|
||||
"""Authenticated transport adapter for the Console-to-KnowledgeFS proxy.
|
||||
|
||||
These raw Blueprint routes deliberately stay outside Dify's OpenAPI surface:
|
||||
KnowledgeFS owns the wire contract consumed by the frontend. The catch-all path
|
||||
avoids resource-specific Dify controllers, while the forwarding module consumes
|
||||
only the operations explicitly enabled by Dify's product registry. The registry
|
||||
can be validated explicitly against the pinned KnowledgeFS contract during development.
|
||||
Console auth and contract-specific dataset RBAC run before forwarding. Request
|
||||
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
|
||||
SSE responses remain streaming with a bounded idle read timeout, and only safe
|
||||
response headers are exposed. Operation-specific upstream error mappings are
|
||||
applied before Console JSON error handling; the default maps 401 to 502 so it
|
||||
cannot trigger browser-session recovery and preserves resource-level 403.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Iterator
|
||||
from functools import wraps
|
||||
from http import HTTPStatus
|
||||
from typing import NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from flask import Response, request, stream_with_context
|
||||
from flask.typing import ResponseReturnValue
|
||||
from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
GatewayTimeout,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
default_exceptions,
|
||||
)
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.console import api, bp
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
cloud_edition_billing_rate_limit_check,
|
||||
setup_required,
|
||||
)
|
||||
from core.helper import ssrf_proxy
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from services.knowledge_fs_operations import KnowledgeFSMethod
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
type _KnowledgeFSRequestForwarder = Callable[
|
||||
[str | None, str | None, bytes | None, bytes | None],
|
||||
KnowledgeFSUpstreamResponse,
|
||||
]
|
||||
|
||||
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
|
||||
_RESPONSE_HEADER_ALLOWLIST = (
|
||||
"Cache-Control",
|
||||
"Content-Disposition",
|
||||
"Content-Type",
|
||||
"Retry-After",
|
||||
"X-Trace-Id",
|
||||
)
|
||||
_RESPONSE_HEADER_DENYLIST = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"connection",
|
||||
"cookie",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"set-cookie",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _console_api_errors[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Route raw Blueprint exceptions through the Console API JSON handlers."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
try:
|
||||
return view(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
return api.handle_error(exc)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _knowledge_fs_enabled[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Hide the complete KnowledgeFS route surface while the bridge is disabled."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
return view(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
|
||||
"""Map forwarding failures to the stable Console HTTP error surface."""
|
||||
if isinstance(exc, KnowledgeFSRouteNotAllowedError):
|
||||
raise NotFound() from exc
|
||||
if isinstance(exc, KnowledgeFSAccessDeniedError):
|
||||
raise Forbidden() from exc
|
||||
if isinstance(exc, KnowledgeFSConfigurationError):
|
||||
logger.error("KnowledgeFS request was blocked by invalid configuration for tenant_id=%s", tenant_id)
|
||||
raise ServiceUnavailable("KnowledgeFS integration is misconfigured") from exc
|
||||
if isinstance(exc, KnowledgeFSTimeoutError):
|
||||
raise GatewayTimeout("KnowledgeFS request timed out") from exc
|
||||
if isinstance(exc, KnowledgeFSTransportError):
|
||||
logger.warning("KnowledgeFS transport request failed for tenant_id=%s", tenant_id)
|
||||
raise BadGateway("KnowledgeFS is unavailable") from exc
|
||||
raise exc
|
||||
|
||||
|
||||
def _knowledge_fs_operation_access_required(
|
||||
view: Callable[[KnowledgeFSAuthorization], ResponseReturnValue],
|
||||
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
|
||||
"""Authorize one declared operation before billing and request-body work."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=current_user,
|
||||
tenant_id=tenant_id,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
except KnowledgeFSAccessDeniedError as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return view(authorization)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _request_body() -> bytes:
|
||||
"""Read the raw body up to the proxy limit or raise RequestEntityTooLarge."""
|
||||
body = request.stream.read(_MAX_PROXY_BODY_BYTES + 1)
|
||||
if len(body) > _MAX_PROXY_BODY_BYTES:
|
||||
raise RequestEntityTooLarge("KnowledgeFS proxy request body is too large")
|
||||
return body
|
||||
|
||||
|
||||
def _stream_response_body(
|
||||
upstream: httpx.Response,
|
||||
*,
|
||||
tenant_id: str,
|
||||
max_response_bytes: int,
|
||||
) -> Iterator[bytes]:
|
||||
"""Yield one bounded SSE response and always release its pooled connection."""
|
||||
total_bytes = 0
|
||||
try:
|
||||
for chunk in upstream.iter_bytes():
|
||||
total_bytes += len(chunk)
|
||||
if total_bytes > max_response_bytes:
|
||||
logger.warning("KnowledgeFS stream exceeded the proxy limit for tenant_id=%s", tenant_id)
|
||||
raise ssrf_proxy.ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
|
||||
yield chunk
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
|
||||
def _proxy_response(
|
||||
upstream_result: KnowledgeFSUpstreamResponse,
|
||||
*,
|
||||
tenant_id: str,
|
||||
contract_response_headers: tuple[str, ...],
|
||||
max_response_bytes: int,
|
||||
) -> Response:
|
||||
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: KnowledgeFS returns a status normalized by the operation contract.
|
||||
"""
|
||||
upstream = upstream_result.response
|
||||
mapped_status = dict(upstream_result.operation.error_status_map).get(upstream.status_code)
|
||||
if mapped_status is not None:
|
||||
upstream.close()
|
||||
description = "KnowledgeFS upstream request failed"
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
description = "KnowledgeFS authentication failed"
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
exception_type = default_exceptions.get(mapped_status)
|
||||
if exception_type is None:
|
||||
exception = HTTPException(description)
|
||||
exception.code = mapped_status
|
||||
raise exception
|
||||
raise exception_type(description)
|
||||
|
||||
allowed_header_names = dict.fromkeys(
|
||||
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
|
||||
)
|
||||
headers = {
|
||||
name: value
|
||||
for name in allowed_header_names
|
||||
if name not in _RESPONSE_HEADER_DENYLIST
|
||||
if (value := upstream.headers.get(name)) is not None
|
||||
}
|
||||
if upstream_result.response_kind == "stream":
|
||||
response = Response(
|
||||
stream_with_context( # pyrefly: ignore[no-matching-overload]
|
||||
_stream_response_body(
|
||||
upstream,
|
||||
tenant_id=tenant_id,
|
||||
max_response_bytes=max_response_bytes,
|
||||
)
|
||||
),
|
||||
status=upstream.status_code,
|
||||
headers=headers,
|
||||
)
|
||||
response.call_on_close(upstream.close)
|
||||
return response
|
||||
|
||||
try:
|
||||
content = upstream.content
|
||||
finally:
|
||||
upstream.close()
|
||||
return Response(content, status=upstream.status_code, headers=headers)
|
||||
|
||||
|
||||
def _proxy_current_request(
|
||||
*,
|
||||
method: KnowledgeFSMethod,
|
||||
tenant_id: str,
|
||||
forward: _KnowledgeFSRequestForwarder,
|
||||
) -> Response:
|
||||
"""Forward the current raw request through one preconfigured service entry."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
try:
|
||||
proxy_result = forward(
|
||||
request.headers.get("Accept"),
|
||||
request.content_type,
|
||||
request.query_string or None,
|
||||
_request_body() if method != "GET" else None,
|
||||
)
|
||||
except (
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
) as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return _proxy_response(
|
||||
proxy_result,
|
||||
tenant_id=tenant_id,
|
||||
contract_response_headers=proxy_result.operation.response_headers,
|
||||
max_response_bytes=proxy_result.operation.max_response_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _proxy_request(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
) -> Response:
|
||||
"""Authorize and forward the current request through the combined service use case."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
def _proxy_authorized_request(authorization: KnowledgeFSAuthorization) -> Response:
|
||||
"""Forward the current request using one previously authorized operation capability.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability produced before billing and body parsing.
|
||||
|
||||
Returns:
|
||||
The filtered response returned by KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: The integration is disabled or forwarding fails.
|
||||
"""
|
||||
operation = authorization.operation
|
||||
tenant_id = authorization.tenant_id
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=operation.method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
@_knowledge_fs_enabled
|
||||
@_knowledge_fs_operation_access_required
|
||||
@cloud_edition_billing_rate_limit_check("knowledge")
|
||||
def _proxy_knowledge_fs_non_get(
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
) -> ResponseReturnValue:
|
||||
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
|
||||
return _proxy_authorized_request(authorization)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["OPTIONS"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
def proxy_knowledge_fs_options(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Complete a CORS preflight only for an enabled Console operation."""
|
||||
requested_method = cast(KnowledgeFSMethod, request.headers.get("Access-Control-Request-Method", "").upper())
|
||||
try:
|
||||
get_knowledge_fs_operation(requested_method, upstream_path)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
return Response(status=HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["GET"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_get(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated, dataset-readable GET request.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
if request.method != "GET":
|
||||
raise NotFound()
|
||||
return _proxy_request("GET", upstream_path)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["DELETE", "PATCH", "POST", "PUT"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_write(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated non-GET request under its contract access policy.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
method = cast(KnowledgeFSMethod, request.method)
|
||||
return _proxy_knowledge_fs_non_get(method, upstream_path)
|
||||
@@ -6,6 +6,7 @@ from sqlalchemy import select
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.fastopenapi import console_router
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from libs.helper import EmailStr, extract_remote_ip
|
||||
from libs.password import valid_password
|
||||
from models.model import DifySetup, db
|
||||
@@ -52,7 +53,7 @@ def get_setup_status_api() -> SetupStatusResponse:
|
||||
|
||||
Only bootstrap-safe status information should be returned by this endpoint.
|
||||
"""
|
||||
if dify_config.EDITION == "SELF_HOSTED":
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
|
||||
setup_status = get_setup_status()
|
||||
if setup_status and not isinstance(setup_status, bool):
|
||||
return SetupStatusResponse(step="finished", setup_at=setup_status.setup_at.isoformat())
|
||||
@@ -102,7 +103,7 @@ def setup_system(payload: SetupRequestPayload) -> SetupResponse:
|
||||
|
||||
|
||||
def get_setup_status() -> DifySetup | bool | None:
|
||||
if dify_config.EDITION == "SELF_HOSTED":
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
|
||||
return db.session.scalar(select(DifySetup).limit(1))
|
||||
|
||||
return True
|
||||
|
||||
@@ -59,7 +59,7 @@ class TagBindingRemovePayload(BaseModel):
|
||||
|
||||
|
||||
class TagListQueryParam(BaseModel):
|
||||
type: Literal["knowledge", "app", "snippet", ""] = Field("", description="Tag type filter")
|
||||
type: TagType | Literal[""] = Field("", description="Tag type filter")
|
||||
keyword: str | None = Field(None, description="Search keyword")
|
||||
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ from controllers.console.wraps import (
|
||||
with_current_tenant_id,
|
||||
with_current_user,
|
||||
)
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from extensions.ext_database import db
|
||||
from fields.base import ResponseModel
|
||||
from fields.member_fields import AccountResponse
|
||||
@@ -262,7 +263,7 @@ class AccountInitApi(Resource):
|
||||
payload = console_ns.payload or {}
|
||||
args = AccountInitPayload.model_validate(payload)
|
||||
|
||||
if dify_config.EDITION == "CLOUD":
|
||||
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
|
||||
if not args.invitation_code:
|
||||
raise ValueError("invitation_code is required")
|
||||
|
||||
|
||||
@@ -442,12 +442,10 @@ register_enum_models(
|
||||
)
|
||||
|
||||
|
||||
def _default_auto_upgrade_settings(
|
||||
tenant_id: str,
|
||||
category: TenantPluginAutoUpgradeCategory,
|
||||
) -> AutoUpgradeSettingsResponse:
|
||||
def _missing_auto_upgrade_settings(tenant_id: str) -> AutoUpgradeSettingsResponse:
|
||||
"""Represent a missing persisted strategy as effectively disabled."""
|
||||
return {
|
||||
"strategy_setting": PluginAutoUpgradeService.default_strategy_setting_for_category(category),
|
||||
"strategy_setting": TenantPluginAutoUpgradeStrategySetting.DISABLED,
|
||||
"upgrade_time_of_day": PluginAutoUpgradeService.default_upgrade_time_of_day(tenant_id),
|
||||
"upgrade_mode": TenantPluginAutoUpgradeMode.EXCLUDE,
|
||||
"exclude_plugins": [],
|
||||
@@ -1135,9 +1133,7 @@ class PluginFetchAutoUpgradeApi(Resource):
|
||||
args = ParserAutoUpgradeFetch.model_validate(request.args.to_dict(flat=True))
|
||||
auto_upgrade = PluginAutoUpgradeService.get_strategy(tenant_id, args.category, session=db.session())
|
||||
auto_upgrade_dict = (
|
||||
_auto_upgrade_settings_to_dict(auto_upgrade)
|
||||
if auto_upgrade
|
||||
else _default_auto_upgrade_settings(tenant_id, args.category)
|
||||
_auto_upgrade_settings_to_dict(auto_upgrade) if auto_upgrade else _missing_auto_upgrade_settings(tenant_id)
|
||||
)
|
||||
|
||||
return jsonable_encoder(
|
||||
|
||||
@@ -346,13 +346,7 @@ class RBACRoleItemApi(Resource):
|
||||
def put(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_RoleUpsertRequest)
|
||||
role = svc.RBACService.KnowledgeFSRoleMutations.update_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
request.to_mutation(),
|
||||
session=db.session(),
|
||||
)
|
||||
role = svc.RBACService.Roles.update(tenant_id, account_id, str(role_id), request.to_mutation())
|
||||
return _dump(role)
|
||||
|
||||
@login_required
|
||||
@@ -362,12 +356,7 @@ class RBACRoleItemApi(Resource):
|
||||
@console_ns.response(200, "Success", console_ns.models[svc.RBACRole.__name__])
|
||||
def delete(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
svc.RBACService.KnowledgeFSRoleMutations.delete_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
session=db.session(),
|
||||
)
|
||||
svc.RBACService.Roles.delete(tenant_id, account_id, str(role_id))
|
||||
return {"result": "success"}
|
||||
|
||||
|
||||
@@ -926,7 +915,7 @@ class RBACMemberRolesApi(Resource):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_ReplaceMemberRolesRequest)
|
||||
return _dump(
|
||||
svc.RBACService.KnowledgeFSRoleMutations.replace_member_roles(
|
||||
svc.RBACService.MemberRoles.replace(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(member_id),
|
||||
|
||||
@@ -0,0 +1,834 @@
|
||||
"""Console API for workspace-level Skill Management."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from flask import request, send_file
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
from controllers.common.fields import BinaryFileResponse
|
||||
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
edit_permission_required,
|
||||
setup_required,
|
||||
with_current_tenant_id,
|
||||
with_current_user,
|
||||
)
|
||||
from fields.base import ResponseModel
|
||||
from libs import helper
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models.account import Account
|
||||
from services.skill_management_service import (
|
||||
SkillAssistMessagePayload,
|
||||
SkillCreatePayload,
|
||||
SkillDraftFileOperationPayload,
|
||||
SkillDraftTreePayload,
|
||||
SkillImportPayload,
|
||||
SkillManagementService,
|
||||
SkillManagementServiceError,
|
||||
SkillMetadataPayload,
|
||||
SkillPublishPayload,
|
||||
SkillRestorePayload,
|
||||
SkillVersionUpdatePayload,
|
||||
)
|
||||
|
||||
_FILE_UPLOAD_PARAMS = {
|
||||
"file": {
|
||||
"description": "Skill draft file payload",
|
||||
"in": "formData",
|
||||
"type": "file",
|
||||
"required": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class WorkspaceSkillsQuery(BaseModel):
|
||||
keyword: str | None = Field(default=None, description="Search keyword matching skill name or description.")
|
||||
page: int = Field(default=1, ge=1, le=99999, description="Page number.")
|
||||
limit: int = Field(default=20, ge=1, le=100, description="Number of items per page.")
|
||||
tag: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Skill tag filters. Repeat the parameter for multiple tags.",
|
||||
)
|
||||
|
||||
|
||||
class SkillDeletePayload(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
confirmation_name: str | None = Field(
|
||||
default=None,
|
||||
description="Required when deleting a referenced Skill. Must match the Skill name.",
|
||||
)
|
||||
|
||||
|
||||
class AgentSkillBindingsPayload(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
skill_ids: list[str] = Field(default_factory=list, description="Ordered Skill IDs bound to the Agent.")
|
||||
|
||||
|
||||
class SkillFileQuery(BaseModel):
|
||||
path: str = Field(description="Skill file path relative to the Skill root.")
|
||||
version_id: str | None = Field(default=None, description="Optional published version ID. Omit for current draft.")
|
||||
|
||||
|
||||
class SkillResponse(ResponseModel):
|
||||
id: str
|
||||
name: str
|
||||
display_name: str
|
||||
icon: str
|
||||
description: str
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
name_manually_edited: bool = False
|
||||
visibility: str
|
||||
latest_published_version_id: str | None = None
|
||||
reference_count: int = 0
|
||||
created_by: str | None = None
|
||||
created_by_name: str | None = None
|
||||
updated_by: str | None = None
|
||||
updated_by_name: str | None = None
|
||||
created_at: int
|
||||
updated_at: int
|
||||
|
||||
|
||||
class SkillFileResponse(ResponseModel):
|
||||
id: str | None = None
|
||||
path: str
|
||||
kind: str
|
||||
storage: str | None = None
|
||||
mime_type: str | None = None
|
||||
content: str | None = None
|
||||
tool_file_id: str | None = None
|
||||
size: int | None = None
|
||||
hash: str | None = None
|
||||
|
||||
|
||||
class SkillFilePreviewResponse(ResponseModel):
|
||||
path: str
|
||||
mime_type: str
|
||||
content: str
|
||||
size: int
|
||||
hash: str
|
||||
|
||||
|
||||
class SkillFileUploadResponse(ResponseModel):
|
||||
id: str
|
||||
name: str
|
||||
mime_type: str
|
||||
size: int
|
||||
hash: str
|
||||
|
||||
|
||||
class SkillDetailResponse(SkillResponse):
|
||||
files: list[SkillFileResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkillListResponse(ResponseModel):
|
||||
data: list[SkillResponse] = Field(default_factory=list)
|
||||
has_more: bool = False
|
||||
limit: int = 20
|
||||
page: int = 1
|
||||
total: int = 0
|
||||
|
||||
|
||||
class SkillTagResponse(ResponseModel):
|
||||
tag: str
|
||||
count: int
|
||||
|
||||
|
||||
class SkillTagListResponse(ResponseModel):
|
||||
data: list[SkillTagResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkillVersionResponse(ResponseModel):
|
||||
id: str
|
||||
skill_id: str
|
||||
version_number: int
|
||||
version_name: str
|
||||
publish_note: str
|
||||
hash_code: str
|
||||
archive_size: int
|
||||
published_by: str | None = None
|
||||
published_by_name: str | None = None
|
||||
is_latest: bool = False
|
||||
created_at: int
|
||||
|
||||
|
||||
class SkillVersionListResponse(ResponseModel):
|
||||
data: list[SkillVersionResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkillVersionDetailResponse(SkillVersionResponse):
|
||||
files: list[SkillFileResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkillVersionDeleteResponse(ResponseModel):
|
||||
id: str
|
||||
deleted: bool
|
||||
latest_published_version_id: str | None = None
|
||||
|
||||
|
||||
class SkillReferenceResponse(ResponseModel):
|
||||
type: str
|
||||
agent_id: str
|
||||
agent_icon: str | None = None
|
||||
agent_icon_background: str | None = None
|
||||
agent_icon_type: str | None = None
|
||||
app_id: str | None = None
|
||||
name: str
|
||||
display_name: str
|
||||
workflow_id: str | None = None
|
||||
workflow_name: str | None = None
|
||||
workflow_icon: str | None = None
|
||||
workflow_icon_background: str | None = None
|
||||
workflow_icon_type: str | None = None
|
||||
workflow_version: str | None = None
|
||||
node_id: str | None = None
|
||||
node_name: str | None = None
|
||||
|
||||
|
||||
class SkillReferenceListResponse(ResponseModel):
|
||||
data: list[SkillReferenceResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkillDeleteResponse(ResponseModel):
|
||||
id: str
|
||||
deleted: bool
|
||||
|
||||
|
||||
class AgentSkillBindingItemResponse(ResponseModel):
|
||||
id: str
|
||||
priority: int
|
||||
name: str
|
||||
display_name: str
|
||||
icon: str
|
||||
description: str
|
||||
tags: list[str] = Field(default_factory=list)
|
||||
status: str
|
||||
file_count: int
|
||||
latest_published_version_id: str | None = None
|
||||
latest_published_at: int | None = None
|
||||
updated_at: int
|
||||
|
||||
|
||||
class AgentSkillBindingsResponse(ResponseModel):
|
||||
agent_id: str
|
||||
skill_ids: list[str] = Field(default_factory=list)
|
||||
data: list[AgentSkillBindingItemResponse] = Field(default_factory=list)
|
||||
|
||||
|
||||
register_schema_models(
|
||||
console_ns,
|
||||
WorkspaceSkillsQuery,
|
||||
SkillCreatePayload,
|
||||
SkillAssistMessagePayload,
|
||||
SkillMetadataPayload,
|
||||
SkillDraftFileOperationPayload,
|
||||
SkillDraftTreePayload,
|
||||
SkillPublishPayload,
|
||||
SkillRestorePayload,
|
||||
SkillVersionUpdatePayload,
|
||||
SkillDeletePayload,
|
||||
SkillFileQuery,
|
||||
AgentSkillBindingsPayload,
|
||||
)
|
||||
|
||||
register_response_schema_models(
|
||||
console_ns,
|
||||
SkillResponse,
|
||||
SkillFileResponse,
|
||||
SkillFilePreviewResponse,
|
||||
SkillFileUploadResponse,
|
||||
SkillDetailResponse,
|
||||
SkillListResponse,
|
||||
SkillTagResponse,
|
||||
SkillTagListResponse,
|
||||
SkillVersionResponse,
|
||||
SkillVersionListResponse,
|
||||
SkillVersionDetailResponse,
|
||||
SkillVersionDeleteResponse,
|
||||
SkillReferenceResponse,
|
||||
SkillReferenceListResponse,
|
||||
SkillDeleteResponse,
|
||||
AgentSkillBindingItemResponse,
|
||||
AgentSkillBindingsResponse,
|
||||
BinaryFileResponse,
|
||||
)
|
||||
|
||||
|
||||
def _error_response(exc: SkillManagementServiceError) -> tuple[dict[str, object], int]:
|
||||
body: dict[str, object] = {"code": exc.code, "message": exc.message}
|
||||
if exc.details:
|
||||
body["details"] = exc.details
|
||||
return body, exc.status_code
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills")
|
||||
class WorkspaceSkillsApi(Resource):
|
||||
@console_ns.doc(params=query_params_from_model(WorkspaceSkillsQuery))
|
||||
@console_ns.response(200, "Workspace skills", console_ns.models[SkillListResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str):
|
||||
query_input: dict[str, object] = {
|
||||
"keyword": request.args.get("keyword"),
|
||||
"tag": request.args.getlist("tag"),
|
||||
}
|
||||
if "limit" in request.args:
|
||||
query_input["limit"] = request.args.get("limit")
|
||||
if "page" in request.args:
|
||||
query_input["page"] = request.args.get("page")
|
||||
query = WorkspaceSkillsQuery.model_validate(query_input)
|
||||
result = SkillManagementService().list_skills(
|
||||
tenant_id=current_tenant_id,
|
||||
keyword=query.keyword,
|
||||
page=query.page,
|
||||
limit=query.limit,
|
||||
tags=[tag for tag in query.tag if tag],
|
||||
)
|
||||
return dump_response(SkillListResponse, result)
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillCreatePayload.__name__])
|
||||
@console_ns.response(201, "Skill created", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account):
|
||||
try:
|
||||
payload = SkillCreatePayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().create_skill(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillDetailResponse, result), 201
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/files/upload")
|
||||
class WorkspaceSkillFileUploadApi(Resource):
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
|
||||
@console_ns.response(201, "Skill draft file uploaded", console_ns.models[SkillFileUploadResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account):
|
||||
if "file" not in request.files:
|
||||
return {"code": "no_file_uploaded", "message": "no file uploaded"}, 400
|
||||
|
||||
file = request.files["file"]
|
||||
if not file.filename:
|
||||
return {"code": "filename_missing", "message": "filename is required"}, 400
|
||||
|
||||
result = SkillManagementService().upload_file(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mime_type=file.mimetype,
|
||||
)
|
||||
return dump_response(SkillFileUploadResponse, result), 201
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/tags")
|
||||
class WorkspaceSkillTagsApi(Resource):
|
||||
@console_ns.response(200, "Workspace Skill tags", console_ns.models[SkillTagListResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str):
|
||||
result = SkillManagementService().list_tags(tenant_id=current_tenant_id)
|
||||
return dump_response(SkillTagListResponse, result)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/import")
|
||||
class WorkspaceSkillImportApi(Resource):
|
||||
@console_ns.doc(description="Import a Skill zip package from multipart form field `file`.")
|
||||
@console_ns.response(201, "Skill imported", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account):
|
||||
upload = request.files.get("file")
|
||||
if upload is None:
|
||||
return {"code": "invalid_request", "message": "file is required"}, 400
|
||||
try:
|
||||
payload = SkillImportPayload(content=upload.read(), filename=upload.filename or "skill.zip")
|
||||
result = SkillManagementService().import_skill(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillDetailResponse, result), 201
|
||||
except (ValidationError, ValueError) as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>")
|
||||
class WorkspaceSkillApi(Resource):
|
||||
@console_ns.response(200, "Skill detail", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
result = SkillManagementService().get_skill(tenant_id=current_tenant_id, skill_id=skill_id)
|
||||
return dump_response(SkillDetailResponse, result)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillMetadataPayload.__name__])
|
||||
@console_ns.response(200, "Skill updated", console_ns.models[SkillResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def patch(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillMetadataPayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().update_metadata(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillDeletePayload.__name__])
|
||||
@console_ns.response(200, "Skill deleted", console_ns.models[SkillDeleteResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_tenant_id
|
||||
def delete(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
payload = SkillDeletePayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().delete_skill(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
confirmation_name=payload.confirmation_name,
|
||||
)
|
||||
return dump_response(SkillDeleteResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/duplicate")
|
||||
class WorkspaceSkillDuplicateApi(Resource):
|
||||
@console_ns.response(201, "Skill duplicated", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
result = SkillManagementService().duplicate_skill(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
)
|
||||
return dump_response(SkillDetailResponse, result), 201
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/export")
|
||||
class WorkspaceSkillExportApi(Resource):
|
||||
@console_ns.response(200, "Published Skill zip archive")
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
result = SkillManagementService().pull_published_archive(tenant_id=current_tenant_id, skill_id=skill_id)
|
||||
return send_file(
|
||||
io.BytesIO(result.payload),
|
||||
mimetype=result.mime_type,
|
||||
as_attachment=True,
|
||||
download_name=result.filename,
|
||||
)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/assist/messages")
|
||||
class WorkspaceSkillAssistMessageApi(Resource):
|
||||
"""Stream read-only Skill Authoring suggestions from the default workspace model."""
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillAssistMessagePayload.__name__])
|
||||
@console_ns.response(200, "Skill Authoring assistant event stream")
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillAssistMessagePayload.model_validate(console_ns.payload or {})
|
||||
response = SkillManagementService().create_assistant_action_stream(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
user_id=current_user.id,
|
||||
message=payload.message,
|
||||
attachments=payload.attachments,
|
||||
model_payload=payload.model,
|
||||
target_path=payload.target_path,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
return helper.compact_generate_response(response)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/files")
|
||||
class WorkspaceSkillFilesApi(Resource):
|
||||
@console_ns.expect(console_ns.models[SkillDraftFileOperationPayload.__name__])
|
||||
@console_ns.response(200, "Draft file operation applied", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def patch(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillDraftFileOperationPayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().apply_draft_file_operation(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillDetailResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillDraftTreePayload.__name__])
|
||||
@console_ns.response(200, "Draft files replaced", console_ns.models[SkillDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def put(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillDraftTreePayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().replace_draft_tree(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillDetailResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/files/preview")
|
||||
class WorkspaceSkillFilePreviewApi(Resource):
|
||||
@console_ns.doc(params=query_params_from_model(SkillFileQuery))
|
||||
@console_ns.response(200, "Skill file text preview", console_ns.models[SkillFilePreviewResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
query = SkillFileQuery.model_validate(
|
||||
{
|
||||
"path": request.args.get("path"),
|
||||
"version_id": request.args.get("version_id"),
|
||||
}
|
||||
)
|
||||
result = SkillManagementService().preview_file(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
path=query.path,
|
||||
version_id=query.version_id,
|
||||
)
|
||||
return dump_response(SkillFilePreviewResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/files/content")
|
||||
class WorkspaceSkillFileContentApi(Resource):
|
||||
@console_ns.doc(params={**query_params_from_model(SkillFileQuery), "download": "Return as an attachment when 1."})
|
||||
@console_ns.response(200, "Skill file content", console_ns.models[BinaryFileResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
query = SkillFileQuery.model_validate(
|
||||
{
|
||||
"path": request.args.get("path"),
|
||||
"version_id": request.args.get("version_id"),
|
||||
}
|
||||
)
|
||||
result = SkillManagementService().pull_file(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
path=query.path,
|
||||
version_id=query.version_id,
|
||||
)
|
||||
return send_file(
|
||||
io.BytesIO(result.payload),
|
||||
mimetype=result.mime_type,
|
||||
as_attachment=request.args.get("download") == "1",
|
||||
download_name=result.filename,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except ValueError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/publish")
|
||||
class WorkspaceSkillPublishApi(Resource):
|
||||
@console_ns.expect(console_ns.models[SkillPublishPayload.__name__])
|
||||
@console_ns.response(200, "Skill published", console_ns.models[SkillVersionResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillPublishPayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().publish_skill(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillVersionResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/restore")
|
||||
class WorkspaceSkillRestoreApi(Resource):
|
||||
@console_ns.expect(console_ns.models[SkillRestorePayload.__name__])
|
||||
@console_ns.response(200, "Skill version restored", console_ns.models[SkillVersionResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, current_user: Account, skill_id: str):
|
||||
try:
|
||||
payload = SkillRestorePayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().restore_version(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillVersionResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/references")
|
||||
class WorkspaceSkillReferencesApi(Resource):
|
||||
@console_ns.response(200, "Skill references", console_ns.models[SkillReferenceListResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
result = SkillManagementService().list_skill_references(tenant_id=current_tenant_id, skill_id=skill_id)
|
||||
return dump_response(SkillReferenceListResponse, result)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/versions")
|
||||
class WorkspaceSkillVersionsApi(Resource):
|
||||
@console_ns.response(200, "Skill versions", console_ns.models[SkillVersionListResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str):
|
||||
try:
|
||||
result = SkillManagementService().list_versions(tenant_id=current_tenant_id, skill_id=skill_id)
|
||||
return dump_response(SkillVersionListResponse, result)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/skills/<string:skill_id>/versions/<string:version_id>")
|
||||
class WorkspaceSkillVersionApi(Resource):
|
||||
@console_ns.response(200, "Skill version detail", console_ns.models[SkillVersionDetailResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, skill_id: str, version_id: str):
|
||||
try:
|
||||
result = SkillManagementService().get_version(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
version_id=version_id,
|
||||
)
|
||||
return dump_response(SkillVersionDetailResponse, result)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
@console_ns.expect(console_ns.models[SkillVersionUpdatePayload.__name__])
|
||||
@console_ns.response(200, "Skill version updated", console_ns.models[SkillVersionResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_tenant_id
|
||||
def patch(self, current_tenant_id: str, skill_id: str, version_id: str):
|
||||
try:
|
||||
payload = SkillVersionUpdatePayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().update_version(
|
||||
tenant_id=current_tenant_id,
|
||||
skill_id=skill_id,
|
||||
version_id=version_id,
|
||||
payload=payload,
|
||||
)
|
||||
return dump_response(SkillVersionResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
@console_ns.response(200, "Skill version deleted", console_ns.models[SkillVersionDeleteResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def delete(self, current_tenant_id: str, current_user: Account, skill_id: str, version_id: str):
|
||||
try:
|
||||
result = SkillManagementService().delete_version(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
skill_id=skill_id,
|
||||
version_id=version_id,
|
||||
)
|
||||
return dump_response(SkillVersionDeleteResponse, result)
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
@console_ns.route("/workspaces/current/agents/<string:agent_id>/skills")
|
||||
class WorkspaceAgentSkillBindingsApi(Resource):
|
||||
@console_ns.response(200, "Agent Skill bindings", console_ns.models[AgentSkillBindingsResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, agent_id: str):
|
||||
result = SkillManagementService().list_agent_bindings(tenant_id=current_tenant_id, agent_id=agent_id)
|
||||
return dump_response(AgentSkillBindingsResponse, result)
|
||||
|
||||
@console_ns.expect(console_ns.models[AgentSkillBindingsPayload.__name__])
|
||||
@console_ns.response(200, "Agent Skill bindings replaced", console_ns.models[AgentSkillBindingsResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def put(self, current_tenant_id: str, current_user: Account, agent_id: str):
|
||||
try:
|
||||
payload = AgentSkillBindingsPayload.model_validate(console_ns.payload or {})
|
||||
result = SkillManagementService().replace_agent_bindings(
|
||||
tenant_id=current_tenant_id,
|
||||
user_id=current_user.id,
|
||||
agent_id=agent_id,
|
||||
skill_ids=payload.skill_ids,
|
||||
)
|
||||
return dump_response(AgentSkillBindingsResponse, result)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"WorkspaceAgentSkillBindingsApi",
|
||||
"WorkspaceSkillApi",
|
||||
"WorkspaceSkillDuplicateApi",
|
||||
"WorkspaceSkillExportApi",
|
||||
"WorkspaceSkillFilesApi",
|
||||
"WorkspaceSkillImportApi",
|
||||
"WorkspaceSkillPublishApi",
|
||||
"WorkspaceSkillReferencesApi",
|
||||
"WorkspaceSkillRestoreApi",
|
||||
"WorkspaceSkillTagsApi",
|
||||
"WorkspaceSkillVersionApi",
|
||||
"WorkspaceSkillVersionsApi",
|
||||
"WorkspaceSkillsApi",
|
||||
]
|
||||
@@ -37,6 +37,7 @@ from controllers.console.wraps import (
|
||||
with_current_user,
|
||||
)
|
||||
from enums.cloud_plan import CloudPlan
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from extensions.ext_database import db
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import dump_response, to_timestamp
|
||||
@@ -233,7 +234,7 @@ class TenantListApi(Resource):
|
||||
tenants = [tenant for tenant, _ in tenant_rows]
|
||||
tenant_dicts = []
|
||||
is_enterprise_only = dify_config.ENTERPRISE_ENABLED and not dify_config.BILLING_ENABLED
|
||||
is_saas = dify_config.EDITION == "CLOUD" and dify_config.BILLING_ENABLED
|
||||
is_saas = dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD and dify_config.BILLING_ENABLED
|
||||
tenant_plans: dict[str, SubscriptionPlan] = {}
|
||||
|
||||
if is_saas:
|
||||
|
||||
@@ -20,6 +20,7 @@ from controllers.common.wraps import (
|
||||
from controllers.console.auth.error import AuthenticationFailedError, EmailCodeError
|
||||
from controllers.console.workspace.error import AccountNotInitializedError
|
||||
from enums.cloud_plan import CloudPlan
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from libs.encryption import FieldEncryption
|
||||
@@ -129,7 +130,7 @@ def account_initialization_required[R](view: Callable[..., R]) -> Callable[...,
|
||||
def only_edition_cloud[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs):
|
||||
if dify_config.EDITION != "CLOUD":
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
|
||||
abort(404)
|
||||
|
||||
return view(*args, **kwargs)
|
||||
@@ -151,7 +152,7 @@ def only_edition_enterprise[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
def only_edition_self_hosted[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs):
|
||||
if dify_config.EDITION != "SELF_HOSTED":
|
||||
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
|
||||
abort(404)
|
||||
|
||||
return view(*args, **kwargs)
|
||||
@@ -327,7 +328,7 @@ def setup_required[R](view: Callable[..., R]) -> Callable[..., R]:
|
||||
# The overloads keep Resource methods method-aware for pyrefly while
|
||||
# preserving support for plain functions used in tests and utilities.
|
||||
# check setup
|
||||
if dify_config.EDITION == "SELF_HOSTED" and not _is_setup_completed():
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD and not _is_setup_completed():
|
||||
if os.environ.get("INIT_PASSWORD"):
|
||||
raise NotInitValidateError()
|
||||
raise NotSetupError()
|
||||
|
||||
@@ -20,10 +20,10 @@ from . import runtime_credentials as _runtime_credentials
|
||||
from .agent import tools as _agent_tools
|
||||
from .app import dsl as _app_dsl
|
||||
from .knowledge import retrieval as _knowledge_retrieval
|
||||
from .knowledge_fs import storage as _knowledge_fs_storage
|
||||
from .plugin import agent_config as _agent_config
|
||||
from .plugin import agent_drive as _agent_drive
|
||||
from .plugin import plugin as _plugin
|
||||
from .plugin import skills as _skills
|
||||
from .workspace import workspace as _workspace
|
||||
|
||||
api.add_namespace(inner_api_ns)
|
||||
@@ -33,11 +33,11 @@ __all__ = [
|
||||
"_agent_drive",
|
||||
"_agent_tools",
|
||||
"_app_dsl",
|
||||
"_knowledge_fs_storage",
|
||||
"_knowledge_retrieval",
|
||||
"_mail",
|
||||
"_plugin",
|
||||
"_runtime_credentials",
|
||||
"_skills",
|
||||
"_workspace",
|
||||
"api",
|
||||
"bp",
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""Trusted KnowledgeFS inner API endpoints."""
|
||||
@@ -1,303 +0,0 @@
|
||||
"""Trusted KnowledgeFS gateway to Dify's configured object-storage backend."""
|
||||
|
||||
import json
|
||||
from base64 import b64decode
|
||||
from binascii import Error as BinasciiError
|
||||
from http import HTTPStatus
|
||||
from typing import NoReturn
|
||||
|
||||
from flask import Response, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
|
||||
from pydantic.alias_generators import to_camel
|
||||
|
||||
from controllers.common.schema import query_params_from_model, register_response_schema_models
|
||||
from controllers.inner_api import inner_api_ns
|
||||
from controllers.inner_api.wraps import knowledge_fs_inner_api_only
|
||||
from fields.base import ResponseModel
|
||||
from libs.exception import BaseHTTPException
|
||||
from libs.helper import dump_response
|
||||
from services.knowledge_fs.object_storage import (
|
||||
KNOWLEDGE_FS_OBJECT_MAX_BYTES,
|
||||
KnowledgeFSObjectList,
|
||||
KnowledgeFSObjectMetadata,
|
||||
KnowledgeFSObjectStorageChecksumError,
|
||||
KnowledgeFSObjectStorageCorruptError,
|
||||
KnowledgeFSObjectStorageError,
|
||||
KnowledgeFSObjectStorageInvalidInputError,
|
||||
KnowledgeFSObjectStorageService,
|
||||
KnowledgeFSObjectStorageTooLargeError,
|
||||
KnowledgeFSObjectStorageUnavailableError,
|
||||
)
|
||||
|
||||
_METADATA_HEADER = "X-Knowledge-FS-Metadata"
|
||||
_CHECKSUM_HEADER = "X-Knowledge-FS-Checksum-Sha256"
|
||||
_CONTENT_TYPE_HEADER = "X-Knowledge-FS-Content-Type"
|
||||
_MAX_ENCODED_METADATA_BYTES = 128 * 1024
|
||||
_metadata_adapter = TypeAdapter(dict[str, str])
|
||||
|
||||
|
||||
class KnowledgeFSObjectStorageHttpError(BaseHTTPException):
|
||||
"""Safe HTTP representation of a KnowledgeFS storage boundary error."""
|
||||
|
||||
error_code = "knowledge_fs_object_storage_failed"
|
||||
description = "KnowledgeFS object storage request failed."
|
||||
code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
|
||||
def __init__(self, *, error_code: str, description: str, status_code: HTTPStatus) -> None:
|
||||
self.error_code = error_code
|
||||
self.description = description
|
||||
self.code = status_code
|
||||
super().__init__(description)
|
||||
|
||||
|
||||
class _CamelCaseResponse(ResponseModel):
|
||||
model_config = ConfigDict(alias_generator=to_camel)
|
||||
|
||||
|
||||
class KnowledgeFSObjectQuery(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
key: str = Field(description="Logical KnowledgeFS object key")
|
||||
|
||||
|
||||
class KnowledgeFSObjectListQuery(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
prefix: str = Field(default="", description="Logical object-key prefix")
|
||||
cursor: str | None = Field(default=None, description="Exclusive lexical key cursor")
|
||||
limit: int = Field(default=100, ge=1, le=100, description="Maximum objects to return")
|
||||
|
||||
|
||||
class KnowledgeFSObjectMetadataResponse(_CamelCaseResponse):
|
||||
checksum_sha256_base64: str
|
||||
content_type: str | None = None
|
||||
key: str
|
||||
metadata: dict[str, str]
|
||||
size_bytes: int
|
||||
|
||||
|
||||
class KnowledgeFSObjectListResponse(_CamelCaseResponse):
|
||||
objects: list[KnowledgeFSObjectMetadataResponse]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class KnowledgeFSObjectHealthResponse(ResponseModel):
|
||||
ok: bool
|
||||
|
||||
|
||||
register_response_schema_models(
|
||||
inner_api_ns,
|
||||
KnowledgeFSObjectMetadataResponse,
|
||||
KnowledgeFSObjectListResponse,
|
||||
KnowledgeFSObjectHealthResponse,
|
||||
)
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/storage/object")
|
||||
class KnowledgeFSObjectApi(Resource):
|
||||
"""Read, write, or delete one logical KnowledgeFS object."""
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
|
||||
@inner_api_ns.response(
|
||||
HTTPStatus.OK,
|
||||
"Object stored",
|
||||
inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__],
|
||||
)
|
||||
def put(self) -> dict[str, object]:
|
||||
try:
|
||||
query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
|
||||
metadata = _decode_metadata_header(request.headers.get(_METADATA_HEADER))
|
||||
body = request.stream.read(KNOWLEDGE_FS_OBJECT_MAX_BYTES + 1)
|
||||
if len(body) > KNOWLEDGE_FS_OBJECT_MAX_BYTES:
|
||||
raise KnowledgeFSObjectStorageTooLargeError(f"object exceeds max bytes {KNOWLEDGE_FS_OBJECT_MAX_BYTES}")
|
||||
result = KnowledgeFSObjectStorageService().put_object(
|
||||
body=body,
|
||||
checksum_sha256_base64=request.headers.get(_CHECKSUM_HEADER),
|
||||
content_type=request.headers.get(_CONTENT_TYPE_HEADER),
|
||||
key=query.key,
|
||||
metadata=metadata,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSObjectStorageError as exc:
|
||||
_raise_http_error(exc)
|
||||
return _metadata_response(result)
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
|
||||
@inner_api_ns.produces(["application/octet-stream"])
|
||||
def get(self) -> Response:
|
||||
try:
|
||||
query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
|
||||
service = KnowledgeFSObjectStorageService()
|
||||
metadata = service.head_object(key=query.key)
|
||||
if metadata is None:
|
||||
raise _not_found_error()
|
||||
body = service.load_stream(key=query.key)
|
||||
if body is None:
|
||||
raise _not_found_error()
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSObjectStorageError as exc:
|
||||
_raise_http_error(exc)
|
||||
|
||||
response = Response(
|
||||
body,
|
||||
content_type=metadata.content_type or "application/octet-stream",
|
||||
)
|
||||
response.content_length = metadata.size_bytes
|
||||
response.headers[_CHECKSUM_HEADER] = metadata.checksum_sha256_base64
|
||||
return response
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
|
||||
@inner_api_ns.response(HTTPStatus.NO_CONTENT, "Object deleted")
|
||||
def delete(self) -> tuple[str, int]:
|
||||
try:
|
||||
query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
|
||||
KnowledgeFSObjectStorageService().delete_object(key=query.key)
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSObjectStorageError as exc:
|
||||
_raise_http_error(exc)
|
||||
return "", HTTPStatus.NO_CONTENT
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/storage/object/metadata")
|
||||
class KnowledgeFSObjectMetadataApi(Resource):
|
||||
"""Read portable metadata for one logical KnowledgeFS object."""
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
|
||||
@inner_api_ns.response(
|
||||
HTTPStatus.OK,
|
||||
"Object metadata",
|
||||
inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__],
|
||||
)
|
||||
def get(self) -> dict[str, object]:
|
||||
try:
|
||||
query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
|
||||
result = KnowledgeFSObjectStorageService().head_object(key=query.key)
|
||||
if result is None:
|
||||
raise _not_found_error()
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSObjectStorageError as exc:
|
||||
_raise_http_error(exc)
|
||||
return _metadata_response(result)
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/storage/objects")
|
||||
class KnowledgeFSObjectListApi(Resource):
|
||||
"""List logical KnowledgeFS objects with bounded keyset pagination."""
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectListQuery))
|
||||
@inner_api_ns.response(
|
||||
HTTPStatus.OK,
|
||||
"Object page",
|
||||
inner_api_ns.models[KnowledgeFSObjectListResponse.__name__],
|
||||
)
|
||||
def get(self) -> dict[str, object]:
|
||||
try:
|
||||
query = KnowledgeFSObjectListQuery.model_validate(request.args.to_dict(flat=True))
|
||||
result = KnowledgeFSObjectStorageService().list_objects(
|
||||
cursor=query.cursor,
|
||||
limit=query.limit,
|
||||
prefix=query.prefix,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise _invalid_request_error() from exc
|
||||
except KnowledgeFSObjectStorageError as exc:
|
||||
_raise_http_error(exc)
|
||||
return _list_response(result)
|
||||
|
||||
|
||||
@inner_api_ns.route("/knowledge-fs/storage/health")
|
||||
class KnowledgeFSObjectHealthApi(Resource):
|
||||
"""Report whether Dify storage satisfies KnowledgeFS portable requirements."""
|
||||
|
||||
@knowledge_fs_inner_api_only
|
||||
@inner_api_ns.response(
|
||||
HTTPStatus.OK,
|
||||
"Storage available",
|
||||
inner_api_ns.models[KnowledgeFSObjectHealthResponse.__name__],
|
||||
)
|
||||
@inner_api_ns.response(HTTPStatus.SERVICE_UNAVAILABLE, "Storage unavailable")
|
||||
def get(self) -> dict[str, bool] | tuple[dict[str, bool], int]:
|
||||
if KnowledgeFSObjectStorageService().health():
|
||||
return {"ok": True}
|
||||
return {"ok": False}, HTTPStatus.SERVICE_UNAVAILABLE
|
||||
|
||||
|
||||
def _decode_metadata_header(value: str | None) -> dict[str, str]:
|
||||
if value is None:
|
||||
return {}
|
||||
if len(value.encode()) > _MAX_ENCODED_METADATA_BYTES:
|
||||
raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is too large")
|
||||
try:
|
||||
padding = "=" * (-len(value) % 4)
|
||||
decoded = b64decode(value + padding, altchars=b"-_", validate=True)
|
||||
return _metadata_adapter.validate_json(decoded)
|
||||
except (BinasciiError, UnicodeEncodeError, ValidationError, json.JSONDecodeError) as exc:
|
||||
raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is invalid") from exc
|
||||
|
||||
|
||||
def _metadata_response(metadata: KnowledgeFSObjectMetadata) -> dict[str, object]:
|
||||
return dump_response(KnowledgeFSObjectMetadataResponse, metadata)
|
||||
|
||||
|
||||
def _list_response(result: KnowledgeFSObjectList) -> dict[str, object]:
|
||||
return dump_response(KnowledgeFSObjectListResponse, result)
|
||||
|
||||
|
||||
def _raise_http_error(error: KnowledgeFSObjectStorageError) -> NoReturn:
|
||||
if isinstance(error, KnowledgeFSObjectStorageTooLargeError):
|
||||
raise KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_too_large",
|
||||
description="KnowledgeFS object exceeds the configured size limit.",
|
||||
status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
|
||||
) from error
|
||||
if isinstance(error, KnowledgeFSObjectStorageChecksumError):
|
||||
raise KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_checksum_mismatch",
|
||||
description="KnowledgeFS object checksum does not match the request body.",
|
||||
status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
|
||||
) from error
|
||||
if isinstance(error, KnowledgeFSObjectStorageInvalidInputError):
|
||||
raise _invalid_request_error() from error
|
||||
if isinstance(error, KnowledgeFSObjectStorageCorruptError):
|
||||
raise KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_corrupt",
|
||||
description="KnowledgeFS object metadata is inconsistent.",
|
||||
status_code=HTTPStatus.BAD_GATEWAY,
|
||||
) from error
|
||||
if isinstance(error, KnowledgeFSObjectStorageUnavailableError):
|
||||
raise KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_storage_unavailable",
|
||||
description="Dify object storage is unavailable for KnowledgeFS.",
|
||||
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
||||
) from error
|
||||
raise KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_storage_failed",
|
||||
description="KnowledgeFS object storage request failed.",
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
) from error
|
||||
|
||||
|
||||
def _invalid_request_error() -> KnowledgeFSObjectStorageHttpError:
|
||||
return KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_storage_invalid_request",
|
||||
description="KnowledgeFS object storage request is invalid.",
|
||||
status_code=HTTPStatus.BAD_REQUEST,
|
||||
)
|
||||
|
||||
|
||||
def _not_found_error() -> KnowledgeFSObjectStorageHttpError:
|
||||
return KnowledgeFSObjectStorageHttpError(
|
||||
error_code="knowledge_fs_object_not_found",
|
||||
description="KnowledgeFS object was not found.",
|
||||
status_code=HTTPStatus.NOT_FOUND,
|
||||
)
|
||||
@@ -8,7 +8,6 @@ from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
|
||||
from controllers.inner_api.wraps import plugin_inner_api_only
|
||||
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
|
||||
from core.plugin.backwards_invocation.datasource import PluginDatasourceBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.encrypt import PluginEncrypter
|
||||
from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
|
||||
@@ -16,12 +15,10 @@ from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
RequestFetchAppInfo,
|
||||
RequestInvokeApp,
|
||||
RequestInvokeDatasource,
|
||||
RequestInvokeEncrypt,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeParameterExtractorNode,
|
||||
RequestInvokeQuestionClassifierNode,
|
||||
RequestInvokeRerank,
|
||||
@@ -30,7 +27,6 @@ from core.plugin.entities.request import (
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTool,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
RequestRequestDownloadFile,
|
||||
RequestRequestUploadFile,
|
||||
)
|
||||
@@ -122,36 +118,6 @@ class PluginInvokeTextEmbeddingApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/multimodal-embedding")
|
||||
class PluginInvokeMultimodalEmbeddingApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeMultimodalEmbedding)
|
||||
@inner_api_ns.doc("plugin_invoke_multimodal_embedding")
|
||||
@inner_api_ns.doc(description="Invoke multimodal embedding models through Dify model management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Multimodal embedding successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeMultimodalEmbedding):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.invoke_multimodal_embedding(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/rerank")
|
||||
class PluginInvokeRerankApi(Resource):
|
||||
@get_user_tenant
|
||||
@@ -178,70 +144,6 @@ class PluginInvokeRerankApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/model-catalog")
|
||||
class PluginModelCatalogApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestListModels)
|
||||
@inner_api_ns.doc("plugin_model_catalog")
|
||||
@inner_api_ns.doc(description="List tenant-active models managed by Dify")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Model catalog lookup successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestListModels):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.list_models(
|
||||
tenant_id=tenant_model.id,
|
||||
user_id=user_model.id,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/datasource")
|
||||
class PluginInvokeDatasourceApi(Resource):
|
||||
"""Invoke an installed datasource with credentials resolved inside Dify."""
|
||||
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeDatasource)
|
||||
@inner_api_ns.doc("plugin_invoke_datasource")
|
||||
@inner_api_ns.doc(description="Invoke datasource plugins through Dify credential management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Datasource invocation successful (streaming response)",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Datasource provider, datasource, or credential not found",
|
||||
}
|
||||
)
|
||||
def post(
|
||||
self,
|
||||
user_model: Account | EndUser,
|
||||
tenant_model: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
):
|
||||
response = PluginDatasourceBackwardsInvocation.invoke(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
return length_prefixed_response(
|
||||
0xF,
|
||||
PluginDatasourceBackwardsInvocation.convert_to_event_stream(response),
|
||||
)
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/tts")
|
||||
class PluginInvokeTTSApi(Resource):
|
||||
@get_user_tenant
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Inner API for published workspace Skills.
|
||||
|
||||
These endpoints are called by trusted runtime services. They expose only
|
||||
published Skill artifacts, never draft files or editable metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
|
||||
from flask import request, send_file
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from controllers.console.wraps import setup_required
|
||||
from controllers.inner_api import inner_api_ns
|
||||
from controllers.inner_api.wraps import plugin_inner_api_only
|
||||
from services.skill_management_service import SkillManagementService, SkillManagementServiceError
|
||||
|
||||
|
||||
class _SkillTargetQuery(BaseModel):
|
||||
tenant_id: str
|
||||
|
||||
|
||||
def _target_query_from_request() -> _SkillTargetQuery:
|
||||
return _SkillTargetQuery.model_validate({"tenant_id": request.args.get("tenant_id")})
|
||||
|
||||
|
||||
def _error_response(exc: SkillManagementServiceError) -> tuple[dict[str, str], int]:
|
||||
return {"code": exc.code, "message": exc.message}, exc.status_code
|
||||
|
||||
|
||||
@inner_api_ns.route("/skills/<string:skill_id>/pull")
|
||||
class PublishedSkillPullApi(Resource):
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@inner_api_ns.doc("published_skill_pull")
|
||||
def get(self, skill_id: str):
|
||||
try:
|
||||
query = _target_query_from_request()
|
||||
result = SkillManagementService().pull_published_archive(tenant_id=query.tenant_id, skill_id=skill_id)
|
||||
return send_file(
|
||||
io.BytesIO(result.payload),
|
||||
mimetype=result.mime_type,
|
||||
as_attachment=True,
|
||||
download_name=result.filename,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
return {"code": "invalid_request", "message": str(exc)}, 400
|
||||
except SkillManagementServiceError as exc:
|
||||
return _error_response(exc)
|
||||
|
||||
|
||||
__all__ = ["PublishedSkillPullApi"]
|
||||
@@ -1,6 +1,5 @@
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from uuid import UUID
|
||||
|
||||
from flask import current_app, request
|
||||
from flask_login import user_logged_in
|
||||
@@ -56,15 +55,14 @@ def get_user(tenant_id: str, user_id: str | None) -> EndUser:
|
||||
# session_id, id is auto-generated) and a fresh EndUser
|
||||
# was created per call, breaking multi-turn chat
|
||||
# continuation (see #36736).
|
||||
if _is_uuid(user_id):
|
||||
user_model = session.scalar(
|
||||
select(EndUser)
|
||||
.where(
|
||||
EndUser.id == user_id,
|
||||
EndUser.tenant_id == tenant_id,
|
||||
)
|
||||
.limit(1)
|
||||
user_model = session.scalar(
|
||||
select(EndUser)
|
||||
.where(
|
||||
EndUser.id == user_id,
|
||||
EndUser.tenant_id == tenant_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
if user_model is None:
|
||||
user_model = session.scalar(
|
||||
select(EndUser)
|
||||
@@ -92,14 +90,6 @@ def get_user(tenant_id: str, user_id: str | None) -> EndUser:
|
||||
return user_model
|
||||
|
||||
|
||||
def _is_uuid(value: str) -> bool:
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_user_tenant[**P, R](view_func: Callable[P, R]) -> Callable[P, R]:
|
||||
@wraps(view_func)
|
||||
def decorated_view(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
|
||||
@@ -107,9 +107,3 @@ def agent_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
"""
|
||||
|
||||
return plugin_inner_api_only(view)
|
||||
|
||||
|
||||
def knowledge_fs_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
"""Authenticate the trusted KnowledgeFS process on the shared inner bridge."""
|
||||
|
||||
return plugin_inner_api_only(view)
|
||||
|
||||
@@ -8,6 +8,7 @@ from werkzeug.exceptions import InternalServerError
|
||||
|
||||
from configs import dify_config
|
||||
from core.rbac import RBACPermission, RBACResourceScope
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models.account import Account, Tenant, TenantAccountRole
|
||||
from models.model import App, EndUser
|
||||
@@ -26,7 +27,7 @@ class CallerKind(StrEnum):
|
||||
|
||||
|
||||
def current_edition() -> Edition:
|
||||
if dify_config.EDITION == "CLOUD":
|
||||
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
|
||||
return Edition.SAAS
|
||||
if dify_config.ENTERPRISE_ENABLED:
|
||||
return Edition.EE
|
||||
|
||||
@@ -38,7 +38,6 @@ from .dataset import (
|
||||
)
|
||||
from .dataset.rag_pipeline import rag_pipeline_workflow
|
||||
from .end_user import end_user
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .workspace import models
|
||||
|
||||
__all__ = [
|
||||
@@ -55,7 +54,6 @@ __all__ = [
|
||||
"hit_testing",
|
||||
"human_input_form",
|
||||
"index",
|
||||
"knowledge_fs_resources",
|
||||
"message",
|
||||
"metadata",
|
||||
"models",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
"""KnowledgeFS-specific Service API authenticated by resource credentials."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -1,49 +0,0 @@
|
||||
"""Stable KnowledgeFS Service API error contract."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSInvalidCredentialHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_credential"
|
||||
description = "Invalid KnowledgeFS service credential."
|
||||
code = 401
|
||||
|
||||
|
||||
class KnowledgeFSServiceOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSServiceUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSServiceInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSServiceRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSServiceQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSInvalidCredentialHTTPError",
|
||||
"KnowledgeFSServiceInvalidRequestHTTPError",
|
||||
"KnowledgeFSServiceOperationUnavailableHTTPError",
|
||||
"KnowledgeFSServiceQuotaExceededHTTPError",
|
||||
"KnowledgeFSServiceRateLimitHTTPError",
|
||||
"KnowledgeFSServiceUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -11,6 +11,7 @@ from werkzeug.exceptions import BadRequest, NotFound, Unauthorized
|
||||
|
||||
from constants import HEADER_NAME_APP_CODE
|
||||
from controllers.web.error import WebAppAuthAccessDeniedError, WebAppAuthRequiredError
|
||||
from core.logging.context import set_identity_context
|
||||
from extensions.ext_database import db
|
||||
from libs.passport import PassportService
|
||||
from libs.token import extract_webapp_passport
|
||||
@@ -28,6 +29,11 @@ def validate_jwt_token[**P, R](
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
app_model, end_user = decode_jwt_token()
|
||||
set_identity_context(
|
||||
tenant_id=end_user.tenant_id,
|
||||
user_id=end_user.id,
|
||||
user_type=end_user.type or "end_user",
|
||||
)
|
||||
return view(app_model, end_user, *args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
@@ -14,10 +14,7 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
|
||||
from core.app.apps.base_app_runner import AppRunner
|
||||
from core.app.entities.app_invoke_entities import (
|
||||
AgentChatAppGenerateEntity,
|
||||
DifyRunContext,
|
||||
InvokeFrom,
|
||||
ModelConfigWithCredentialsEntity,
|
||||
UserFrom,
|
||||
)
|
||||
from core.app.file_access import DatabaseFileAccessController
|
||||
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
|
||||
@@ -151,15 +148,6 @@ class BaseAgentRunner(AppRunner):
|
||||
user_id=self.user_id,
|
||||
invoke_from=self.application_generate_entity.invoke_from,
|
||||
)
|
||||
invoke_from = self.application_generate_entity.invoke_from
|
||||
if isinstance(invoke_from, InvokeFrom):
|
||||
tool_entity.runtime.dify_run_context = DifyRunContext(
|
||||
tenant_id=self.tenant_id,
|
||||
app_id=self.app_config.app_id,
|
||||
user_id=self.user_id,
|
||||
user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER),
|
||||
invoke_from=invoke_from,
|
||||
)
|
||||
assert tool_entity.entity.description
|
||||
message_tool = PromptMessageTool(
|
||||
name=tool.tool_name,
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Publication visibility rules for calling roster Agents from Workflows.
|
||||
|
||||
``Agent.active_config_is_published`` describes whether the editable shared
|
||||
draft still matches the active snapshot. It is false both before the first
|
||||
publish and after a published Agent receives new draft edits, so it must not be
|
||||
used as a runtime availability flag. App-backed Agents are callable from a
|
||||
Workflow only when the active snapshot has a revision created by a
|
||||
publish-visible operation. Direct roster Agents are publish-visible by
|
||||
construction and only need an active snapshot.
|
||||
"""
|
||||
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from models.agent import Agent, AgentConfigRevision, AgentConfigRevisionOperation, AgentScope, AgentSource
|
||||
|
||||
PUBLISH_VISIBLE_APP_BACKED_REVISION_OPERATIONS = frozenset(
|
||||
{
|
||||
AgentConfigRevisionOperation.PUBLISH_DRAFT,
|
||||
AgentConfigRevisionOperation.SAVE_CURRENT_VERSION,
|
||||
AgentConfigRevisionOperation.SAVE_NEW_VERSION,
|
||||
AgentConfigRevisionOperation.SAVE_NEW_AGENT,
|
||||
AgentConfigRevisionOperation.SAVE_TO_ROSTER,
|
||||
AgentConfigRevisionOperation.RESTORE_VERSION,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def workflow_callable_active_snapshot_filter() -> ColumnElement[bool]:
|
||||
"""Return the SQL predicate for an Agent with a Workflow-callable active snapshot.
|
||||
|
||||
The caller remains responsible for tenant, roster scope, lifecycle status,
|
||||
and model configuration filters. The correlated revision lookup makes the
|
||||
predicate safe to compose into roster pagination queries.
|
||||
"""
|
||||
|
||||
app_backed_agent = or_(
|
||||
Agent.source == AgentSource.AGENT_APP,
|
||||
and_(
|
||||
Agent.source == AgentSource.IMPORTED,
|
||||
Agent.scope == AgentScope.ROSTER,
|
||||
Agent.app_id.is_not(None),
|
||||
),
|
||||
)
|
||||
publish_visible_revision_exists = (
|
||||
select(AgentConfigRevision.id)
|
||||
.where(
|
||||
AgentConfigRevision.tenant_id == Agent.tenant_id,
|
||||
AgentConfigRevision.agent_id == Agent.id,
|
||||
AgentConfigRevision.current_snapshot_id == Agent.active_config_snapshot_id,
|
||||
AgentConfigRevision.operation.in_(PUBLISH_VISIBLE_APP_BACKED_REVISION_OPERATIONS),
|
||||
)
|
||||
.correlate(Agent)
|
||||
.exists()
|
||||
)
|
||||
return and_(
|
||||
Agent.active_config_snapshot_id.is_not(None),
|
||||
or_(
|
||||
~app_backed_agent,
|
||||
publish_visible_revision_exists,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def agent_has_workflow_callable_active_snapshot(*, session: Session, agent: Agent) -> bool:
|
||||
"""Return whether ``agent`` has an active snapshot visible to Workflow.
|
||||
|
||||
This object-level form is useful after ownership and lifecycle checks have
|
||||
already loaded an Agent. It intentionally ignores dirty draft state so a
|
||||
previously published snapshot keeps serving while later edits remain
|
||||
unpublished.
|
||||
"""
|
||||
|
||||
if not agent.active_config_snapshot_id:
|
||||
return False
|
||||
is_app_backed = agent.source == AgentSource.AGENT_APP or (
|
||||
agent.source == AgentSource.IMPORTED and agent.scope == AgentScope.ROSTER and agent.app_id is not None
|
||||
)
|
||||
if not is_app_backed:
|
||||
return True
|
||||
return bool(
|
||||
session.scalar(
|
||||
select(AgentConfigRevision.id)
|
||||
.where(
|
||||
AgentConfigRevision.tenant_id == agent.tenant_id,
|
||||
AgentConfigRevision.agent_id == agent.id,
|
||||
AgentConfigRevision.current_snapshot_id == agent.active_config_snapshot_id,
|
||||
AgentConfigRevision.operation.in_(PUBLISH_VISIBLE_APP_BACKED_REVISION_OPERATIONS),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
)
|
||||
@@ -683,42 +683,27 @@ class AgentAppGenerator(MessageBasedAppGenerator):
|
||||
if draft_type == AgentConfigDraftType.DEBUG_BUILD.value
|
||||
else AgentConfigDraftType.DRAFT
|
||||
)
|
||||
if effective_draft_type == AgentConfigDraftType.DRAFT:
|
||||
from services.agent.composer_service import AgentComposerService
|
||||
|
||||
return AgentComposerService.get_or_create_normal_agent_draft(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
agent=agent,
|
||||
created_by=agent.updated_by or agent.created_by,
|
||||
)
|
||||
if not account_id:
|
||||
raise AgentAppGeneratorError("Build draft requires an account user")
|
||||
stmt = select(AgentConfigDraft).where(
|
||||
AgentConfigDraft.tenant_id == tenant_id,
|
||||
AgentConfigDraft.agent_id == agent.id,
|
||||
AgentConfigDraft.draft_type == effective_draft_type,
|
||||
AgentConfigDraft.draft_type == AgentConfigDraftType.DEBUG_BUILD,
|
||||
AgentConfigDraft.account_id == account_id,
|
||||
)
|
||||
if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
|
||||
if not account_id:
|
||||
raise AgentAppGeneratorError("Build draft requires an account user")
|
||||
stmt = stmt.where(AgentConfigDraft.account_id == account_id)
|
||||
else:
|
||||
stmt = stmt.where(AgentConfigDraft.account_id.is_(None))
|
||||
draft = session.scalar(stmt.order_by(AgentConfigDraft.updated_at.desc()).limit(1))
|
||||
if draft is not None:
|
||||
return draft
|
||||
if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
|
||||
raise AgentAppGeneratorError("Agent build draft not found")
|
||||
_, snapshot, agent_soul = AgentAppGenerator._resolve_agent_by_id(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent.id,
|
||||
snapshot_id=agent.active_config_snapshot_id,
|
||||
session=session,
|
||||
)
|
||||
draft = AgentConfigDraft(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent.id,
|
||||
draft_type=AgentConfigDraftType.DRAFT,
|
||||
account_id=None,
|
||||
draft_owner_key="",
|
||||
base_snapshot_id=snapshot.id,
|
||||
config_snapshot=agent_soul,
|
||||
created_by=agent.created_by,
|
||||
updated_by=agent.updated_by,
|
||||
)
|
||||
session.add(draft)
|
||||
session.flush()
|
||||
return draft
|
||||
raise AgentAppGeneratorError("Agent build draft not found")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_agent_by_id(
|
||||
|
||||
@@ -43,6 +43,7 @@ from core.workflow.nodes.agent_v2.runtime_request_builder import (
|
||||
build_config_layer_config,
|
||||
build_knowledge_layer_config,
|
||||
build_shell_layer_config,
|
||||
load_runtime_agent_skill_configs,
|
||||
)
|
||||
from models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfig
|
||||
from models.provider_ids import ModelProviderID
|
||||
@@ -125,14 +126,22 @@ class AgentAppRuntimeRequestBuilder:
|
||||
"cli_tool_count": len(agent_soul.tools.cli_tools),
|
||||
}
|
||||
|
||||
runtime_config_skills = load_runtime_agent_skill_configs(
|
||||
tenant_id=context.dify_context.tenant_id,
|
||||
agent_id=context.agent_id,
|
||||
)
|
||||
config_layer_config, config_warnings = build_config_layer_config(
|
||||
agent_soul,
|
||||
agent_id=context.agent_id,
|
||||
config_version_id=context.agent_config_snapshot_id,
|
||||
config_version_kind=context.agent_config_version_kind,
|
||||
runtime_config_skills=runtime_config_skills,
|
||||
)
|
||||
append_runtime_warnings(metadata, config_warnings)
|
||||
soul_prompt_resolver = build_config_aware_soul_mention_resolver(agent_soul)
|
||||
soul_prompt_resolver = build_config_aware_soul_mention_resolver(
|
||||
agent_soul,
|
||||
runtime_config_skills=runtime_config_skills,
|
||||
)
|
||||
knowledge_config = build_knowledge_layer_config(agent_soul)
|
||||
|
||||
request = self._request_builder.build_for_agent_app(
|
||||
@@ -159,7 +168,6 @@ class AgentAppRuntimeRequestBuilder:
|
||||
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
agent_mode="agent_app",
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
# ENG-616: expand slash-menu mention tokens to canonical names so
|
||||
# no frontend-internal {{#…#}} marker ever reaches the model.
|
||||
|
||||
@@ -38,11 +38,6 @@ class DatasourcePluginProviderController(ABC):
|
||||
):
|
||||
raise ToolProviderCredentialValidationError("Invalid credentials")
|
||||
|
||||
def validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
"""Validate credential shape and value against this installed provider declaration."""
|
||||
self.validate_credentials_format(credentials)
|
||||
self._validate_credentials(user_id, credentials)
|
||||
|
||||
@property
|
||||
def provider_type(self) -> DatasourceProviderType:
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,7 @@ from pydantic import BaseModel
|
||||
from configs import dify_config
|
||||
from core.entities import DEFAULT_PLUGIN_ID
|
||||
from core.entities.provider_entities import ProviderQuotaType, QuotaUnit, RestrictModel
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
|
||||
|
||||
@@ -49,7 +50,7 @@ class HostingConfiguration:
|
||||
self.moderation_config = None
|
||||
|
||||
def init_app(self, app: Flask):
|
||||
if dify_config.EDITION != "CLOUD":
|
||||
if dify_config.DEPLOYMENT_EDITION != DeploymentEdition.CLOUD:
|
||||
return
|
||||
|
||||
self.provider_map[f"{DEFAULT_PLUGIN_ID}/azure_openai/azure_openai"] = self.init_azure_openai()
|
||||
|
||||
+46
-44
@@ -21,6 +21,7 @@ from core.model_manager import ModelInstance, ModelManager
|
||||
from core.rag.cleaner.clean_processor import CleanProcessor
|
||||
from core.rag.datasource.keyword.keyword_factory import Keyword
|
||||
from core.rag.docstore.dataset_docstore import DatasetDocumentStore
|
||||
from core.rag.embedding.token_counter import calculate_segment_token_counts
|
||||
from core.rag.extractor.entity.datasource_type import DatasourceType
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting, NotionInfo, WebsiteInfo
|
||||
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
|
||||
@@ -113,17 +114,25 @@ class IndexingRunner:
|
||||
current_user=current_user,
|
||||
session=session,
|
||||
)
|
||||
token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)
|
||||
total_tokens = sum(token_counts)
|
||||
# save segment
|
||||
self._load_segments(dataset, requeried_document, documents, session)
|
||||
self._load_segments(
|
||||
session=session,
|
||||
dataset=dataset,
|
||||
dataset_document=requeried_document,
|
||||
documents=documents,
|
||||
token_counts=token_counts,
|
||||
)
|
||||
session.commit()
|
||||
|
||||
# load
|
||||
self._load(
|
||||
index_processor=index_processor,
|
||||
session=session,
|
||||
dataset=dataset,
|
||||
dataset_document=requeried_document,
|
||||
documents=documents,
|
||||
session=session,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
except DocumentIsPausedError:
|
||||
raise DocumentIsPausedError(f"Document paused, document id: {document_id}")
|
||||
@@ -190,17 +199,25 @@ class IndexingRunner:
|
||||
current_user=current_user,
|
||||
session=session,
|
||||
)
|
||||
token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)
|
||||
total_tokens = sum(token_counts)
|
||||
# save segment
|
||||
self._load_segments(dataset, requeried_document, documents, session)
|
||||
self._load_segments(
|
||||
session=session,
|
||||
dataset=dataset,
|
||||
dataset_document=requeried_document,
|
||||
documents=documents,
|
||||
token_counts=token_counts,
|
||||
)
|
||||
session.commit()
|
||||
|
||||
# load
|
||||
self._load(
|
||||
index_processor=index_processor,
|
||||
session=session,
|
||||
dataset=dataset,
|
||||
dataset_document=requeried_document,
|
||||
documents=documents,
|
||||
session=session,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
except DocumentIsPausedError:
|
||||
raise DocumentIsPausedError(f"Document paused, document id: {document_id}")
|
||||
@@ -225,7 +242,7 @@ class IndexingRunner:
|
||||
if not dataset:
|
||||
raise ValueError("no dataset found")
|
||||
|
||||
# get exist document_segment list and delete
|
||||
# get existing document segments
|
||||
document_segments = session.scalars(
|
||||
select(DocumentSegment).where(
|
||||
DocumentSegment.dataset_id == dataset.id,
|
||||
@@ -264,15 +281,15 @@ class IndexingRunner:
|
||||
child_documents.append(child_document)
|
||||
document.children = child_documents
|
||||
documents.append(document)
|
||||
# Preserve the full document total even when only incomplete segments are re-indexed.
|
||||
total_tokens = sum(document_segment.tokens for document_segment in document_segments)
|
||||
# build index
|
||||
index_type = requeried_document.doc_form
|
||||
index_processor = IndexProcessorFactory(index_type).init_index_processor()
|
||||
self._load(
|
||||
index_processor=index_processor,
|
||||
session=session,
|
||||
dataset=dataset,
|
||||
dataset_document=requeried_document,
|
||||
documents=documents,
|
||||
session=session,
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
except DocumentIsPausedError:
|
||||
raise DocumentIsPausedError(f"Document paused, document id: {document_id}")
|
||||
@@ -601,28 +618,16 @@ class IndexingRunner:
|
||||
|
||||
def _load(
|
||||
self,
|
||||
index_processor: BaseIndexProcessor,
|
||||
session: Session,
|
||||
dataset: Dataset,
|
||||
dataset_document: DatasetDocument,
|
||||
documents: list[Document],
|
||||
session: Session,
|
||||
):
|
||||
"""
|
||||
insert index and update document/segment status to completed
|
||||
"""
|
||||
total_tokens: int,
|
||||
) -> None:
|
||||
"""Build indexes and mark the document complete using the token total computed before hash sharding."""
|
||||
|
||||
embedding_model_instance = None
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
embedding_model_instance = self._get_model_manager(dataset.tenant_id).get_model_instance(
|
||||
tenant_id=dataset.tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
|
||||
# chunk nodes by chunk size
|
||||
# Build indexes using the existing hash-based worker groups.
|
||||
indexing_start_at = time.perf_counter()
|
||||
tokens = 0
|
||||
create_keyword_thread = None
|
||||
if (
|
||||
dataset_document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
|
||||
@@ -659,12 +664,11 @@ class IndexingRunner:
|
||||
chunk_documents,
|
||||
dataset.id,
|
||||
dataset_document.id,
|
||||
embedding_model_instance,
|
||||
)
|
||||
)
|
||||
|
||||
for future in futures:
|
||||
tokens += future.result()
|
||||
future.result()
|
||||
if (
|
||||
dataset_document.doc_form != IndexStructureType.PARENT_CHILD_INDEX
|
||||
and dataset.indexing_technique == IndexTechniqueType.ECONOMY
|
||||
@@ -679,7 +683,7 @@ class IndexingRunner:
|
||||
document_id=dataset_document.id,
|
||||
after_indexing_status=IndexingStatus.COMPLETED,
|
||||
extra_update_params={
|
||||
DatasetDocument.tokens: tokens,
|
||||
DatasetDocument.tokens: total_tokens,
|
||||
DatasetDocument.completed_at: naive_utc_now(),
|
||||
DatasetDocument.indexing_latency: indexing_end_at - indexing_start_at,
|
||||
DatasetDocument.error: None,
|
||||
@@ -720,8 +724,7 @@ class IndexingRunner:
|
||||
chunk_documents: list[Document],
|
||||
dataset_id: str,
|
||||
dataset_document_id: str,
|
||||
embedding_model_instance: ModelInstance | None,
|
||||
):
|
||||
) -> None:
|
||||
with flask_app.app_context():
|
||||
with session_factory.create_session() as session:
|
||||
dataset = session.get(Dataset, dataset_id)
|
||||
@@ -735,11 +738,6 @@ class IndexingRunner:
|
||||
# check document is paused
|
||||
self._check_document_paused_status(dataset_document.id)
|
||||
|
||||
tokens = 0
|
||||
if embedding_model_instance:
|
||||
page_content_list = [document.page_content for document in chunk_documents]
|
||||
tokens += sum(embedding_model_instance.get_text_embedding_num_tokens(page_content_list))
|
||||
|
||||
multimodal_documents = []
|
||||
for document in chunk_documents:
|
||||
if document.attachments and dataset.is_multimodal:
|
||||
@@ -773,8 +771,6 @@ class IndexingRunner:
|
||||
|
||||
session.commit()
|
||||
|
||||
return tokens
|
||||
|
||||
@staticmethod
|
||||
def _check_document_paused_status(document_id: str):
|
||||
indexing_cache_key = f"document_{document_id}_is_paused"
|
||||
@@ -864,8 +860,14 @@ class IndexingRunner:
|
||||
return documents
|
||||
|
||||
def _load_segments(
|
||||
self, dataset: Dataset, dataset_document: DatasetDocument, documents: list[Document], session: Session
|
||||
):
|
||||
self,
|
||||
session: Session,
|
||||
dataset: Dataset,
|
||||
dataset_document: DatasetDocument,
|
||||
documents: list[Document],
|
||||
token_counts: list[int],
|
||||
) -> None:
|
||||
"""Persist transformed documents and their precomputed token counts before indexing starts."""
|
||||
# save node to document segment
|
||||
doc_store = DatasetDocumentStore(
|
||||
dataset=dataset, user_id=dataset_document.created_by, document_id=dataset_document.id
|
||||
@@ -873,9 +875,10 @@ class IndexingRunner:
|
||||
|
||||
# add document segments
|
||||
doc_store.add_documents(
|
||||
session=session,
|
||||
docs=documents,
|
||||
save_child=dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX,
|
||||
session=session,
|
||||
token_counts=token_counts,
|
||||
)
|
||||
|
||||
# update document status to indexing
|
||||
@@ -900,7 +903,6 @@ class IndexingRunner:
|
||||
DocumentSegment.indexing_at: naive_utc_now(),
|
||||
},
|
||||
)
|
||||
pass
|
||||
|
||||
|
||||
class DocumentIsPausedError(Exception):
|
||||
|
||||
@@ -6,9 +6,21 @@ using Python's contextvars for thread-safe and async-safe storage.
|
||||
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
class IdentityContext(NamedTuple):
|
||||
"""Immutable identity values captured for logging."""
|
||||
|
||||
tenant_id: str
|
||||
user_id: str
|
||||
user_type: str
|
||||
|
||||
|
||||
_request_id: ContextVar[str] = ContextVar("log_request_id", default="")
|
||||
_trace_id: ContextVar[str] = ContextVar("log_trace_id", default="")
|
||||
_EMPTY_IDENTITY_CONTEXT = IdentityContext(tenant_id="", user_id="", user_type="")
|
||||
_identity: ContextVar[IdentityContext] = ContextVar("log_identity", default=_EMPTY_IDENTITY_CONTEXT)
|
||||
|
||||
|
||||
def get_request_id() -> str:
|
||||
@@ -21,15 +33,35 @@ def get_trace_id() -> str:
|
||||
return _trace_id.get()
|
||||
|
||||
|
||||
def get_identity_context() -> IdentityContext:
|
||||
"""Get the immutable tenant, user, and user-type snapshot for logging."""
|
||||
return _identity.get()
|
||||
|
||||
|
||||
def set_identity_context(
|
||||
*, tenant_id: str | None = None, user_id: str | None = None, user_type: str | None = None
|
||||
) -> None:
|
||||
"""Set primitive identity values already resolved by an authentication boundary."""
|
||||
_identity.set(
|
||||
IdentityContext(
|
||||
tenant_id=tenant_id or "",
|
||||
user_id=user_id or "",
|
||||
user_type=user_type or "",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def init_request_context() -> None:
|
||||
"""Initialize request context. Call at start of each request."""
|
||||
"""Initialize request context and discard identity left by earlier work."""
|
||||
req_id = uuid.uuid4().hex[:10]
|
||||
trace_id = uuid.uuid5(uuid.NAMESPACE_DNS, req_id).hex
|
||||
_request_id.set(req_id)
|
||||
_trace_id.set(trace_id)
|
||||
_identity.set(_EMPTY_IDENTITY_CONTEXT)
|
||||
|
||||
|
||||
def clear_request_context() -> None:
|
||||
"""Clear request context. Call at end of request (optional)."""
|
||||
"""Clear request context at a request or task lifecycle boundary."""
|
||||
_request_id.set("")
|
||||
_trace_id.set("")
|
||||
_identity.set(_EMPTY_IDENTITY_CONTEXT)
|
||||
|
||||
@@ -4,10 +4,7 @@ import contextlib
|
||||
import logging
|
||||
from typing import override
|
||||
|
||||
import flask
|
||||
|
||||
from core.logging.context import get_request_id, get_trace_id
|
||||
from core.logging.structured_formatter import IdentityDict
|
||||
from core.logging.context import get_identity_context, get_request_id, get_trace_id
|
||||
|
||||
|
||||
class TraceContextFilter(logging.Filter):
|
||||
@@ -51,49 +48,16 @@ class TraceContextFilter(logging.Filter):
|
||||
|
||||
|
||||
class IdentityContextFilter(logging.Filter):
|
||||
"""
|
||||
Filter that adds user identity context to log records.
|
||||
Extracts tenant_id, user_id, and user_type from Flask-Login current_user.
|
||||
"""Add an identity snapshot without invoking authentication or database work.
|
||||
|
||||
Logging can run while other libraries hold internal locks, so this filter must
|
||||
only read primitive ContextVar values populated by authentication boundaries.
|
||||
"""
|
||||
|
||||
@override
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
identity = self._extract_identity()
|
||||
record.tenant_id = identity.get("tenant_id", "")
|
||||
record.user_id = identity.get("user_id", "")
|
||||
record.user_type = identity.get("user_type", "")
|
||||
identity = get_identity_context()
|
||||
record.tenant_id = identity.tenant_id
|
||||
record.user_id = identity.user_id
|
||||
record.user_type = identity.user_type
|
||||
return True
|
||||
|
||||
def _extract_identity(self) -> IdentityDict:
|
||||
"""Extract identity from current_user if in request context."""
|
||||
try:
|
||||
if not flask.has_request_context():
|
||||
return {}
|
||||
from flask_login import current_user
|
||||
|
||||
# Check if user is authenticated using the proxy
|
||||
if not current_user.is_authenticated:
|
||||
return {}
|
||||
|
||||
# Access the underlying user object
|
||||
user = current_user
|
||||
|
||||
from models import Account
|
||||
from models.model import EndUser
|
||||
|
||||
identity: IdentityDict = {}
|
||||
|
||||
match user:
|
||||
case Account():
|
||||
if user.current_tenant_id:
|
||||
identity["tenant_id"] = user.current_tenant_id
|
||||
identity["user_id"] = user.id
|
||||
identity["user_type"] = "account"
|
||||
case EndUser():
|
||||
identity["tenant_id"] = user.tenant_id
|
||||
identity["user_id"] = user.id
|
||||
identity["user_type"] = user.type or "end_user"
|
||||
|
||||
return identity
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
"""Datasource backward invocation through Dify's tenant-bound runtime.
|
||||
|
||||
This module is the internal service boundary used by trusted callers such as
|
||||
KnowledgeFS. It resolves installed provider declarations and Dify-owned
|
||||
credential references before reaching ``PluginDatasourceManager``; callers
|
||||
must never provide raw datasource credentials or a plugin-daemon API key.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.datasource.datasource_manager import DatasourceManager
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
OnlineDriveBrowseFilesRequest,
|
||||
OnlineDriveDownloadFileRequest,
|
||||
)
|
||||
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
|
||||
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
|
||||
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import RequestInvokeDatasource
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import DatasourceProviderID
|
||||
from services.datasource_provider_service import DatasourceProviderService
|
||||
|
||||
|
||||
class PluginDatasourceBackwardsInvocation(BaseBackwardsInvocation):
|
||||
"""Resolve and invoke a datasource without exposing credential material to the caller."""
|
||||
|
||||
@classmethod
|
||||
def invoke(
|
||||
cls,
|
||||
*,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
) -> Generator[BaseModel | dict[str, Any], None, None]:
|
||||
"""Yield datasource messages for one validated inner-runtime request."""
|
||||
provider_id = DatasourceProviderID(payload.provider)
|
||||
canonical_provider_id = str(provider_id)
|
||||
controller = DatasourceManager.get_datasource_plugin_provider(
|
||||
provider_id=canonical_provider_id,
|
||||
tenant_id=tenant.id,
|
||||
datasource_type=payload.datasource_type,
|
||||
)
|
||||
if controller.entity.provider_type != payload.datasource_type:
|
||||
raise ValueError("Datasource provider type mismatch")
|
||||
|
||||
# Resolving the datasource from the installed declaration prevents a caller
|
||||
# from dispatching an arbitrary datasource name under a valid plugin ID.
|
||||
runtime = controller.get_datasource(payload.datasource)
|
||||
credentials = DatasourceProviderService().get_datasource_credentials(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider_id.provider_name,
|
||||
plugin_id=provider_id.plugin_id,
|
||||
credential_id=payload.credential_id,
|
||||
)
|
||||
if controller.need_credentials and not credentials:
|
||||
raise ValueError("Datasource credential not found")
|
||||
|
||||
if payload.operation == "validate_credentials":
|
||||
controller.validate_credentials(user_id=user_id, credentials=credentials)
|
||||
yield {"result": True}
|
||||
return
|
||||
|
||||
runtime.runtime.credentials = credentials
|
||||
provider_type = runtime.datasource_provider_type()
|
||||
|
||||
match payload.operation:
|
||||
case "get_website_crawl":
|
||||
website = cast(WebsiteCrawlDatasourcePlugin, runtime)
|
||||
yield from website.get_website_crawl(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_pages":
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_pages(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_page_content":
|
||||
if payload.page is None:
|
||||
raise ValueError("Online-document page input is required")
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_page_content(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.page,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_browse_files":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_browse_files(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveBrowseFilesRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_download_file":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_download_file(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveDownloadFileRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case _:
|
||||
raise ValueError(f"Unsupported datasource operation: {payload.operation}")
|
||||
@@ -1,31 +1,22 @@
|
||||
import tempfile
|
||||
from binascii import hexlify, unhexlify
|
||||
from collections.abc import Generator, Mapping
|
||||
from enum import Enum
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.app.llm import deduct_llm_quota
|
||||
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
|
||||
from core.model_manager import ModelManager
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
InvokableModelCatalogItem,
|
||||
InvokableModelCatalogPage,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeRerank,
|
||||
RequestInvokeSpeech2Text,
|
||||
RequestInvokeSummary,
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
)
|
||||
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
|
||||
from core.plugin.plugin_service import PluginService
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
from core.tools.utils.model_invocation_utils import ModelInvocationUtils
|
||||
from graphon.model_runtime.entities.llm_entities import (
|
||||
@@ -42,20 +33,6 @@ from graphon.model_runtime.entities.message_entities import (
|
||||
)
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import ModelProviderID
|
||||
|
||||
|
||||
def _json_compatible(value: Any) -> Any:
|
||||
"""Convert model-runtime metadata into stable JSON-compatible values."""
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(mode="json")
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Mapping):
|
||||
return {str(_json_compatible(key)): _json_compatible(child) for key, child in value.items()}
|
||||
if isinstance(value, list | tuple | set):
|
||||
return [_json_compatible(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
@@ -206,30 +183,7 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
)
|
||||
|
||||
# invoke model
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts, input_type=payload.input_type)
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def invoke_multimodal_embedding(
|
||||
cls,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeMultimodalEmbedding,
|
||||
):
|
||||
"""Invoke multimodal embedding through the tenant-bound model instance."""
|
||||
model_instance = cls._get_bound_model_instance(
|
||||
tenant_id=tenant.id,
|
||||
user_id=user_id,
|
||||
provider=payload.provider,
|
||||
model_type=payload.model_type,
|
||||
model=payload.model,
|
||||
)
|
||||
|
||||
response = model_instance.invoke_multimodal_embedding(
|
||||
multimodel_documents=[document.model_dump(exclude_none=True) for document in payload.documents],
|
||||
input_type=payload.input_type,
|
||||
)
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts)
|
||||
|
||||
return response
|
||||
|
||||
@@ -256,67 +210,6 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def list_models(
|
||||
cls,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
payload: RequestListModels,
|
||||
) -> InvokableModelCatalogPage:
|
||||
"""List only models that are active for the tenant's Dify configuration."""
|
||||
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id, user_id=user_id)
|
||||
active_models = provider_manager.get_configurations(tenant_id).get_models(
|
||||
model_type=payload.model_type,
|
||||
only_active=True,
|
||||
)
|
||||
|
||||
installed_identities: dict[str, str] = {}
|
||||
for plugin in PluginService.list(tenant_id):
|
||||
existing = installed_identities.get(plugin.plugin_id)
|
||||
if existing is not None and existing != plugin.plugin_unique_identifier:
|
||||
raise ValueError(f"Ambiguous installed identity for model plugin {plugin.plugin_id}")
|
||||
installed_identities[plugin.plugin_id] = plugin.plugin_unique_identifier
|
||||
|
||||
requested_provider = str(ModelProviderID(payload.provider)) if payload.provider else None
|
||||
matched_models = [
|
||||
model
|
||||
for model in active_models
|
||||
if (requested_provider is None or model.provider.provider == requested_provider)
|
||||
and (payload.model is None or model.model == payload.model)
|
||||
]
|
||||
matched_models.sort(key=lambda model: (model.provider.provider, model.model))
|
||||
|
||||
page_models = matched_models[payload.offset : payload.offset + payload.limit]
|
||||
items: list[InvokableModelCatalogItem] = []
|
||||
for model in page_models:
|
||||
provider_id = ModelProviderID(model.provider.provider)
|
||||
unique_identifier = installed_identities.get(provider_id.plugin_id)
|
||||
if unique_identifier is None:
|
||||
raise ValueError(f"Installed identity not found for active model plugin {provider_id.plugin_id}")
|
||||
items.append(
|
||||
InvokableModelCatalogItem(
|
||||
plugin_id=provider_id.plugin_id,
|
||||
plugin_unique_identifier=unique_identifier,
|
||||
provider=provider_id.provider_name,
|
||||
model=model.model,
|
||||
model_type=model.model_type,
|
||||
capabilities={
|
||||
"deprecated": model.deprecated,
|
||||
"features": _json_compatible(model.features or []),
|
||||
"fetchFrom": _json_compatible(model.fetch_from),
|
||||
"modelProperties": _json_compatible(model.model_properties),
|
||||
"modelType": model.model_type.value,
|
||||
"status": _json_compatible(model.status),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
next_offset = payload.offset + len(page_models)
|
||||
return InvokableModelCatalogPage(
|
||||
items=items,
|
||||
next_offset=next_offset if next_offset < len(matched_models) else None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def invoke_tts(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTTS):
|
||||
"""
|
||||
|
||||
@@ -6,11 +6,6 @@ from typing import Any, Literal
|
||||
from flask import Response
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
DatasourceProviderType,
|
||||
GetOnlineDocumentPageContentRequest,
|
||||
)
|
||||
from core.entities.embedding_type import EmbeddingInputType
|
||||
from core.entities.provider_entities import BasicProviderConfig
|
||||
from core.plugin.utils.http_parser import deserialize_response
|
||||
from core.workflow.file_reference import is_canonical_file_reference
|
||||
@@ -59,61 +54,6 @@ class RequestInvokeTool(BaseModel):
|
||||
credential_id: str | None = None
|
||||
|
||||
|
||||
DatasourceInvocationOperation = Literal[
|
||||
"get_online_document_page_content",
|
||||
"get_online_document_pages",
|
||||
"get_website_crawl",
|
||||
"online_drive_browse_files",
|
||||
"online_drive_download_file",
|
||||
"validate_credentials",
|
||||
]
|
||||
|
||||
|
||||
class RequestInvokeDatasource(BaseModel):
|
||||
"""Invoke one installed datasource using a Dify-owned credential reference.
|
||||
|
||||
Raw credentials are intentionally not part of this contract. ``tenant_id`` and
|
||||
``user_id`` are consumed by the inner-API request context, while the remaining
|
||||
fields select an installed provider declaration and an operation-specific input.
|
||||
"""
|
||||
|
||||
tenant_id: str = Field(min_length=1, max_length=512)
|
||||
user_id: str = Field(min_length=1, max_length=512)
|
||||
provider: str = Field(min_length=1, max_length=768)
|
||||
datasource: str = Field(min_length=1, max_length=256)
|
||||
datasource_type: DatasourceProviderType
|
||||
credential_id: str = Field(min_length=1, max_length=512)
|
||||
operation: DatasourceInvocationOperation
|
||||
datasource_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
page: GetOnlineDocumentPageContentRequest | None = None
|
||||
request: dict[str, Any] | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_operation_payload(self) -> "RequestInvokeDatasource":
|
||||
expected_type = {
|
||||
"get_online_document_page_content": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_online_document_pages": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_website_crawl": DatasourceProviderType.WEBSITE_CRAWL,
|
||||
"online_drive_browse_files": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"online_drive_download_file": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"validate_credentials": self.datasource_type,
|
||||
}[self.operation]
|
||||
if self.datasource_type != expected_type:
|
||||
raise ValueError(f"{self.operation} requires datasource_type {expected_type.value}")
|
||||
|
||||
page_required = self.operation == "get_online_document_page_content"
|
||||
if page_required != (self.page is not None):
|
||||
raise ValueError("page is required only for get_online_document_page_content")
|
||||
|
||||
request_required = self.operation in {"online_drive_browse_files", "online_drive_download_file"}
|
||||
if request_required != (self.request is not None):
|
||||
raise ValueError("request is required only for online-drive operations")
|
||||
|
||||
return self
|
||||
|
||||
|
||||
class BaseRequestInvokeModel(BaseModel):
|
||||
provider: str
|
||||
model: str
|
||||
@@ -175,25 +115,6 @@ class RequestInvokeTextEmbedding(BaseRequestInvokeModel):
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
texts: list[str]
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class MultimodalEmbeddingDocument(BaseModel):
|
||||
"""A document accepted by a multimodal text-embedding model."""
|
||||
|
||||
content: str
|
||||
content_type: str
|
||||
file_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RequestInvokeMultimodalEmbedding(BaseRequestInvokeModel):
|
||||
"""Request to invoke a multimodal text-embedding model."""
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
documents: list[MultimodalEmbeddingDocument] = Field(min_length=1)
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
@@ -204,40 +125,8 @@ class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
model_type: ModelType = ModelType.RERANK
|
||||
query: str
|
||||
docs: list[str]
|
||||
score_threshold: float | None = None
|
||||
top_n: int | None = None
|
||||
|
||||
|
||||
class RequestListModels(BaseModel):
|
||||
"""Tenant-scoped query for models that Dify can invoke."""
|
||||
|
||||
model_type: Literal[ModelType.LLM, ModelType.TEXT_EMBEDDING, ModelType.RERANK]
|
||||
provider: str | None = None
|
||||
model: str | None = None
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=50, ge=1, le=100)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogItem(BaseModel):
|
||||
"""Installed identity and active Dify capability metadata for one model."""
|
||||
|
||||
plugin_id: str
|
||||
plugin_unique_identifier: str
|
||||
provider: str
|
||||
model: str
|
||||
model_type: ModelType
|
||||
capabilities: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogPage(BaseModel):
|
||||
"""Offset page returned by the internal model catalog endpoint."""
|
||||
|
||||
items: list[InvokableModelCatalogItem] = Field(default_factory=list)
|
||||
next_offset: int | None = None
|
||||
score_threshold: float
|
||||
top_n: int
|
||||
|
||||
|
||||
class RequestInvokeTTS(BaseRequestInvokeModel):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Generator
|
||||
from collections.abc import Callable, Generator, Mapping
|
||||
from typing import Any, cast
|
||||
from urllib.parse import unquote
|
||||
|
||||
@@ -23,6 +23,7 @@ from core.plugin.impl.exc import (
|
||||
PluginLLMPollingUnsupportedError,
|
||||
PluginNotFoundError,
|
||||
PluginPermissionDeniedError,
|
||||
PluginRuntimeError,
|
||||
PluginUniqueIdentifierError,
|
||||
)
|
||||
from core.trigger.errors import (
|
||||
@@ -375,6 +376,18 @@ class BasePluginClient:
|
||||
# type `PluginLLMPollingUnsupportedError`.
|
||||
case PluginLLMPollingUnsupportedError.__name__:
|
||||
raise PluginLLMPollingUnsupportedError(description=error_object.get("message"))
|
||||
case PluginRuntimeError.__name__:
|
||||
args = error_object.get("args")
|
||||
lambda_request_id = args.get("request_id") if isinstance(args, Mapping) else None
|
||||
if not isinstance(lambda_request_id, str):
|
||||
lambda_request_id = None
|
||||
runtime_message = error_object.get("message")
|
||||
if not isinstance(runtime_message, str):
|
||||
runtime_message = "Plugin runtime request failed"
|
||||
raise PluginRuntimeError(
|
||||
description=runtime_message,
|
||||
lambda_request_id=lambda_request_id,
|
||||
)
|
||||
case _:
|
||||
raise PluginInvokeError(description=message)
|
||||
case PluginDaemonInternalServerError.__name__:
|
||||
|
||||
@@ -49,6 +49,18 @@ class PluginDaemonBadRequestError(PluginDaemonClientSideError):
|
||||
description: str = "Bad Request"
|
||||
|
||||
|
||||
class PluginRuntimeError(PluginDaemonInternalError):
|
||||
"""A plugin runtime failed before it could return a valid plugin response."""
|
||||
|
||||
lambda_request_id: str | None
|
||||
|
||||
def __init__(self, description: str, lambda_request_id: str | None = None) -> None:
|
||||
self.lambda_request_id = lambda_request_id
|
||||
if lambda_request_id:
|
||||
description = description.replace(f"RequestId: {lambda_request_id} Error: ", "", 1)
|
||||
super().__init__(description)
|
||||
|
||||
|
||||
class PluginInvokeError(PluginDaemonClientSideError, ValueError):
|
||||
description: str = "Invoke Error"
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from core.entities.provider_entities import (
|
||||
from core.helper import encrypter
|
||||
from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
|
||||
from core.helper.position_helper import is_filtered
|
||||
from enums.deployment_edition import DeploymentEdition
|
||||
from extensions import ext_hosting_provider
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
@@ -743,7 +744,7 @@ class ProviderManager:
|
||||
|
||||
if preferred_provider_type_record:
|
||||
preferred_provider_type = preferred_provider_type_record.preferred_provider_type
|
||||
elif dify_config.EDITION == "CLOUD" and system_configuration.enabled:
|
||||
elif dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD and system_configuration.enabled:
|
||||
preferred_provider_type = ProviderType.SYSTEM
|
||||
elif custom_configuration.provider or custom_configuration.models:
|
||||
preferred_provider_type = ProviderType.CUSTOM
|
||||
@@ -1538,7 +1539,7 @@ class ProviderManager:
|
||||
quota_type_to_provider_records_dict[provider_record.quota_type] = provider_record # type: ignore[index]
|
||||
quota_configurations = []
|
||||
|
||||
if dify_config.EDITION == "CLOUD":
|
||||
if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
|
||||
from services.credit_pool_service import CreditPoolService
|
||||
|
||||
trail_pool = CreditPoolService.get_pool(
|
||||
|
||||
@@ -10,6 +10,7 @@ from core.rag.rerank.entity.weight import KeywordSetting, VectorSetting, Weights
|
||||
from core.rag.rerank.rerank_base import BaseRerankRunner
|
||||
from core.rag.rerank.rerank_factory import RerankRunnerFactory
|
||||
from core.rag.rerank.rerank_type import RerankMode
|
||||
from extensions.otel import trace_span
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from graphon.model_runtime.errors.invoke import InvokeAuthorizationError
|
||||
|
||||
@@ -52,6 +53,7 @@ class DataPostProcessor:
|
||||
)
|
||||
self.reorder_runner = self._get_reorder_runner(reorder_enabled)
|
||||
|
||||
@trace_span()
|
||||
def invoke(
|
||||
self,
|
||||
query: str,
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import concurrent.futures
|
||||
import functools
|
||||
import logging
|
||||
from collections.abc import Callable, Sequence
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Any, NotRequired, TypedDict
|
||||
|
||||
from flask import Flask, current_app
|
||||
from opentelemetry import context as otel_context
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session, load_only
|
||||
|
||||
@@ -26,7 +24,7 @@ from core.rag.rerank.rerank_type import RerankMode
|
||||
from core.rag.retrieval.retrieval_methods import RetrievalMethod
|
||||
from core.tools.signature import sign_upload_file_preview_url
|
||||
from extensions.ext_database import db
|
||||
from extensions.otel import trace_span
|
||||
from extensions.otel import propagate_context, trace_span
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.dataset import (
|
||||
ChildChunk,
|
||||
@@ -92,20 +90,6 @@ default_retrieval_model: DefaultRetrievalModelDict = {
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _propagate_otel_context[**P, R](func: Callable[P, R]) -> Callable[P, R]:
|
||||
captured_context = otel_context.get_current()
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
token = otel_context.attach(captured_context)
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
finally:
|
||||
otel_context.detach(token)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class RetrievalService:
|
||||
# Cache precompiled regular expressions to avoid repeated compilation
|
||||
@classmethod
|
||||
@@ -139,7 +123,7 @@ class RetrievalService:
|
||||
if query:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(retrieval_service._retrieve),
|
||||
propagate_context(retrieval_service._retrieve),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
retrieval_method=retrieval_method,
|
||||
dataset=dataset,
|
||||
@@ -159,7 +143,7 @@ class RetrievalService:
|
||||
for attachment_id in attachment_ids:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(retrieval_service._retrieve),
|
||||
propagate_context(retrieval_service._retrieve),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
retrieval_method=retrieval_method,
|
||||
dataset=dataset,
|
||||
@@ -820,7 +804,7 @@ class RetrievalService:
|
||||
if retrieval_method == RetrievalMethod.KEYWORD_SEARCH and query:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(self.keyword_search),
|
||||
propagate_context(self.keyword_search),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
dataset_id=dataset.id,
|
||||
query=query,
|
||||
@@ -834,7 +818,7 @@ class RetrievalService:
|
||||
if query:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(self.embedding_search),
|
||||
propagate_context(self.embedding_search),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
dataset_id=dataset.id,
|
||||
query=query,
|
||||
@@ -851,7 +835,7 @@ class RetrievalService:
|
||||
if attachment_id:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(self.embedding_search),
|
||||
propagate_context(self.embedding_search),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
dataset_id=dataset.id,
|
||||
query=attachment_id,
|
||||
@@ -868,7 +852,7 @@ class RetrievalService:
|
||||
if RetrievalMethod.is_support_fulltext_search(retrieval_method) and query:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
_propagate_otel_context(self.full_text_index_search),
|
||||
propagate_context(self.full_text_index_search),
|
||||
flask_app=current_app._get_current_object(), # type: ignore
|
||||
dataset_id=dataset.id,
|
||||
query=query,
|
||||
|
||||
@@ -6,10 +6,7 @@ from typing import Any
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.model_manager import ModelManager
|
||||
from core.rag.index_processor.constant.index_type import IndexTechniqueType
|
||||
from core.rag.models.document import AttachmentDocument, Document
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.dataset import ChildChunk, Dataset, DocumentSegment, SegmentAttachmentBinding
|
||||
from models.enums import SegmentType
|
||||
|
||||
@@ -69,34 +66,22 @@ class DatasetDocumentStore:
|
||||
|
||||
def add_documents(
|
||||
self,
|
||||
docs: Sequence[Document],
|
||||
session: Session,
|
||||
docs: Sequence[Document],
|
||||
token_counts: list[int],
|
||||
allow_update: bool = True,
|
||||
save_child: bool = False,
|
||||
):
|
||||
) -> None:
|
||||
document_token_pairs = list(zip(docs, token_counts, strict=True))
|
||||
|
||||
max_position = session.scalar(
|
||||
select(func.max(DocumentSegment.position)).where(DocumentSegment.document_id == self._document_id)
|
||||
)
|
||||
|
||||
if max_position is None:
|
||||
max_position = 0
|
||||
embedding_model = None
|
||||
if self._dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
model_manager = ModelManager.for_tenant(tenant_id=self._dataset.tenant_id)
|
||||
embedding_model = model_manager.get_model_instance(
|
||||
tenant_id=self._dataset.tenant_id,
|
||||
provider=self._dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=self._dataset.embedding_model,
|
||||
)
|
||||
|
||||
if embedding_model:
|
||||
page_content_list = [doc.page_content for doc in docs]
|
||||
tokens_list = embedding_model.get_text_embedding_num_tokens(page_content_list)
|
||||
else:
|
||||
tokens_list = [0] * len(docs)
|
||||
|
||||
for doc, tokens in zip(docs, tokens_list):
|
||||
for doc, tokens in document_token_pairs:
|
||||
if not isinstance(doc, Document):
|
||||
raise ValueError("doc must be a Document")
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Token counting for document segments."""
|
||||
|
||||
from core.model_manager import ModelManager
|
||||
from core.rag.index_processor.constant.index_type import IndexTechniqueType
|
||||
from core.rag.models.document import Document
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.dataset import Dataset
|
||||
|
||||
|
||||
def calculate_segment_token_counts(dataset: Dataset, documents: list[Document]) -> list[int]:
|
||||
"""Return one token count per document, invoking the embedding model only for high-quality indexes."""
|
||||
if not documents:
|
||||
return []
|
||||
|
||||
if dataset.indexing_technique != IndexTechniqueType.HIGH_QUALITY:
|
||||
return [0] * len(documents)
|
||||
|
||||
model_manager = ModelManager.for_tenant(tenant_id=dataset.tenant_id)
|
||||
embedding_model = model_manager.get_model_instance(
|
||||
tenant_id=dataset.tenant_id,
|
||||
provider=dataset.embedding_model_provider,
|
||||
model_type=ModelType.TEXT_EMBEDDING,
|
||||
model=dataset.embedding_model,
|
||||
)
|
||||
return embedding_model.get_text_embedding_num_tokens([document.page_content for document in documents])
|
||||
@@ -19,6 +19,7 @@ from core.rag.cleaner.clean_processor import CleanProcessor
|
||||
from core.rag.datasource.keyword.keyword_factory import Keyword
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.docstore.dataset_docstore import DatasetDocumentStore
|
||||
from core.rag.embedding.token_counter import calculate_segment_token_counts
|
||||
from core.rag.entities import Rule
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from core.rag.extractor.extract_processor import ExtractProcessor
|
||||
@@ -244,10 +245,16 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
|
||||
all_multimodal_documents.extend(doc.attachments)
|
||||
documents.append(doc)
|
||||
if documents:
|
||||
token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)
|
||||
# save node to document segment
|
||||
doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id)
|
||||
# add document segments
|
||||
doc_store.add_documents(docs=documents, save_child=False, session=session)
|
||||
doc_store.add_documents(
|
||||
session=session,
|
||||
docs=documents,
|
||||
token_counts=token_counts,
|
||||
save_child=False,
|
||||
)
|
||||
session.commit()
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
vector = Vector(dataset, session=session)
|
||||
|
||||
@@ -15,6 +15,7 @@ from core.model_manager import ModelInstance
|
||||
from core.rag.cleaner.clean_processor import CleanProcessor
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.docstore.dataset_docstore import DatasetDocumentStore
|
||||
from core.rag.embedding.token_counter import calculate_segment_token_counts
|
||||
from core.rag.entities import ParentMode, Rule
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from core.rag.extractor.extract_processor import ExtractProcessor
|
||||
@@ -304,6 +305,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
|
||||
doc.attachments = self._get_content_files(doc, current_user=account, session=session)
|
||||
documents.append(doc)
|
||||
if documents:
|
||||
token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)
|
||||
# update document parent mode
|
||||
dataset_process_rule = DatasetProcessRule(
|
||||
dataset_id=dataset.id,
|
||||
@@ -321,7 +323,12 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
|
||||
# save node to document segment
|
||||
doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id)
|
||||
# add document segments
|
||||
doc_store.add_documents(docs=documents, save_child=True, session=session)
|
||||
doc_store.add_documents(
|
||||
session=session,
|
||||
docs=documents,
|
||||
token_counts=token_counts,
|
||||
save_child=True,
|
||||
)
|
||||
session.commit()
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
all_child_documents = []
|
||||
|
||||
@@ -17,6 +17,7 @@ from core.llm_generator.llm_generator import LLMGenerator
|
||||
from core.rag.cleaner.clean_processor import CleanProcessor
|
||||
from core.rag.datasource.vdb.vector_factory import Vector
|
||||
from core.rag.docstore.dataset_docstore import DatasetDocumentStore
|
||||
from core.rag.embedding.token_counter import calculate_segment_token_counts
|
||||
from core.rag.entities import Rule
|
||||
from core.rag.extractor.entity.extract_setting import ExtractSetting
|
||||
from core.rag.extractor.extract_processor import ExtractProcessor
|
||||
@@ -205,9 +206,15 @@ class QAIndexProcessor(BaseIndexProcessor):
|
||||
doc = Document(page_content=qa_chunk.question, metadata=metadata)
|
||||
documents.append(doc)
|
||||
if documents:
|
||||
token_counts = calculate_segment_token_counts(dataset=dataset, documents=documents)
|
||||
# save node to document segment
|
||||
doc_store = DatasetDocumentStore(dataset=dataset, user_id=document.created_by, document_id=document.id)
|
||||
doc_store.add_documents(docs=documents, save_child=False, session=session)
|
||||
doc_store.add_documents(
|
||||
session=session,
|
||||
docs=documents,
|
||||
token_counts=token_counts,
|
||||
save_child=False,
|
||||
)
|
||||
session.commit()
|
||||
if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
|
||||
vector = Vector(dataset, session=session)
|
||||
|
||||
@@ -9,6 +9,7 @@ from core.rag.index_processor.constant.query_type import QueryType
|
||||
from core.rag.models.document import Document
|
||||
from core.rag.rerank.rerank_base import BaseRerankRunner
|
||||
from extensions.ext_storage import storage
|
||||
from extensions.otel import trace_span
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from graphon.model_runtime.entities.rerank_entities import MultimodalRerankInput, RerankResult
|
||||
from models.model import UploadFile
|
||||
@@ -22,6 +23,7 @@ class RerankModelRunner(BaseRerankRunner):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
@trace_span()
|
||||
def run(
|
||||
self,
|
||||
query: str,
|
||||
|
||||
@@ -65,6 +65,7 @@ from core.workflow.nodes.knowledge_retrieval.retrieval import (
|
||||
)
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from extensions.otel import propagate_context, trace_span
|
||||
from graphon.file import File, FileTransferMethod, FileType
|
||||
from graphon.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMUsage
|
||||
from graphon.model_runtime.entities.message_entities import PromptMessage, PromptMessageRole, PromptMessageTool
|
||||
@@ -116,6 +117,7 @@ class DatasetRetrieval:
|
||||
else:
|
||||
self._llm_usage = self._llm_usage.plus(usage)
|
||||
|
||||
@trace_span()
|
||||
def knowledge_retrieval(self, session: Session, request: KnowledgeRetrievalRequest) -> list[Source]:
|
||||
self._check_knowledge_rate_limit(request.tenant_id)
|
||||
available_datasets = self._get_available_datasets(request.tenant_id, request.dataset_ids)
|
||||
@@ -599,6 +601,7 @@ class DatasetRetrieval:
|
||||
return "\n".join([document_context.content for document_context in document_context_list]), context_files
|
||||
return "", context_files
|
||||
|
||||
@trace_span()
|
||||
def single_retrieve(
|
||||
self,
|
||||
session: Session,
|
||||
@@ -724,7 +727,7 @@ class DatasetRetrieval:
|
||||
|
||||
if results:
|
||||
thread = threading.Thread(
|
||||
target=self._on_retrieval_end,
|
||||
target=propagate_context(self._on_retrieval_end),
|
||||
kwargs={
|
||||
"flask_app": current_app._get_current_object(), # type: ignore
|
||||
"documents": results,
|
||||
@@ -737,6 +740,7 @@ class DatasetRetrieval:
|
||||
return results
|
||||
return []
|
||||
|
||||
@trace_span()
|
||||
def multiple_retrieve(
|
||||
self,
|
||||
app_id: str,
|
||||
@@ -798,7 +802,7 @@ class DatasetRetrieval:
|
||||
|
||||
if query:
|
||||
query_thread = threading.Thread(
|
||||
target=self._multiple_retrieve_thread,
|
||||
target=propagate_context(self._multiple_retrieve_thread_safely),
|
||||
kwargs={
|
||||
"flask_app": current_app._get_current_object(), # type: ignore
|
||||
"available_datasets": available_datasets,
|
||||
@@ -824,7 +828,7 @@ class DatasetRetrieval:
|
||||
if attachment_ids:
|
||||
for attachment_id in attachment_ids:
|
||||
attachment_thread = threading.Thread(
|
||||
target=self._multiple_retrieve_thread,
|
||||
target=propagate_context(self._multiple_retrieve_thread_safely),
|
||||
kwargs={
|
||||
"flask_app": current_app._get_current_object(), # type: ignore
|
||||
"available_datasets": available_datasets,
|
||||
@@ -865,7 +869,7 @@ class DatasetRetrieval:
|
||||
if all_documents:
|
||||
# add thread to call _on_retrieval_end
|
||||
retrieval_end_thread = threading.Thread(
|
||||
target=self._on_retrieval_end,
|
||||
target=propagate_context(self._on_retrieval_end),
|
||||
kwargs={
|
||||
"flask_app": current_app._get_current_object(), # type: ignore
|
||||
"documents": all_documents,
|
||||
@@ -1161,6 +1165,7 @@ class DatasetRetrieval:
|
||||
|
||||
all_documents.extend(documents)
|
||||
|
||||
@trace_span()
|
||||
def _run_retriever_thread(
|
||||
self,
|
||||
*,
|
||||
@@ -1172,27 +1177,51 @@ class DatasetRetrieval:
|
||||
document_ids_filter: list[str] | None,
|
||||
metadata_condition: MetadataFilteringCondition | None,
|
||||
attachment_ids: list[str] | None,
|
||||
) -> None:
|
||||
with session_factory.create_session() as session:
|
||||
self._retriever(
|
||||
flask_app=flask_app,
|
||||
session=session,
|
||||
dataset_id=dataset_id,
|
||||
query=query or "",
|
||||
top_k=top_k,
|
||||
all_documents=all_documents,
|
||||
document_ids_filter=document_ids_filter,
|
||||
metadata_condition=metadata_condition,
|
||||
attachment_ids=attachment_ids,
|
||||
)
|
||||
|
||||
def _run_retriever_thread_safely(
|
||||
self,
|
||||
*,
|
||||
flask_app: Flask,
|
||||
dataset_id: str,
|
||||
query: str | None,
|
||||
top_k: int,
|
||||
all_documents: list[Document],
|
||||
document_ids_filter: list[str] | None,
|
||||
metadata_condition: MetadataFilteringCondition | None,
|
||||
attachment_ids: list[str] | None,
|
||||
cancel_event: threading.Event | None,
|
||||
thread_exceptions: list[Exception] | None,
|
||||
) -> None:
|
||||
"""Collect errors only after they pass through the traced retrieval method."""
|
||||
try:
|
||||
with session_factory.create_session() as session:
|
||||
self._retriever(
|
||||
flask_app=flask_app,
|
||||
session=session,
|
||||
dataset_id=dataset_id,
|
||||
query=query or "",
|
||||
top_k=top_k,
|
||||
all_documents=all_documents,
|
||||
document_ids_filter=document_ids_filter,
|
||||
metadata_condition=metadata_condition,
|
||||
attachment_ids=attachment_ids,
|
||||
)
|
||||
except Exception as e:
|
||||
self._run_retriever_thread(
|
||||
flask_app=flask_app,
|
||||
dataset_id=dataset_id,
|
||||
query=query,
|
||||
top_k=top_k,
|
||||
all_documents=all_documents,
|
||||
document_ids_filter=document_ids_filter,
|
||||
metadata_condition=metadata_condition,
|
||||
attachment_ids=attachment_ids,
|
||||
)
|
||||
except Exception as exc:
|
||||
if cancel_event:
|
||||
cancel_event.set()
|
||||
if thread_exceptions is not None:
|
||||
thread_exceptions.append(e)
|
||||
thread_exceptions.append(exc)
|
||||
|
||||
def to_dataset_retriever_tool(
|
||||
self,
|
||||
@@ -1795,6 +1824,7 @@ class DatasetRetrieval:
|
||||
|
||||
return full_text, usage
|
||||
|
||||
@trace_span()
|
||||
def _multiple_retrieve_thread(
|
||||
self,
|
||||
flask_app: Flask,
|
||||
@@ -1813,11 +1843,11 @@ class DatasetRetrieval:
|
||||
attachment_id: str | None,
|
||||
dataset_count: int,
|
||||
cancel_event: threading.Event | None = None,
|
||||
thread_exceptions: list[Exception] | None = None,
|
||||
):
|
||||
) -> None:
|
||||
try:
|
||||
with flask_app.app_context():
|
||||
threads = []
|
||||
retrieval_thread_exceptions: list[Exception] = []
|
||||
all_documents_item: list[Document] = []
|
||||
index_type = None
|
||||
for dataset in available_datasets:
|
||||
@@ -1836,7 +1866,7 @@ class DatasetRetrieval:
|
||||
else:
|
||||
continue
|
||||
retrieval_thread = threading.Thread(
|
||||
target=self._run_retriever_thread,
|
||||
target=propagate_context(self._run_retriever_thread_safely),
|
||||
kwargs={
|
||||
"flask_app": flask_app,
|
||||
"dataset_id": dataset.id,
|
||||
@@ -1847,7 +1877,7 @@ class DatasetRetrieval:
|
||||
"metadata_condition": metadata_condition,
|
||||
"attachment_ids": [attachment_id] if attachment_id else None,
|
||||
"cancel_event": cancel_event,
|
||||
"thread_exceptions": thread_exceptions,
|
||||
"thread_exceptions": retrieval_thread_exceptions,
|
||||
},
|
||||
)
|
||||
threads.append(retrieval_thread)
|
||||
@@ -1862,6 +1892,9 @@ class DatasetRetrieval:
|
||||
if cancel_event and cancel_event.is_set():
|
||||
break
|
||||
|
||||
if retrieval_thread_exceptions:
|
||||
raise retrieval_thread_exceptions[0]
|
||||
|
||||
# Skip second reranking when there is only one dataset
|
||||
if reranking_enable and dataset_count > 1:
|
||||
# do rerank for searched documents
|
||||
@@ -1902,11 +1935,55 @@ class DatasetRetrieval:
|
||||
all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item
|
||||
if all_documents_item:
|
||||
all_documents.extend(all_documents_item)
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
def _multiple_retrieve_thread_safely(
|
||||
self,
|
||||
*,
|
||||
flask_app: Flask,
|
||||
available_datasets: list[Dataset],
|
||||
metadata_condition: MetadataFilteringCondition | None,
|
||||
metadata_filter_document_ids: dict[str, list[str]] | None,
|
||||
all_documents: list[Document],
|
||||
tenant_id: str,
|
||||
reranking_enable: bool,
|
||||
reranking_mode: str,
|
||||
reranking_model: RerankingModelDict | None,
|
||||
weights: WeightsDict | None,
|
||||
top_k: int,
|
||||
score_threshold: float,
|
||||
query: str | None,
|
||||
attachment_id: str | None,
|
||||
dataset_count: int,
|
||||
cancel_event: threading.Event | None = None,
|
||||
thread_exceptions: list[Exception] | None = None,
|
||||
) -> None:
|
||||
"""Collect errors only after they pass through the traced multi-retrieval method."""
|
||||
try:
|
||||
self._multiple_retrieve_thread(
|
||||
flask_app=flask_app,
|
||||
available_datasets=available_datasets,
|
||||
metadata_condition=metadata_condition,
|
||||
metadata_filter_document_ids=metadata_filter_document_ids,
|
||||
all_documents=all_documents,
|
||||
tenant_id=tenant_id,
|
||||
reranking_enable=reranking_enable,
|
||||
reranking_mode=reranking_mode,
|
||||
reranking_model=reranking_model,
|
||||
weights=weights,
|
||||
top_k=top_k,
|
||||
score_threshold=score_threshold,
|
||||
query=query,
|
||||
attachment_id=attachment_id,
|
||||
dataset_count=dataset_count,
|
||||
cancel_event=cancel_event,
|
||||
)
|
||||
except Exception as exc:
|
||||
if cancel_event:
|
||||
cancel_event.set()
|
||||
if thread_exceptions is not None:
|
||||
thread_exceptions.append(e)
|
||||
thread_exceptions.append(exc)
|
||||
|
||||
def _get_available_datasets(self, tenant_id: str, dataset_ids: list[str]) -> list[Dataset]:
|
||||
with session_factory.create_session() as session:
|
||||
|
||||
@@ -10,7 +10,6 @@ class RBACResourceScope(StrEnum):
|
||||
|
||||
APP = "app"
|
||||
DATASET = "dataset"
|
||||
KNOWLEDGE_FS = "knowledge_space"
|
||||
WORKSPACE = "workspace"
|
||||
|
||||
|
||||
@@ -58,19 +57,11 @@ class RBACPermission(StrEnum):
|
||||
DATASET_EXTERNAL_CONNECT = "dataset_external_connect"
|
||||
DATASET_IMPORT_EXPORT_DSL = "dataset_import_export_dsl"
|
||||
|
||||
KNOWLEDGE_FS_READ = "knowledge_space_read"
|
||||
KNOWLEDGE_FS_CREATE = "knowledge_space_create"
|
||||
KNOWLEDGE_FS_EDIT = "knowledge_space_edit"
|
||||
KNOWLEDGE_FS_DELETE = "knowledge_space_delete"
|
||||
KNOWLEDGE_FS_ACCESS_CONFIG = "knowledge_space_access_config"
|
||||
KNOWLEDGE_FS_API_KEY_MANAGE = "knowledge_space_api_key_manage"
|
||||
KNOWLEDGE_FS_DOCUMENT_WRITE = "knowledge_space_document_write"
|
||||
KNOWLEDGE_FS_QUERY = "knowledge_space_query"
|
||||
|
||||
WORKSPACE_MEMBER_MANAGE = "workspace_member_manage"
|
||||
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
|
||||
API_EXTENSION_MANAGE = "api_extension_manage"
|
||||
CUSTOMIZATION_MANAGE = "customization_manage"
|
||||
AGENT_MANAGE = "agent_manage"
|
||||
|
||||
SNIPPETS_CREATE_AND_MODIFY = "snippets_create_and_modify"
|
||||
SNIPPETS_MANAGE = "snippets_management"
|
||||
|
||||
@@ -58,7 +58,6 @@ class Tool(ABC):
|
||||
if self.runtime and self.runtime.runtime_parameters:
|
||||
tool_parameters.update(self.runtime.runtime_parameters)
|
||||
|
||||
# try parse tool parameters into the correct type
|
||||
tool_parameters = self._transform_tool_parameters_type(tool_parameters)
|
||||
|
||||
result = self._invoke(
|
||||
@@ -87,14 +86,14 @@ class Tool(ABC):
|
||||
return result
|
||||
|
||||
def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Transform tool parameters type
|
||||
"""
|
||||
# Temp fix for the issue that the tool parameters will be converted to empty while validating the credentials
|
||||
"""Transform declared tool parameter values without resolving runtime schemas."""
|
||||
result = deepcopy(tool_parameters)
|
||||
for parameter in self.entity.parameters or []:
|
||||
if parameter.name in tool_parameters:
|
||||
result[parameter.name] = parameter.type.cast_value(tool_parameters[parameter.name])
|
||||
if parameter.multiple:
|
||||
result[parameter.name] = parameter.init_frontend_parameter(result.get(parameter.name))
|
||||
else:
|
||||
result[parameter.name] = parameter.type.cast_value(tool_parameters[parameter.name])
|
||||
|
||||
return result
|
||||
|
||||
@@ -196,17 +195,31 @@ class Tool(ABC):
|
||||
}:
|
||||
continue
|
||||
|
||||
parameter_schema: dict[str, Any] = (
|
||||
{
|
||||
"type": parameter.type.as_normal_type(),
|
||||
"description": parameter.llm_description or "",
|
||||
}
|
||||
if parameter.input_schema is None
|
||||
else deepcopy(parameter.input_schema)
|
||||
)
|
||||
is_multiple_select = parameter.multiple and parameter.type in {
|
||||
ToolParameter.ToolParameterType.SELECT,
|
||||
ToolParameter.ToolParameterType.DYNAMIC_SELECT,
|
||||
}
|
||||
if is_multiple_select:
|
||||
item_schema: dict[str, Any] = {"type": "string"}
|
||||
if parameter.type == ToolParameter.ToolParameterType.SELECT and parameter.options:
|
||||
item_schema["enum"] = [option.value for option in parameter.options]
|
||||
parameter_schema: dict[str, Any] = {"type": "array", "items": item_schema}
|
||||
else:
|
||||
parameter_schema = (
|
||||
{
|
||||
"type": parameter.type.as_normal_type(),
|
||||
"description": parameter.llm_description or "",
|
||||
}
|
||||
if parameter.input_schema is None
|
||||
else deepcopy(parameter.input_schema)
|
||||
)
|
||||
parameter_schema.setdefault("description", parameter.llm_description or "")
|
||||
|
||||
if parameter.type == ToolParameter.ToolParameterType.SELECT and parameter.options:
|
||||
if (
|
||||
not is_multiple_select
|
||||
and parameter.type == ToolParameter.ToolParameterType.SELECT
|
||||
and parameter.options
|
||||
):
|
||||
parameter_schema["enum"] = [option.value for option in parameter.options]
|
||||
|
||||
schema["properties"][parameter.name] = parameter_schema
|
||||
|
||||
@@ -2,7 +2,7 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from core.plugin.entities.plugin_daemon import CredentialType
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom
|
||||
|
||||
@@ -20,7 +20,6 @@ class ToolRuntime(BaseModel):
|
||||
tool_id: str | None = None
|
||||
invoke_from: InvokeFrom | None = None
|
||||
tool_invoke_from: ToolInvokeFrom | None = None
|
||||
dify_run_context: DifyRunContext | None = Field(default=None, exclude=True, repr=False)
|
||||
credentials: dict[str, Any] = Field(default_factory=dict)
|
||||
credential_type: CredentialType = Field(default=CredentialType.API_KEY)
|
||||
runtime_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
- audio
|
||||
- code
|
||||
- knowledge_fs
|
||||
- time
|
||||
- webscraper
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="none">
|
||||
<rect width="64" height="64" rx="14" fill="#155EEF"/>
|
||||
<path d="M17 17h20c5.5 0 10 4.5 10 10v20H27c-5.5 0-10-4.5-10-10V17Z" fill="white" fill-opacity=".96"/>
|
||||
<path d="M26 27h12M26 34h12M26 41h7" stroke="#155EEF" stroke-width="4" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 340 B |
@@ -1,9 +0,0 @@
|
||||
from typing import Any, override
|
||||
|
||||
from core.tools.builtin_tool.provider import BuiltinToolProviderController
|
||||
|
||||
|
||||
class KnowledgeFSProvider(BuiltinToolProviderController):
|
||||
@override
|
||||
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
_ = (user_id, credentials)
|
||||
@@ -1,13 +0,0 @@
|
||||
identity:
|
||||
author: Dify
|
||||
name: knowledge_fs
|
||||
label:
|
||||
en_US: KnowledgeFS
|
||||
zh_Hans: KnowledgeFS
|
||||
description:
|
||||
en_US: Run explicitly bound KnowledgeFS operations from an Agent or Workflow.
|
||||
zh_Hans: 从 Agent 或 Workflow 执行显式绑定的 KnowledgeFS 操作。
|
||||
icon: icon.svg
|
||||
tags:
|
||||
- rag
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any, override
|
||||
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.tools.builtin_tool.tool import BuiltinTool
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage
|
||||
from core.tools.errors import ToolInvokeError
|
||||
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
|
||||
from services.knowledge_fs.app_execution_capability import KnowledgeResourceRef
|
||||
from services.knowledge_fs.product_dto import KnowledgeFSResearchTaskCreatePayload
|
||||
from services.knowledge_fs.runtime import create_knowledge_fs_runtime
|
||||
|
||||
|
||||
class KnowledgeFSCreateResearchTaskTool(BuiltinTool):
|
||||
@override
|
||||
def _invoke(
|
||||
self,
|
||||
session: Session,
|
||||
user_id: str,
|
||||
tool_parameters: dict[str, Any],
|
||||
conversation_id: str | None = None,
|
||||
app_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
) -> Generator[ToolInvokeMessage, None, None]:
|
||||
_ = (user_id, conversation_id, app_id, message_id)
|
||||
run_context = self.runtime.dify_run_context
|
||||
if run_context is None or self.runtime.tenant_id != run_context.tenant_id:
|
||||
raise ToolInvokeError("KnowledgeFS requires a trusted Dify run context")
|
||||
match self.runtime.tool_invoke_from:
|
||||
case ToolInvokeFrom.AGENT:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
|
||||
case ToolInvokeFrom.WORKFLOW:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
|
||||
case _:
|
||||
raise ToolInvokeError("KnowledgeFS is only available to Agent and Workflow callers")
|
||||
|
||||
try:
|
||||
resource = KnowledgeResourceRef.model_validate(tool_parameters.get("resource"))
|
||||
payload_data: dict[str, object] = {
|
||||
"query": tool_parameters.get("query"),
|
||||
}
|
||||
mode = tool_parameters.get("mode")
|
||||
if mode:
|
||||
payload_data["mode"] = mode
|
||||
payload = KnowledgeFSResearchTaskCreatePayload.model_validate(payload_data)
|
||||
runtime = create_knowledge_fs_runtime(sessionmaker(bind=session.get_bind(), expire_on_commit=False))
|
||||
response = runtime.app_capabilities.create_research_task(
|
||||
run_context=run_context,
|
||||
caller_kind=caller_kind,
|
||||
resource=resource,
|
||||
payload=payload,
|
||||
)
|
||||
except ToolInvokeError:
|
||||
raise
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
raise ToolInvokeError(str(exc)) from exc
|
||||
|
||||
yield self.create_json_message(response.model_dump(mode="json", by_alias=True))
|
||||
@@ -1,75 +0,0 @@
|
||||
identity:
|
||||
name: create_research_task
|
||||
author: Dify
|
||||
label:
|
||||
en_US: Create KnowledgeFS Research Task
|
||||
zh_Hans: 创建 KnowledgeFS Research 任务
|
||||
description:
|
||||
human:
|
||||
en_US: Create a Research task in an explicitly bound KnowledgeFS space.
|
||||
zh_Hans: 在显式绑定的 KnowledgeFS 空间中创建 Research 任务。
|
||||
llm: Create a durable research task using an explicitly configured KnowledgeFS resource.
|
||||
parameters:
|
||||
- name: resource
|
||||
type: object
|
||||
required: true
|
||||
label:
|
||||
en_US: KnowledgeFS resource
|
||||
zh_Hans: KnowledgeFS 资源
|
||||
human_description:
|
||||
en_US: A typed KnowledgeFS control-space reference configured by the app author.
|
||||
zh_Hans: 由应用作者配置的类型化 KnowledgeFS control-space 引用。
|
||||
form: form
|
||||
input_schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
const: knowledge_fs
|
||||
control_space_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
required:
|
||||
- kind
|
||||
- control_space_id
|
||||
- name: query
|
||||
type: string
|
||||
required: true
|
||||
label:
|
||||
en_US: Research query
|
||||
zh_Hans: Research 查询
|
||||
human_description:
|
||||
en_US: The question the Research task should investigate.
|
||||
zh_Hans: Research 任务需要调查的问题。
|
||||
llm_description: The question to investigate with KnowledgeFS.
|
||||
form: llm
|
||||
- name: mode
|
||||
type: select
|
||||
required: false
|
||||
label:
|
||||
en_US: Mode
|
||||
zh_Hans: 模式
|
||||
human_description:
|
||||
en_US: Optional KnowledgeFS retrieval mode.
|
||||
zh_Hans: 可选的 KnowledgeFS 检索模式。
|
||||
llm_description: Optional retrieval mode. Use auto unless the task needs a specific mode.
|
||||
form: llm
|
||||
options:
|
||||
- value: auto
|
||||
label:
|
||||
en_US: Auto
|
||||
zh_Hans: 自动
|
||||
- value: fast
|
||||
label:
|
||||
en_US: Fast
|
||||
zh_Hans: 快速
|
||||
- value: deep
|
||||
label:
|
||||
en_US: Deep
|
||||
zh_Hans: 深度
|
||||
- value: research
|
||||
label:
|
||||
en_US: Research
|
||||
zh_Hans: Research
|
||||
|
||||
@@ -292,9 +292,7 @@ class ToolInvokeMessageBinary(BaseModel):
|
||||
|
||||
|
||||
class ToolParameter(PluginParameter):
|
||||
"""
|
||||
Overrides type
|
||||
"""
|
||||
"""Tool-specific parameter declaration and invocation-value normalization."""
|
||||
|
||||
class ToolParameterType(StrEnum):
|
||||
"""
|
||||
@@ -333,12 +331,28 @@ class ToolParameter(PluginParameter):
|
||||
LLM = auto() # will be set by LLM
|
||||
|
||||
type: ToolParameterType = Field(..., description="The type of the parameter")
|
||||
multiple: bool = Field(
|
||||
default=False,
|
||||
description="Whether the parameter is multiple select, only valid for select or dynamic-select type",
|
||||
)
|
||||
human_description: I18nObject | None = Field(default=None, description="The description presented to the user")
|
||||
form: ToolParameterForm = Field(..., description="The form of the parameter, schema/form/llm")
|
||||
llm_description: str | None = None
|
||||
# MCP object and array type parameters use this field to store the schema
|
||||
input_schema: dict[str, Any] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_multiple(self) -> ToolParameter:
|
||||
supports_multiple = self.type in {
|
||||
self.ToolParameterType.SELECT,
|
||||
self.ToolParameterType.DYNAMIC_SELECT,
|
||||
}
|
||||
if self.multiple and not supports_multiple:
|
||||
raise ValueError("multiple is only valid for select and dynamic-select parameters")
|
||||
if supports_multiple and self.default is not None and (isinstance(self.default, list) != self.multiple):
|
||||
raise ValueError("default must be a list exactly when multiple is true")
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def get_simple_instance(
|
||||
cls,
|
||||
@@ -378,8 +392,25 @@ class ToolParameter(PluginParameter):
|
||||
options=option_objs,
|
||||
)
|
||||
|
||||
def init_frontend_parameter(self, value: Any):
|
||||
return init_frontend_parameter(self, self.type, value)
|
||||
def init_frontend_parameter(self, value: Any) -> Any:
|
||||
"""Normalize a value against this tool parameter's full declaration."""
|
||||
if not self.multiple:
|
||||
return init_frontend_parameter(self, self.type, value)
|
||||
|
||||
parameter_value = self.default if value is None else value
|
||||
if parameter_value is None:
|
||||
parameter_value = []
|
||||
if not isinstance(parameter_value, list):
|
||||
raise ValueError(f"tool parameter {self.name} must be a list when multiple is true")
|
||||
if not all(isinstance(item, str) for item in parameter_value):
|
||||
raise ValueError(f"tool parameter {self.name} must contain only strings")
|
||||
if self.required and not parameter_value:
|
||||
raise ValueError(f"tool parameter {self.name} not found in tool config")
|
||||
if self.type == self.ToolParameterType.SELECT:
|
||||
options = [option.value for option in self.options]
|
||||
if any(item not in options for item in parameter_value):
|
||||
raise ValueError(f"tool parameter {self.name} value {parameter_value} not in options {options}")
|
||||
return parameter_value
|
||||
|
||||
|
||||
class ToolProviderIdentity(BaseModel):
|
||||
|
||||
@@ -493,7 +493,6 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):
|
||||
self._run_context.invoke_from,
|
||||
variable_pool,
|
||||
)
|
||||
tool_runtime.runtime.dify_run_context = self._run_context
|
||||
except ToolNodeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
||||
@@ -198,23 +198,8 @@ class AgentRuntimeSupport:
|
||||
if model_schema:
|
||||
model_schema = self._remove_unsupported_model_features_for_old_version(model_schema)
|
||||
value["entity"] = model_schema.model_dump(mode="json")
|
||||
# The model selector value from the workflow frontend only
|
||||
# carries provider/model/mode — it does NOT include
|
||||
# completion_params. AgentStrategy plugins (cot_agent,
|
||||
# function_calling) read completion_params to build the
|
||||
# LLMModelConfig that is backwards-invoked, and some model
|
||||
# providers raise KeyError('required') when
|
||||
# completion_params is empty because their parameter_rules
|
||||
# declare required fields with no default. Populate
|
||||
# completion_params with the defaults declared in the model
|
||||
# schema so the plugin daemon always receives a valid set
|
||||
# of model parameters.
|
||||
if "completion_params" not in value:
|
||||
value["completion_params"] = self._extract_default_completion_params(model_schema)
|
||||
else:
|
||||
value["entity"] = None
|
||||
if "completion_params" not in value:
|
||||
value["completion_params"] = {}
|
||||
result[parameter_name] = value
|
||||
|
||||
return result
|
||||
@@ -290,24 +275,6 @@ class AgentRuntimeSupport:
|
||||
model_schema.features.remove(feature)
|
||||
return model_schema
|
||||
|
||||
@staticmethod
|
||||
def _extract_default_completion_params(model_schema: AIModelEntity) -> dict[str, Any]:
|
||||
"""Build a completion_params dict from the model schema's parameter_rules.
|
||||
|
||||
The workflow Agent node's model-selector parameter only stores
|
||||
provider/model/mode — it never carries completion_params. When the
|
||||
value is forwarded to the plugin daemon, AgentModelConfig defaults
|
||||
completion_params to ``{}``, which causes some model providers to fail
|
||||
because their parameter_rules declare required fields. This helper
|
||||
collects the ``default`` value of every parameter_rule that has one so
|
||||
the plugin daemon receives a valid, non-empty set of model parameters.
|
||||
"""
|
||||
completion_params: dict[str, Any] = {}
|
||||
for rule in model_schema.parameter_rules:
|
||||
if rule.default is not None:
|
||||
completion_params[rule.name] = rule.default
|
||||
return completion_params
|
||||
|
||||
@staticmethod
|
||||
def _filter_mcp_type_tool(
|
||||
strategy: ResolvedAgentStrategy,
|
||||
|
||||
@@ -4,8 +4,16 @@ from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from core.agent.publish_visibility import workflow_callable_active_snapshot_filter
|
||||
from core.db.session_factory import session_factory
|
||||
from models.agent import Agent, AgentConfigSnapshot, AgentStatus, WorkflowAgentBindingType, WorkflowAgentNodeBinding
|
||||
from models.agent import (
|
||||
Agent,
|
||||
AgentConfigSnapshot,
|
||||
AgentScope,
|
||||
AgentStatus,
|
||||
WorkflowAgentBindingType,
|
||||
WorkflowAgentNodeBinding,
|
||||
)
|
||||
|
||||
|
||||
class WorkflowAgentBindingError(Exception):
|
||||
@@ -24,7 +32,7 @@ class WorkflowAgentBindingBundle:
|
||||
|
||||
|
||||
class WorkflowAgentBindingResolver:
|
||||
"""Resolve the Agent binding owned by the current workflow id and node id."""
|
||||
"""Resolve an owned binding without allowing unpublished roster snapshots to run."""
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
@@ -53,18 +61,20 @@ class WorkflowAgentBindingResolver:
|
||||
if binding.agent_id is None:
|
||||
raise WorkflowAgentBindingError("agent_not_available", "Workflow Agent binding has no agent.")
|
||||
|
||||
agent = session.scalar(
|
||||
select(Agent)
|
||||
.where(
|
||||
Agent.tenant_id == tenant_id,
|
||||
Agent.id == binding.agent_id,
|
||||
)
|
||||
.limit(1)
|
||||
agent_stmt = select(Agent).where(
|
||||
Agent.tenant_id == tenant_id,
|
||||
Agent.id == binding.agent_id,
|
||||
)
|
||||
if binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT:
|
||||
agent_stmt = agent_stmt.where(
|
||||
Agent.scope == AgentScope.ROSTER,
|
||||
workflow_callable_active_snapshot_filter(),
|
||||
)
|
||||
agent = session.scalar(agent_stmt.limit(1))
|
||||
if agent is None or agent.status == AgentStatus.ARCHIVED:
|
||||
raise WorkflowAgentBindingError(
|
||||
"agent_not_available",
|
||||
f"Agent {binding.agent_id} is not available.",
|
||||
f"Agent {binding.agent_id} is not available or has not been published.",
|
||||
)
|
||||
|
||||
snapshot_id = (
|
||||
|
||||
@@ -38,6 +38,7 @@ from dify_agent.layers.shell import (
|
||||
)
|
||||
from dify_agent.protocol import CreateRunRequest, DeferredToolResultsPayload
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from clients.agent_backend import (
|
||||
AgentBackendModelConfig,
|
||||
@@ -206,14 +207,22 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
"cli_tool_count": len(agent_soul.tools.cli_tools),
|
||||
}
|
||||
|
||||
runtime_config_skills = load_runtime_agent_skill_configs(
|
||||
tenant_id=context.dify_context.tenant_id,
|
||||
agent_id=context.agent.id,
|
||||
)
|
||||
config_layer_config, config_warnings = build_config_layer_config(
|
||||
agent_soul,
|
||||
agent_id=context.agent.id,
|
||||
config_version_id=context.snapshot.id,
|
||||
config_version_kind="snapshot",
|
||||
runtime_config_skills=runtime_config_skills,
|
||||
)
|
||||
append_runtime_warnings(metadata, config_warnings)
|
||||
soul_prompt_resolver = build_config_aware_soul_mention_resolver(agent_soul)
|
||||
soul_prompt_resolver = build_config_aware_soul_mention_resolver(
|
||||
agent_soul,
|
||||
runtime_config_skills=runtime_config_skills,
|
||||
)
|
||||
soul_prompt = expand_prompt_mentions(agent_soul.prompt.system_prompt, soul_prompt_resolver).strip()
|
||||
knowledge_config = build_knowledge_layer_config(agent_soul)
|
||||
|
||||
@@ -250,7 +259,6 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
agent_config_version_kind="snapshot",
|
||||
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
agent_soul_prompt=soul_prompt or None,
|
||||
workflow_node_job_prompt=workflow_job_prompt,
|
||||
@@ -884,11 +892,16 @@ def append_runtime_warnings(metadata: dict[str, Any], warnings: list[dict[str, s
|
||||
existing.extend(warnings)
|
||||
|
||||
|
||||
def build_config_aware_soul_mention_resolver(agent_soul: AgentSoulConfig):
|
||||
def build_config_aware_soul_mention_resolver(
|
||||
agent_soul: AgentSoulConfig,
|
||||
*,
|
||||
runtime_config_skills: Sequence[DifyConfigSkillConfig] = (),
|
||||
):
|
||||
"""Resolve config skill/file mentions and delegate the rest to Agent Soul."""
|
||||
|
||||
base_resolver = build_soul_mention_resolver(agent_soul)
|
||||
skill_names = {item.name for item in agent_soul.config_skills if not item.is_missing}
|
||||
skill_names.update(item.name for item in runtime_config_skills)
|
||||
file_names = {item.name for item in agent_soul.config_files if not item.is_missing}
|
||||
|
||||
def _resolve(mention: object) -> str | None:
|
||||
@@ -906,12 +919,34 @@ def build_config_aware_soul_mention_resolver(agent_soul: AgentSoulConfig):
|
||||
return _resolve
|
||||
|
||||
|
||||
def load_runtime_agent_skill_configs(*, tenant_id: str, agent_id: str) -> list[DifyConfigSkillConfig]:
|
||||
"""Return workspace-bound Skills as prompt-safe runtime config skills."""
|
||||
from services.skill_management_service import SkillManagementService
|
||||
|
||||
try:
|
||||
runtime_skills = SkillManagementService().list_runtime_agent_skills(tenant_id=tenant_id, agent_id=agent_id)
|
||||
except OperationalError as exc:
|
||||
if "no such table: agent_skill_bindings" not in str(exc.orig):
|
||||
raise
|
||||
runtime_skills = []
|
||||
return [
|
||||
DifyConfigSkillConfig(
|
||||
name=str(item["name"]),
|
||||
description=str(item.get("description") or ""),
|
||||
size=cast(int | None, item.get("size")),
|
||||
mime_type=cast(str | None, item.get("mime_type")),
|
||||
)
|
||||
for item in runtime_skills
|
||||
]
|
||||
|
||||
|
||||
def build_config_layer_config(
|
||||
agent_soul: AgentSoulConfig,
|
||||
*,
|
||||
agent_id: str | None = None,
|
||||
config_version_id: str | None = None,
|
||||
config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
|
||||
runtime_config_skills: Sequence[DifyConfigSkillConfig] = (),
|
||||
) -> tuple[DifyConfigLayerConfig, list[dict[str, str]]]:
|
||||
"""Build the always-present Agent config layer from Agent Soul state.
|
||||
|
||||
@@ -928,8 +963,23 @@ def build_config_layer_config(
|
||||
)
|
||||
)
|
||||
available_skills = [skill for skill in agent_soul.config_skills if not skill.is_missing]
|
||||
skill_configs = [
|
||||
DifyConfigSkillConfig(
|
||||
name=skill.name,
|
||||
description=skill.description,
|
||||
size=skill.size,
|
||||
mime_type=skill.mime_type,
|
||||
)
|
||||
for skill in available_skills
|
||||
]
|
||||
seen_skill_names = {skill.name for skill in skill_configs}
|
||||
for skill in runtime_config_skills:
|
||||
if skill.name in seen_skill_names:
|
||||
continue
|
||||
seen_skill_names.add(skill.name)
|
||||
skill_configs.append(skill)
|
||||
available_files = [file_ref for file_ref in agent_soul.config_files if not file_ref.is_missing]
|
||||
skill_names = {skill.name for skill in available_skills}
|
||||
skill_names = {skill.name for skill in skill_configs}
|
||||
file_names = {file_ref.name for file_ref in available_files}
|
||||
warnings: list[dict[str, str]] = [
|
||||
{
|
||||
@@ -966,15 +1016,7 @@ def build_config_layer_config(
|
||||
kind=config_version_kind,
|
||||
writable=config_version_kind == "build_draft",
|
||||
),
|
||||
skills=[
|
||||
DifyConfigSkillConfig(
|
||||
name=skill.name,
|
||||
description=skill.description,
|
||||
size=skill.size,
|
||||
mime_type=skill.mime_type,
|
||||
)
|
||||
for skill in available_skills
|
||||
],
|
||||
skills=skill_configs,
|
||||
files=[
|
||||
DifyConfigFileConfig(
|
||||
name=file_ref.name,
|
||||
|
||||
@@ -6,9 +6,17 @@ from typing import Any
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.agent.publish_visibility import workflow_callable_active_snapshot_filter
|
||||
from core.workflow.graph_topology import WorkflowGraphTopology
|
||||
from graphon.enums import BuiltinNodeTypes
|
||||
from models.agent import Agent, AgentConfigSnapshot, AgentStatus, WorkflowAgentBindingType, WorkflowAgentNodeBinding
|
||||
from models.agent import (
|
||||
Agent,
|
||||
AgentConfigSnapshot,
|
||||
AgentScope,
|
||||
AgentStatus,
|
||||
WorkflowAgentBindingType,
|
||||
WorkflowAgentNodeBinding,
|
||||
)
|
||||
from models.agent_config_entities import (
|
||||
AgentFileRefConfig,
|
||||
AgentHumanContactConfig,
|
||||
@@ -117,21 +125,28 @@ class WorkflowAgentNodeValidator:
|
||||
binding: WorkflowAgentNodeBinding,
|
||||
topology: _WorkflowGraphTopology | None = None,
|
||||
) -> None:
|
||||
"""Validate binding ownership, publication state, Agent Soul, and node-job references."""
|
||||
|
||||
if binding.agent_id is None:
|
||||
raise WorkflowAgentNodeValidationError(f"Workflow Agent node {binding.node_id} is missing agent binding.")
|
||||
|
||||
agent = session.scalar(
|
||||
select(Agent)
|
||||
.where(
|
||||
Agent.tenant_id == binding.tenant_id,
|
||||
Agent.id == binding.agent_id,
|
||||
)
|
||||
.limit(1)
|
||||
agent_stmt = select(Agent).where(
|
||||
Agent.tenant_id == binding.tenant_id,
|
||||
Agent.id == binding.agent_id,
|
||||
)
|
||||
if agent is None or agent.status == AgentStatus.ARCHIVED:
|
||||
raise WorkflowAgentNodeValidationError(
|
||||
f"Workflow Agent node {binding.node_id} references an unavailable agent."
|
||||
if binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT:
|
||||
agent_stmt = agent_stmt.where(
|
||||
Agent.scope == AgentScope.ROSTER,
|
||||
workflow_callable_active_snapshot_filter(),
|
||||
)
|
||||
agent = session.scalar(agent_stmt.limit(1))
|
||||
if agent is None or agent.status == AgentStatus.ARCHIVED:
|
||||
availability = (
|
||||
"an unavailable or unpublished roster agent"
|
||||
if binding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT
|
||||
else "an unavailable agent"
|
||||
)
|
||||
raise WorkflowAgentNodeValidationError(f"Workflow Agent node {binding.node_id} references {availability}.")
|
||||
|
||||
snapshot_id = (
|
||||
agent.active_config_snapshot_id
|
||||
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
|
||||
|
||||
The primary threshold aggregates statement and branch coverage for every Dify
|
||||
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
|
||||
that only contain narrow integration hooks are checked with changed-line
|
||||
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TypedDict, cast
|
||||
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
|
||||
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
|
||||
{
|
||||
"api/dev/check_knowledge_fs_coverage.py",
|
||||
"api/dev/generate_knowledge_fs_contract.py",
|
||||
"api/dev/knowledge_fs_product_contract.py",
|
||||
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
|
||||
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
|
||||
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
|
||||
"api/migrations/versions/2026_07_21_1500-d4f6e8a1c305_add_knowledge_fs_remote_freeze_evidence.py",
|
||||
"api/migrations/versions/2026_07_21_1600-e5a7c9b2d416_add_knowledge_fs_cleanup_completion.py",
|
||||
}
|
||||
)
|
||||
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoverageSummary(TypedDict):
|
||||
"""Coverage.py counts required by the aggregate gate."""
|
||||
|
||||
covered_lines: int
|
||||
num_statements: int
|
||||
covered_branches: int
|
||||
num_branches: int
|
||||
|
||||
|
||||
class CoverageFile(TypedDict):
|
||||
"""Per-file coverage data emitted by ``coverage json``."""
|
||||
|
||||
executed_lines: list[int]
|
||||
missing_lines: list[int]
|
||||
summary: CoverageSummary
|
||||
|
||||
|
||||
class CoverageReport(TypedDict):
|
||||
"""Relevant top-level shape of a coverage.py JSON report."""
|
||||
|
||||
files: dict[str, CoverageFile]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CoverageTotals:
|
||||
"""Covered and measurable units for one gate surface."""
|
||||
|
||||
covered: int
|
||||
total: int
|
||||
|
||||
@property
|
||||
def percent(self) -> float:
|
||||
return 100.0 if self.total == 0 else self.covered * 100 / self.total
|
||||
|
||||
|
||||
class CoverageGateError(RuntimeError):
|
||||
"""Raised when coverage input is incomplete or below its threshold."""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Validate focused module coverage and changed integration glue."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--coverage-json", type=Path, required=True)
|
||||
parser.add_argument("--glue-manifest", type=Path, required=True)
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--base", default="")
|
||||
parser.add_argument("--minimum", type=float, default=90.0)
|
||||
parser.add_argument("--glue-minimum", type=float, default=90.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
report = load_coverage_report(args.coverage_json)
|
||||
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
|
||||
base = resolve_diff_base(workspace_root, args.base)
|
||||
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
|
||||
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
|
||||
glue_totals = validate_changed_glue_coverage(
|
||||
report,
|
||||
changed_lines=changed_lines,
|
||||
minimum=args.glue_minimum,
|
||||
)
|
||||
logger.info(
|
||||
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
|
||||
core_totals.percent,
|
||||
core_totals.covered,
|
||||
core_totals.total,
|
||||
glue_totals.percent,
|
||||
glue_totals.covered,
|
||||
glue_totals.total,
|
||||
)
|
||||
|
||||
|
||||
def load_coverage_report(path: Path) -> CoverageReport:
|
||||
"""Load the detailed JSON report used by both coverage checks."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"coverage JSON does not exist: {path}")
|
||||
document = json.loads(path.read_text())
|
||||
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
|
||||
raise CoverageGateError(f"coverage JSON has no files object: {path}")
|
||||
return cast(CoverageReport, document)
|
||||
|
||||
|
||||
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
|
||||
try:
|
||||
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
|
||||
except UnicodeDecodeError as error:
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
|
||||
if len(paths) != len(set(paths)):
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
|
||||
invalid_paths = [
|
||||
candidate
|
||||
for candidate in paths
|
||||
if not candidate.startswith("api/")
|
||||
or not candidate.endswith(".py")
|
||||
or not (workspace_root / candidate).is_file()
|
||||
]
|
||||
if invalid_paths:
|
||||
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
|
||||
return paths
|
||||
|
||||
|
||||
def is_core_coverage_path(path: str) -> bool:
|
||||
"""Return whether a repository-relative path belongs to the focused aggregate."""
|
||||
if not path.endswith(".py"):
|
||||
return False
|
||||
if path in {
|
||||
"api/commands/knowledge_fs.py",
|
||||
"api/configs/extra/knowledge_fs_config.py",
|
||||
"api/extensions/ext_knowledge_fs_observability.py",
|
||||
"api/services/knowledge_fs_capability.py",
|
||||
}:
|
||||
return True
|
||||
if path.startswith(
|
||||
(
|
||||
"api/controllers/console/knowledge_fs/",
|
||||
"api/controllers/inner_api/knowledge_fs/",
|
||||
"api/controllers/service_api/knowledge_fs/",
|
||||
"api/core/tools/builtin_tool/providers/knowledge_fs/",
|
||||
"api/services/knowledge_fs/",
|
||||
)
|
||||
):
|
||||
return True
|
||||
filename = path.rsplit("/", maxsplit=1)[-1]
|
||||
return (
|
||||
path.startswith("api/models/")
|
||||
and filename.startswith("knowledge_fs")
|
||||
or path.startswith("api/repositories/")
|
||||
and "knowledge_fs" in filename
|
||||
or path.startswith("api/tasks/")
|
||||
and "knowledge_fs" in filename
|
||||
)
|
||||
|
||||
|
||||
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Classify every KnowledgeFS-named production file or fail closed."""
|
||||
named_paths = discover_knowledge_fs_production_paths(workspace_root)
|
||||
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
|
||||
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
|
||||
if unclassified_paths:
|
||||
raise CoverageGateError(
|
||||
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
|
||||
+ ", ".join(sorted(unclassified_paths))
|
||||
)
|
||||
if not core_paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
|
||||
return tuple(sorted(core_paths))
|
||||
|
||||
|
||||
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
|
||||
api_root = workspace_root / "api"
|
||||
if not api_root.is_dir():
|
||||
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
|
||||
paths: set[str] = set()
|
||||
for directory, child_directories, filenames in os.walk(api_root):
|
||||
current_directory = Path(directory)
|
||||
if current_directory == api_root:
|
||||
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
|
||||
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
|
||||
for filename in filenames:
|
||||
path = (current_directory / filename).relative_to(workspace_root).as_posix()
|
||||
if filename.endswith(".py") and "knowledge_fs" in path:
|
||||
paths.add(path)
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
|
||||
return tuple(sorted(paths))
|
||||
|
||||
|
||||
def validate_core_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
workspace_root: Path,
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require the exact combined line-and-branch percentage for all core files."""
|
||||
paths = discover_core_coverage_paths(workspace_root)
|
||||
missing_paths = [path for path in paths if path not in report["files"]]
|
||||
if missing_paths:
|
||||
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
|
||||
|
||||
covered = 0
|
||||
total = 0
|
||||
for path in paths:
|
||||
summary = report["files"][path]["summary"]
|
||||
covered += summary["covered_lines"] + summary["covered_branches"]
|
||||
total += summary["num_statements"] + summary["num_branches"]
|
||||
if total == 0:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
|
||||
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
|
||||
base = preferred.strip()
|
||||
if not base or set(base) == {"0"}:
|
||||
base = "HEAD^"
|
||||
result = subprocess.run(
|
||||
["git", "cat-file", "-e", f"{base}^{{commit}}"],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "commit is unavailable"
|
||||
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
|
||||
return base
|
||||
|
||||
|
||||
def collect_changed_glue_lines(
|
||||
workspace_root: Path,
|
||||
base: str,
|
||||
*,
|
||||
glue_paths: tuple[str, ...],
|
||||
) -> dict[str, set[int]]:
|
||||
"""Return added line numbers in the narrow Dify modules touched by this integration."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-color",
|
||||
"--unified=0",
|
||||
base,
|
||||
"--",
|
||||
*glue_paths,
|
||||
],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "git diff failed"
|
||||
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
|
||||
return parse_added_lines(result.stdout)
|
||||
|
||||
|
||||
def parse_added_lines(diff: str) -> dict[str, set[int]]:
|
||||
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
|
||||
changed_lines: dict[str, set[int]] = {}
|
||||
current_path: str | None = None
|
||||
current_line: int | None = None
|
||||
for raw_line in diff.splitlines():
|
||||
if raw_line.startswith("diff --git "):
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("+++ "):
|
||||
candidate = raw_line[4:]
|
||||
current_path = candidate[2:] if candidate.startswith("b/") else None
|
||||
if current_path is not None:
|
||||
changed_lines.setdefault(current_path, set())
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("@@ "):
|
||||
match = HUNK_HEADER.match(raw_line)
|
||||
current_line = int(match.group(1)) if match is not None else None
|
||||
continue
|
||||
if current_path is None or current_line is None:
|
||||
continue
|
||||
if raw_line.startswith("+"):
|
||||
changed_lines[current_path].add(current_line)
|
||||
current_line += 1
|
||||
elif raw_line.startswith("-") or raw_line.startswith("\\"):
|
||||
continue
|
||||
else:
|
||||
current_line += 1
|
||||
return changed_lines
|
||||
|
||||
|
||||
def validate_changed_glue_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
changed_lines: dict[str, set[int]],
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require added executable glue lines to be exercised by the focused unit suite."""
|
||||
covered = 0
|
||||
total = 0
|
||||
for path, lines in sorted(changed_lines.items()):
|
||||
if not lines:
|
||||
continue
|
||||
file_coverage = report["files"].get(path)
|
||||
if file_coverage is None:
|
||||
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
|
||||
executed_lines = set(file_coverage["executed_lines"])
|
||||
executable_lines = executed_lines | set(file_coverage["missing_lines"])
|
||||
changed_executable_lines = lines & executable_lines
|
||||
covered += len(changed_executable_lines & executed_lines)
|
||||
total += len(changed_executable_lines)
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
|
||||
if totals.percent + 1e-12 < minimum:
|
||||
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,14 +1,7 @@
|
||||
"""Pin the in-repository KnowledgeFS contract and validate every Dify product operation.
|
||||
"""Validate Dify Console KnowledgeFS declarations against a pinned OpenAPI document.
|
||||
|
||||
The lock is intentionally independent of the enclosing Dify commit: it records the staged ``knowledge-fs/`` tree,
|
||||
the complete generated OpenAPI document, both explicit product-operation manifests, and the active Capability v2
|
||||
profile and deterministic public-key vector. The full OpenAPI hash
|
||||
covers request/response schemas, status codes, security, deprecation, and stream metadata. Field-level validation
|
||||
cross-checks the Dify product registry, Python Capability issuer, TypeScript request guard, and exported OpenAPI;
|
||||
each product operation must be ready or an explicit gap, and KFS-only activation remains explicitly internal.
|
||||
|
||||
Contract export reads the working tree only after proving it matches the staged KnowledgeFS index. This keeps the
|
||||
OpenAPI bytes and auth manifest aligned with the exact subtree tree ID that will be reviewed and committed.
|
||||
The OpenAPI document is exported only during explicit development validation. Runtime declarations live with Dify
|
||||
product policy; this module validates their transport metadata without generating a complete operation catalog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,37 +12,29 @@ import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict, cast
|
||||
|
||||
import jwt
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
|
||||
from jwt.algorithms import RSAAlgorithm
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from dev.knowledge_fs_product_contract import (
|
||||
capability_operation_runtime_contracts,
|
||||
parse_capability_operation_policy,
|
||||
parse_product_operation_gap_manifest,
|
||||
parse_product_operation_manifest,
|
||||
product_operation_runtime_contracts,
|
||||
validate_product_operation_contracts,
|
||||
)
|
||||
|
||||
WORKSPACE_ROOT = API_ROOT.parent
|
||||
KNOWLEDGE_FS_DIRECTORY = "knowledge-fs"
|
||||
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
|
||||
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
|
||||
UPSTREAM_PROVENANCE_RELATIVE_PATH = Path("upstream-provenance.json")
|
||||
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
|
||||
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
|
||||
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
|
||||
LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
|
||||
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
LOCK_SCHEMA_VERSION = 5
|
||||
CONSOLE_PROXY_ERROR_SCHEMA_NAME = "ConsoleProxyError"
|
||||
CONSOLE_PROXY_ERROR_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["code", "message", "status"],
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"status": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
@@ -64,18 +49,7 @@ class ContractDeclaration(TypedDict):
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
class ContractLock(TypedDict):
|
||||
"""Content-addressed contract inputs that must move together."""
|
||||
|
||||
schemaVersion: int
|
||||
subtreeTree: str
|
||||
openapiSha256: str
|
||||
capabilityV2AuthManifestSha256: str
|
||||
capabilityV2AuthTestVectorSha256: str
|
||||
productOperationManifestSha256: str
|
||||
productOperationGapManifestSha256: str
|
||||
error_status_map: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
@@ -102,507 +76,68 @@ DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Update or verify the monorepo pin and validate Dify product declarations."""
|
||||
"""Update or verify the pin and validate Console declarations against its OpenAPI document."""
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--update-lock", action="store_true")
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--repository", type=Path, default=DEFAULT_REPOSITORY)
|
||||
parser.add_argument("--output-openapi", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
knowledge_fs_root = workspace_root / KNOWLEDGE_FS_DIRECTORY
|
||||
lock_path = workspace_root / LOCK_RELATIVE_PATH
|
||||
capability_v2_auth_manifest_path = knowledge_fs_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH
|
||||
capability_v2_auth_test_vector_path = knowledge_fs_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH
|
||||
product_operations_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
|
||||
product_operation_gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
|
||||
upstream_provenance_path = knowledge_fs_root / UPSTREAM_PROVENANCE_RELATIVE_PATH
|
||||
ensure_clean_knowledge_fs_worktree(workspace_root)
|
||||
ensure_contract_inputs_exist(
|
||||
knowledge_fs_root=knowledge_fs_root,
|
||||
capability_v2_auth_manifest_path=capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path=capability_v2_auth_test_vector_path,
|
||||
product_operations_path=product_operations_path,
|
||||
product_operation_gaps_path=product_operation_gaps_path,
|
||||
upstream_provenance_path=upstream_provenance_path,
|
||||
)
|
||||
repository = args.repository.resolve()
|
||||
lock = json.loads(LOCK_PATH.read_text())
|
||||
tracked_changes = run("git", "status", "--porcelain", "--untracked-files=no", cwd=repository).strip()
|
||||
if tracked_changes:
|
||||
raise RuntimeError("KnowledgeFS checkout must not contain tracked changes during contract export")
|
||||
|
||||
subtree_tree = staged_subtree_tree(workspace_root)
|
||||
capability_v2_auth_manifest_content = capability_v2_auth_manifest_path.read_bytes()
|
||||
capability_v2_auth_test_vector_content = capability_v2_auth_test_vector_path.read_bytes()
|
||||
product_operation_manifest_content = product_operations_path.read_bytes()
|
||||
product_operation_gap_manifest_content = product_operation_gaps_path.read_bytes()
|
||||
capability_v2_auth_manifest = load_json_object(capability_v2_auth_manifest_path)
|
||||
validate_capability_v2_auth_manifest(capability_v2_auth_manifest)
|
||||
validate_capability_v2_auth_test_vector(
|
||||
load_json_object(capability_v2_auth_test_vector_path),
|
||||
capability_v2_auth_manifest,
|
||||
)
|
||||
validate_upstream_provenance(load_json_object(upstream_provenance_path))
|
||||
product_manifest = parse_product_operation_manifest(load_json_object(product_operations_path))
|
||||
product_gap_manifest = parse_product_operation_gap_manifest(load_json_object(product_operation_gaps_path))
|
||||
product_runtime_operations = product_operation_runtime_contracts()
|
||||
capability_runtime_operations = capability_operation_runtime_contracts()
|
||||
declarations = console_contract_declarations()
|
||||
commit = run("git", "rev-parse", "HEAD", cwd=repository).strip()
|
||||
if not args.update_lock and commit != lock["commit"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS checkout mismatch: expected {lock['commit']}, received {commit}. "
|
||||
"Use the pinned commit or pass --update-lock intentionally."
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
|
||||
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
|
||||
capability_policy_path = Path(directory) / "dify-capability-v2-operations.json"
|
||||
subprocess.run(
|
||||
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
|
||||
cwd=knowledge_fs_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["pnpm", "capability:export", "--", "--output", str(capability_policy_path)],
|
||||
cwd=knowledge_fs_root,
|
||||
cwd=repository,
|
||||
check=True,
|
||||
)
|
||||
openapi_content = openapi_path.read_bytes()
|
||||
capability_policy = parse_capability_operation_policy(load_json_object(capability_policy_path))
|
||||
|
||||
openapi_sha256 = sha256(openapi_content)
|
||||
if not args.update_lock and openapi_sha256 != lock["openapiSha256"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS OpenAPI hash mismatch: expected {lock['openapiSha256']}, received {openapi_sha256}"
|
||||
)
|
||||
|
||||
document: dict[str, Any] = json.loads(openapi_content)
|
||||
validate_product_operation_contracts(
|
||||
capability_operations=capability_runtime_operations,
|
||||
capability_policy=capability_policy,
|
||||
document=document,
|
||||
gap_manifest=product_gap_manifest,
|
||||
manifest=product_manifest,
|
||||
product_operations=product_runtime_operations,
|
||||
)
|
||||
declarations = console_contract_declarations()
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
expected_lock: ContractLock = {
|
||||
"schemaVersion": LOCK_SCHEMA_VERSION,
|
||||
"subtreeTree": subtree_tree,
|
||||
"openapiSha256": sha256(openapi_content),
|
||||
"capabilityV2AuthManifestSha256": sha256(capability_v2_auth_manifest_content),
|
||||
"capabilityV2AuthTestVectorSha256": sha256(capability_v2_auth_test_vector_content),
|
||||
"productOperationManifestSha256": sha256(product_operation_manifest_content),
|
||||
"productOperationGapManifestSha256": sha256(product_operation_gap_manifest_content),
|
||||
}
|
||||
if args.output_openapi:
|
||||
filtered_document = filter_openapi_document(document, declarations)
|
||||
filtered_document["x-dify-source-openapi-sha256"] = openapi_sha256
|
||||
filtered_document["x-dify-console-declarations-sha256"] = contract_declarations_sha256(declarations)
|
||||
args.output_openapi.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_openapi.write_text(json.dumps(filtered_document, indent=2) + "\n")
|
||||
|
||||
if args.update_lock:
|
||||
lock_path.write_text(json.dumps(expected_lock, indent=2) + "\n")
|
||||
return
|
||||
|
||||
received_lock = parse_contract_lock(load_json_object(lock_path))
|
||||
lock_fields = (
|
||||
(
|
||||
"capabilityV2AuthManifestSha256",
|
||||
received_lock["capabilityV2AuthManifestSha256"],
|
||||
expected_lock["capabilityV2AuthManifestSha256"],
|
||||
),
|
||||
(
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
received_lock["capabilityV2AuthTestVectorSha256"],
|
||||
expected_lock["capabilityV2AuthTestVectorSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationManifestSha256",
|
||||
received_lock["productOperationManifestSha256"],
|
||||
expected_lock["productOperationManifestSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationGapManifestSha256",
|
||||
received_lock["productOperationGapManifestSha256"],
|
||||
expected_lock["productOperationGapManifestSha256"],
|
||||
),
|
||||
("schemaVersion", received_lock["schemaVersion"], expected_lock["schemaVersion"]),
|
||||
("subtreeTree", received_lock["subtreeTree"], expected_lock["subtreeTree"]),
|
||||
("openapiSha256", received_lock["openapiSha256"], expected_lock["openapiSha256"]),
|
||||
)
|
||||
for field, received_value, expected_value in lock_fields:
|
||||
if received_value != expected_value:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS contract lock field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}. "
|
||||
"Run --update-lock intentionally after reviewing the staged subtree and contract changes."
|
||||
LOCK_PATH.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"commit": commit,
|
||||
"openapiSha256": openapi_sha256,
|
||||
"repository": lock["repository"],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
|
||||
def ensure_contract_inputs_exist(
|
||||
*,
|
||||
knowledge_fs_root: Path,
|
||||
capability_v2_auth_manifest_path: Path,
|
||||
capability_v2_auth_test_vector_path: Path,
|
||||
product_operations_path: Path,
|
||||
product_operation_gaps_path: Path,
|
||||
upstream_provenance_path: Path,
|
||||
) -> None:
|
||||
"""Fail with a stable error before invoking package tooling when a contract input is absent."""
|
||||
required_paths = (
|
||||
knowledge_fs_root / "package.json",
|
||||
capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path,
|
||||
product_operations_path,
|
||||
product_operation_gaps_path,
|
||||
upstream_provenance_path,
|
||||
)
|
||||
missing_paths = [path for path in required_paths if not path.is_file()]
|
||||
if missing_paths:
|
||||
missing = ", ".join(str(path) for path in missing_paths)
|
||||
raise RuntimeError(f"KnowledgeFS contract input is missing: {missing}")
|
||||
|
||||
|
||||
def ensure_clean_knowledge_fs_worktree(workspace_root: Path) -> None:
|
||||
"""Require exported KnowledgeFS files to exactly match the staged index tree.
|
||||
|
||||
Staged changes are expected during intentional lock updates. Unstaged tracked changes and untracked files are
|
||||
rejected because the export process reads the working tree while the tree ID is calculated from the index.
|
||||
"""
|
||||
subtree_path = f"{KNOWLEDGE_FS_DIRECTORY}/"
|
||||
unstaged = subprocess.run(
|
||||
["git", "diff", "--quiet", "--", subtree_path],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
)
|
||||
if unstaged.returncode > 1:
|
||||
raise RuntimeError("git diff failed while validating the staged KnowledgeFS subtree")
|
||||
untracked = run(
|
||||
"git",
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"--",
|
||||
subtree_path,
|
||||
cwd=workspace_root,
|
||||
).strip()
|
||||
if unstaged.returncode != 0 or untracked:
|
||||
raise RuntimeError(
|
||||
"knowledge-fs/ contains unstaged or untracked changes; stage or remove them before contract export"
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def staged_subtree_tree(workspace_root: Path) -> str:
|
||||
"""Return the Git tree object for the staged ``knowledge-fs/`` subtree."""
|
||||
return run("git", "write-tree", f"--prefix={KNOWLEDGE_FS_DIRECTORY}/", cwd=workspace_root).strip()
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
|
||||
return cast(dict[str, Any], value)
|
||||
|
||||
|
||||
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
|
||||
"""Validate the compact, non-self-referential contract lock schema."""
|
||||
expected_fields = {
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"schemaVersion",
|
||||
"subtreeTree",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
|
||||
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
|
||||
for field in (
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"subtreeTree",
|
||||
):
|
||||
field_value = value.get(field)
|
||||
expected_length = 40 if field == "subtreeTree" else 64
|
||||
if (
|
||||
not isinstance(field_value, str)
|
||||
or len(field_value) != expected_length
|
||||
or any(character not in "0123456789abcdef" for character in field_value)
|
||||
):
|
||||
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
|
||||
return cast(ContractLock, value)
|
||||
|
||||
|
||||
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
|
||||
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
|
||||
available_operation_ids = [
|
||||
operation_id
|
||||
for path_item in document.get("paths", {}).values()
|
||||
for method in OPENAPI_METHODS
|
||||
if isinstance(path_item, dict)
|
||||
for operation in (path_item.get(method),)
|
||||
if isinstance(operation, dict)
|
||||
for operation_id in (operation.get("operationId"),)
|
||||
if isinstance(operation_id, str) and operation_id
|
||||
]
|
||||
for operation_id in required_operation_ids:
|
||||
count = available_operation_ids.count(operation_id)
|
||||
if count != 1:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
|
||||
)
|
||||
|
||||
|
||||
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
|
||||
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
|
||||
expected_fields = {
|
||||
"active",
|
||||
"audience",
|
||||
"callerProfiles",
|
||||
"claimBindings",
|
||||
"issuer",
|
||||
"lifecycle",
|
||||
"maxTtlSeconds",
|
||||
"productionReady",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"requiredClaims",
|
||||
"resourceContract",
|
||||
"runtimeAssembly",
|
||||
"schemaVersion",
|
||||
"signatureAlgorithms",
|
||||
"tokenKind",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
|
||||
fixed_values = {
|
||||
"active": True,
|
||||
"audience": "knowledge-fs",
|
||||
"issuer": "dify-control-plane",
|
||||
"lifecycle": "active",
|
||||
"maxTtlSeconds": 60,
|
||||
"productionReady": True,
|
||||
"profileId": "dify-capability-v2",
|
||||
"schemaVersion": 3,
|
||||
"signatureAlgorithms": ["RS256"],
|
||||
"tokenKind": "jwt",
|
||||
}
|
||||
for field, expected in fixed_values.items():
|
||||
if value.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
|
||||
if value.get("protectedHeader") != {
|
||||
"algorithm": "RS256",
|
||||
"keyIdClaim": "kid",
|
||||
"keyIdRequired": True,
|
||||
"type": "JWT",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
|
||||
required_claims = [
|
||||
"action",
|
||||
"actor",
|
||||
"aud",
|
||||
"authz_revision",
|
||||
"azp",
|
||||
"caller_kind",
|
||||
"cap_ver",
|
||||
"content_policy_revision",
|
||||
"content_scope_ids",
|
||||
"control_space_id",
|
||||
"exp",
|
||||
"grant_id",
|
||||
"iat",
|
||||
"iss",
|
||||
"jti",
|
||||
"namespace_id",
|
||||
"nbf",
|
||||
"resource",
|
||||
"sub",
|
||||
"trace_id",
|
||||
]
|
||||
if value.get("requiredClaims") != required_claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
|
||||
if value.get("claimBindings") != {
|
||||
"action": "action",
|
||||
"callerKind": "caller_kind",
|
||||
"controlSpace": "control_space_id",
|
||||
"namespace": "namespace_id",
|
||||
"resource": "resource",
|
||||
"resourceParent": "resource.parent_id",
|
||||
"subject": "sub",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
|
||||
if value.get("resourceContract") != {
|
||||
"fields": ["id", "parent_id", "type"],
|
||||
"parentForbiddenFor": ["namespace", "knowledge_space"],
|
||||
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
|
||||
if value.get("callerProfiles") != {
|
||||
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
|
||||
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
|
||||
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
|
||||
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
|
||||
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-kfs-credential:"},
|
||||
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
|
||||
if value.get("runtimeAssembly") != {
|
||||
"failClosed": True,
|
||||
"keySelection": "kid",
|
||||
"maximumPublishedKeys": 3,
|
||||
"verificationKeySource": "jwks",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
|
||||
|
||||
|
||||
def validate_capability_v2_auth_test_vector(
|
||||
value: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Verify the deterministic public-key vector and every security-sensitive binding."""
|
||||
expected_fields = {
|
||||
"algorithm",
|
||||
"audience",
|
||||
"expectedClaims",
|
||||
"expectedPrincipal",
|
||||
"issuer",
|
||||
"operation",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"publicJwk",
|
||||
"schemaVersion",
|
||||
"testOnly",
|
||||
"token",
|
||||
"ttlSeconds",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
|
||||
if (
|
||||
value.get("schemaVersion") != 2
|
||||
or value.get("profileId") != manifest.get("profileId")
|
||||
or value.get("testOnly") is not True
|
||||
or value.get("algorithm") != "RS256"
|
||||
or value.get("issuer") != manifest.get("issuer")
|
||||
or value.get("audience") != manifest.get("audience")
|
||||
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
|
||||
protected_header = value.get("protectedHeader")
|
||||
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
kid = protected_header.get("kid")
|
||||
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
public_jwk = value.get("publicJwk")
|
||||
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
if (
|
||||
public_jwk.get("alg") != "RS256"
|
||||
or public_jwk.get("kid") != kid
|
||||
or public_jwk.get("kty") != "RSA"
|
||||
or public_jwk.get("use") != "sig"
|
||||
or not _is_non_blank(public_jwk.get("e"))
|
||||
or not _is_non_blank(public_jwk.get("n"))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
|
||||
claims = value.get("expectedClaims")
|
||||
required_claims = manifest.get("requiredClaims")
|
||||
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
|
||||
operation = value.get("operation")
|
||||
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
|
||||
resource = claims.get("resource")
|
||||
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
|
||||
expected_operation = {
|
||||
"action": "documents.read",
|
||||
"method": "GET",
|
||||
"operationId": "getDocument",
|
||||
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
|
||||
}
|
||||
if operation != expected_operation:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
|
||||
exact_claims = {
|
||||
"action": operation["action"],
|
||||
"aud": value["audience"],
|
||||
"caller_kind": "interactive",
|
||||
"cap_ver": 2,
|
||||
"control_space_id": "control-space-contract-vector",
|
||||
"iss": value["issuer"],
|
||||
"namespace_id": "workspace-contract-vector",
|
||||
"resource": {
|
||||
"id": "document-contract-vector",
|
||||
"parent_id": "space-contract-vector",
|
||||
"type": "document",
|
||||
},
|
||||
"sub": "dify-account:account-contract-vector",
|
||||
}
|
||||
for field, expected in exact_claims.items():
|
||||
if claims.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
|
||||
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
|
||||
issued_at = claims.get("iat")
|
||||
not_before = claims.get("nbf")
|
||||
expires_at = claims.get("exp")
|
||||
if (
|
||||
not isinstance(issued_at, int)
|
||||
or isinstance(issued_at, bool)
|
||||
or not_before != issued_at
|
||||
or not isinstance(expires_at, int)
|
||||
or isinstance(expires_at, bool)
|
||||
or expires_at - issued_at != value["ttlSeconds"]
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
|
||||
expected_principal = {
|
||||
"callerKind": claims["caller_kind"],
|
||||
"subject": {
|
||||
"scopes": ["knowledge-spaces:read"],
|
||||
"subjectId": claims["sub"],
|
||||
"tenantId": claims["namespace_id"],
|
||||
},
|
||||
}
|
||||
if value.get("expectedPrincipal") != expected_principal:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
|
||||
|
||||
token = value.get("token")
|
||||
if not isinstance(token, str) or not _is_non_blank(token):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
|
||||
try:
|
||||
verification_key = RSAAlgorithm.from_jwk(public_jwk)
|
||||
if not isinstance(verification_key, RSAPublicKey):
|
||||
raise ValueError("Capability vector verification key is not RSA public material")
|
||||
header = jwt.get_unverified_header(token)
|
||||
decoded_claims = jwt.decode(
|
||||
token,
|
||||
verification_key,
|
||||
algorithms=["RS256"],
|
||||
audience=cast(str, value["audience"]),
|
||||
issuer=cast(str, value["issuer"]),
|
||||
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
|
||||
)
|
||||
except (jwt.PyJWTError, TypeError, ValueError) as exc:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
|
||||
if header != protected_header or decoded_claims != claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
|
||||
|
||||
|
||||
def _is_non_blank(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
|
||||
|
||||
|
||||
def validate_upstream_provenance(value: dict[str, Any]) -> None:
|
||||
"""Validate the imported-source provenance that is itself covered by the subtree tree ID."""
|
||||
expected_fields = {"commit", "release", "repository", "schemaVersion"}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS upstream provenance fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS upstream provenance must use schemaVersion 1")
|
||||
repository = value.get("repository")
|
||||
commit = value.get("commit")
|
||||
if not isinstance(repository, str) or not repository.startswith("https://"):
|
||||
raise ValueError("KnowledgeFS upstream provenance repository must be an HTTPS URL")
|
||||
if (
|
||||
not isinstance(commit, str)
|
||||
or len(commit) != 40
|
||||
or any(character not in "0123456789abcdef" for character in commit)
|
||||
):
|
||||
raise ValueError("KnowledgeFS upstream provenance commit must be a lowercase full Git SHA")
|
||||
if value.get("release") is not None and not isinstance(value["release"], str):
|
||||
raise ValueError("KnowledgeFS upstream provenance release must be null or a string")
|
||||
|
||||
|
||||
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
|
||||
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
|
||||
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
|
||||
@@ -633,11 +168,10 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: ContractDeclaration = {
|
||||
"operation_id": operation_id,
|
||||
expected: dict[DeclarationField, object] = {
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(document, operation),
|
||||
"required_scope": required_scope(operation),
|
||||
"response_kind": response_kind(operation),
|
||||
"max_response_bytes": required_max_response_bytes(operation),
|
||||
"request_headers": request_header_names(path_item, operation),
|
||||
@@ -652,12 +186,136 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
validate_error_status_map(operation_id, declaration["error_status_map"])
|
||||
|
||||
|
||||
def filter_openapi_document(
|
||||
document: dict[str, Any],
|
||||
declarations: tuple[ContractDeclaration, ...],
|
||||
) -> dict[str, Any]:
|
||||
"""Return a code-generation document containing only Console-allowlisted operations."""
|
||||
filtered_document: dict[str, Any] = {
|
||||
key: value for key, value in document.items() if key not in {"components", "paths"}
|
||||
}
|
||||
source_paths = document.get("paths", {})
|
||||
filtered_paths: dict[str, Any] = {}
|
||||
|
||||
for declaration in declarations:
|
||||
path = f"/{declaration['path']}"
|
||||
method = declaration["method"].lower()
|
||||
source_path_item = source_paths[path]
|
||||
path_metadata = {key: value for key, value in source_path_item.items() if key not in OPENAPI_METHODS}
|
||||
filtered_path_item = filtered_paths.setdefault(path, path_metadata)
|
||||
filtered_operation = deepcopy(source_path_item[method])
|
||||
_rewrite_proxy_error_responses(filtered_operation, declaration["error_status_map"])
|
||||
filtered_path_item[method] = filtered_operation
|
||||
|
||||
filtered_document["paths"] = filtered_paths
|
||||
|
||||
source_components = document.get("components", {})
|
||||
filtered_components = {key: value for key, value in source_components.items() if key != "schemas"}
|
||||
source_schemas = source_components.get("schemas", {})
|
||||
available_schemas = {**source_schemas, CONSOLE_PROXY_ERROR_SCHEMA_NAME: CONSOLE_PROXY_ERROR_SCHEMA}
|
||||
schema_names = _referenced_schema_names(filtered_paths, available_schemas)
|
||||
filtered_components["schemas"] = {
|
||||
name: schema for name, schema in available_schemas.items() if name in schema_names
|
||||
}
|
||||
filtered_document["components"] = filtered_components
|
||||
return filtered_document
|
||||
|
||||
|
||||
def validate_error_status_map(operation_id: str, error_status_map: tuple[tuple[int, int], ...]) -> None:
|
||||
"""Validate the status normalization advertised by one Console operation."""
|
||||
upstream_statuses: set[int] = set()
|
||||
for upstream_status, console_status in error_status_map:
|
||||
if upstream_status in upstream_statuses:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has duplicate error status: {upstream_status}")
|
||||
if not 400 <= upstream_status <= 599 or not 400 <= console_status <= 599:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has invalid error status mapping")
|
||||
upstream_statuses.add(upstream_status)
|
||||
|
||||
|
||||
def _rewrite_proxy_error_responses(
|
||||
operation: dict[str, Any],
|
||||
error_status_map: tuple[tuple[int, int], ...],
|
||||
) -> None:
|
||||
responses = operation.setdefault("responses", {})
|
||||
proxy_error_response = {
|
||||
"description": "Error normalized by the Dify Console KnowledgeFS proxy.",
|
||||
"content": {
|
||||
"application/json": {"schema": {"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"}}
|
||||
},
|
||||
}
|
||||
for upstream_status, console_status in error_status_map:
|
||||
existing_target = responses.get(str(console_status)) if upstream_status != console_status else None
|
||||
responses.pop(str(upstream_status), None)
|
||||
normalized_response: dict[str, Any] = deepcopy(proxy_error_response)
|
||||
existing_schema = (
|
||||
existing_target.get("content", {}).get("application/json", {}).get("schema")
|
||||
if isinstance(existing_target, dict)
|
||||
else None
|
||||
)
|
||||
if existing_schema is not None:
|
||||
normalized_response["content"]["application/json"]["schema"] = {
|
||||
"oneOf": [
|
||||
deepcopy(existing_schema),
|
||||
{"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"},
|
||||
]
|
||||
}
|
||||
responses[str(console_status)] = normalized_response
|
||||
|
||||
|
||||
def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
reference_prefix = "#/components/schemas/"
|
||||
selected: set[str] = set()
|
||||
pending: list[Any] = [value]
|
||||
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if isinstance(current, list):
|
||||
pending.extend(current)
|
||||
continue
|
||||
if not isinstance(current, dict):
|
||||
continue
|
||||
|
||||
reference = current.get("$ref")
|
||||
if isinstance(reference, str) and reference.startswith(reference_prefix):
|
||||
name = reference.removeprefix(reference_prefix)
|
||||
if name not in selected:
|
||||
if name not in schemas:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI references missing schema: {name}")
|
||||
selected.add(name)
|
||||
pending.append(schemas[name])
|
||||
pending.extend(current.values())
|
||||
|
||||
return selected
|
||||
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
|
||||
"""Return transport declarations from the runtime Console operation registry."""
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
|
||||
return ()
|
||||
return tuple(
|
||||
{
|
||||
"operation_id": operation.operation_id,
|
||||
"method": operation.method,
|
||||
"path": operation.path,
|
||||
"required_scope": operation.required_scope,
|
||||
"response_kind": operation.response_kind,
|
||||
"max_response_bytes": operation.max_response_bytes,
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
)
|
||||
|
||||
|
||||
def contract_declarations_sha256(declarations: tuple[ContractDeclaration, ...]) -> str:
|
||||
"""Return a stable digest for the runtime Console operation declarations."""
|
||||
content = json.dumps(declarations, separators=(",", ":"), sort_keys=True).encode()
|
||||
return sha256(content)
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
@@ -677,17 +335,12 @@ def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(sorted(media_types))
|
||||
|
||||
|
||||
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
|
||||
def required_scope(operation: dict[str, Any]) -> str | None:
|
||||
scope = operation.get("x-knowledge-fs-required-scope")
|
||||
security = operation["security"] if "security" in operation else document.get("security")
|
||||
if security == []:
|
||||
if scope is not None:
|
||||
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
|
||||
return None
|
||||
if security != [{"bearerAuth": []}]:
|
||||
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
|
||||
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
|
||||
return scope
|
||||
if operation.get("security") == []:
|
||||
return None
|
||||
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
|
||||
|
||||
|
||||
|
||||
@@ -1,494 +0,0 @@
|
||||
"""Cross-service KnowledgeFS product-operation contract validation.
|
||||
|
||||
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
|
||||
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
|
||||
policy remain executable sources of truth. A product operation must appear exactly once as ready
|
||||
or as an explicit gap; internal KFS-only operations require a named exclusion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, Literal, NamedTuple, TypedDict, cast
|
||||
|
||||
|
||||
class ProductOperationLimits(TypedDict):
|
||||
kfsMaxResponseBytes: int
|
||||
productMaxRequestBytes: int
|
||||
productMaxResponseBytes: int
|
||||
|
||||
|
||||
class ProductOperationStream(TypedDict):
|
||||
kfsResponseKind: str
|
||||
productKind: str
|
||||
|
||||
|
||||
class ProductOperationManifestEntry(TypedDict):
|
||||
action: str
|
||||
kfsOperationId: str
|
||||
limits: ProductOperationLimits
|
||||
method: str
|
||||
path: str
|
||||
productOperationId: str
|
||||
resource: str
|
||||
stream: ProductOperationStream
|
||||
transport: str
|
||||
|
||||
|
||||
type ProductOperationManifestField = Literal[
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
]
|
||||
|
||||
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
)
|
||||
|
||||
|
||||
class ProductOperationManifest(TypedDict):
|
||||
operations: list[ProductOperationManifestEntry]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationGapEntry(ProductOperationManifestEntry):
|
||||
reason: str
|
||||
reasonCode: str
|
||||
replacementProductOperationIds: list[str]
|
||||
|
||||
|
||||
class InternalKfsOperationExclusion(TypedDict):
|
||||
kfsOperationId: str
|
||||
reason: str
|
||||
reasonCode: str
|
||||
|
||||
|
||||
class ProductOperationGapManifest(TypedDict):
|
||||
gaps: list[ProductOperationGapEntry]
|
||||
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
kfs_operation_id: str
|
||||
max_request_bytes: int
|
||||
max_response_bytes: int
|
||||
method: str
|
||||
path: str
|
||||
product_operation_id: str
|
||||
ready: bool
|
||||
resource: str
|
||||
stream_kind: str
|
||||
transport: str
|
||||
|
||||
|
||||
class CapabilityOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
path: str
|
||||
resource: str
|
||||
|
||||
|
||||
class CapabilityOperationPolicy(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
parent_resource_binding: dict[str, object] | None
|
||||
path: str
|
||||
resource: str
|
||||
resource_binding: dict[str, object]
|
||||
|
||||
|
||||
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
|
||||
"""Read the complete typed Dify product registry without normalizing away gaps."""
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
|
||||
contracts: list[ProductOperationRuntimeContract] = []
|
||||
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
|
||||
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
|
||||
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
|
||||
contracts.append(
|
||||
ProductOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
kfs_operation_id=operation.capability_operation_id,
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
product_operation_id=product_operation_id,
|
||||
ready=is_product_operation_ready(product_operation_id),
|
||||
resource=operation.resource_resolver,
|
||||
stream_kind=operation.stream_kind,
|
||||
transport=operation.transport,
|
||||
)
|
||||
)
|
||||
return tuple(contracts)
|
||||
|
||||
|
||||
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
|
||||
"""Read the Python issuer policy used to mint each operation Capability."""
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
CapabilityOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
allowed_caller_kinds=operation.allowed_caller_kinds,
|
||||
method=operation.method,
|
||||
operation_id=operation_id,
|
||||
path=operation.path,
|
||||
resource=operation.resource_type,
|
||||
)
|
||||
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
|
||||
)
|
||||
|
||||
|
||||
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
|
||||
"""Parse the ready product-operation inventory with a closed schema."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
|
||||
return {
|
||||
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
|
||||
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
|
||||
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
|
||||
if set(value) != expected_fields or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
|
||||
gaps = value.get("gaps")
|
||||
exclusions = value.get("internalKfsOperationExclusions")
|
||||
if not isinstance(gaps, list) or not isinstance(exclusions, list):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
|
||||
parsed_exclusions: list[InternalKfsOperationExclusion] = []
|
||||
for raw in exclusions:
|
||||
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
|
||||
for field in ("kfsOperationId", "reason", "reasonCode"):
|
||||
_required_string(raw.get(field), f"internal exclusion {field}")
|
||||
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
|
||||
return {
|
||||
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
|
||||
"internalKfsOperationExclusions": parsed_exclusions,
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
|
||||
"""Parse the runtime-exported TypeScript request-guard registry."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
|
||||
parsed: list[CapabilityOperationPolicy] = []
|
||||
for raw in operations:
|
||||
expected_fields = {
|
||||
"action",
|
||||
"allowedCallerKinds",
|
||||
"method",
|
||||
"operationId",
|
||||
"parentResourceBinding",
|
||||
"path",
|
||||
"resourceBinding",
|
||||
"resourceType",
|
||||
}
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
|
||||
callers = raw.get("allowedCallerKinds")
|
||||
resource_binding = raw.get("resourceBinding")
|
||||
parent_binding = raw.get("parentResourceBinding")
|
||||
if (
|
||||
not isinstance(callers, list)
|
||||
or not callers
|
||||
or not all(isinstance(caller, str) and caller for caller in callers)
|
||||
or not isinstance(resource_binding, dict)
|
||||
or not resource_binding
|
||||
or (parent_binding is not None and not isinstance(parent_binding, dict))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
|
||||
for field in ("action", "method", "operationId", "path", "resourceType"):
|
||||
_required_string(raw.get(field), f"Capability operation {field}")
|
||||
parsed.append(
|
||||
CapabilityOperationPolicy(
|
||||
action=cast(str, raw["action"]),
|
||||
allowed_caller_kinds=tuple(callers),
|
||||
method=cast(str, raw["method"]),
|
||||
operation_id=cast(str, raw["operationId"]),
|
||||
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
|
||||
path=cast(str, raw["path"]),
|
||||
resource=cast(str, raw["resourceType"]),
|
||||
resource_binding=cast(dict[str, object], resource_binding),
|
||||
)
|
||||
)
|
||||
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def validate_product_operation_contracts(
|
||||
*,
|
||||
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
|
||||
capability_policy: tuple[CapabilityOperationPolicy, ...],
|
||||
document: dict[str, Any],
|
||||
gap_manifest: ProductOperationGapManifest,
|
||||
manifest: ProductOperationManifest,
|
||||
product_operations: tuple[ProductOperationRuntimeContract, ...],
|
||||
) -> None:
|
||||
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
|
||||
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
|
||||
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
|
||||
if len(ready_entries) != len(manifest["operations"]):
|
||||
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
|
||||
if len(gap_entries) != len(gap_manifest["gaps"]):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
|
||||
if set(ready_entries) & set(gap_entries):
|
||||
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
|
||||
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
|
||||
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
|
||||
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
|
||||
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
|
||||
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
|
||||
|
||||
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
|
||||
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
|
||||
openapi_operations = _openapi_operations(document)
|
||||
for product_operation_id, runtime in runtime_by_product.items():
|
||||
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
|
||||
if runtime.ready != (product_operation_id in ready_entries):
|
||||
state = "ready" if runtime.ready else "gap"
|
||||
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
|
||||
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
|
||||
if product_operation_id in gap_entries:
|
||||
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
|
||||
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
|
||||
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
|
||||
if not replacements or any(replacement not in ready_entries for replacement in replacements):
|
||||
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
|
||||
|
||||
exclusions = gap_manifest["internalKfsOperationExclusions"]
|
||||
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
|
||||
if len(exclusion_by_id) != len(exclusions):
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
|
||||
lifecycle_operation_ids = {
|
||||
"activateDifyWorkspaceIntegration",
|
||||
"freezeDifyWorkspaceIntegration",
|
||||
}
|
||||
if set(exclusion_by_id) != lifecycle_operation_ids:
|
||||
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
|
||||
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
|
||||
for operation_id in exclusion_by_id:
|
||||
if operation_id in product_capability_ids:
|
||||
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
|
||||
_validate_capability_alignment(
|
||||
operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
python_operation = python_capabilities[operation_id]
|
||||
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
|
||||
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
|
||||
|
||||
|
||||
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
|
||||
base_fields = {
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
}
|
||||
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS product operation entry fields are invalid")
|
||||
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
|
||||
_required_string(raw.get(field), f"product operation {field}")
|
||||
limits = raw.get("limits")
|
||||
stream = raw.get("stream")
|
||||
if not isinstance(limits, dict) or set(limits) != {
|
||||
"kfsMaxResponseBytes",
|
||||
"productMaxRequestBytes",
|
||||
"productMaxResponseBytes",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS product operation limits are invalid")
|
||||
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
|
||||
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
|
||||
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
|
||||
raise ValueError("KnowledgeFS product operation stream contract is invalid")
|
||||
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
|
||||
_required_string(stream.get("productKind"), "product stream kind")
|
||||
if gap:
|
||||
_required_string(raw.get("reason"), "product gap reason")
|
||||
_required_string(raw.get("reasonCode"), "product gap reasonCode")
|
||||
replacements = raw.get("replacementProductOperationIds")
|
||||
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
|
||||
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
|
||||
return cast(ProductOperationManifestEntry, raw)
|
||||
|
||||
|
||||
def _validate_product_entry(
|
||||
entry: ProductOperationManifestEntry,
|
||||
runtime: ProductOperationRuntimeContract,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
_validate_capability_alignment(
|
||||
runtime.kfs_operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
|
||||
expected: ProductOperationManifestEntry = {
|
||||
"action": runtime.action,
|
||||
"kfsOperationId": runtime.kfs_operation_id,
|
||||
"limits": {
|
||||
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
|
||||
"productMaxRequestBytes": runtime.max_request_bytes,
|
||||
"productMaxResponseBytes": runtime.max_response_bytes,
|
||||
},
|
||||
"method": runtime.method,
|
||||
"path": runtime.path,
|
||||
"productOperationId": runtime.product_operation_id,
|
||||
"resource": runtime.resource,
|
||||
"stream": {
|
||||
"kfsResponseKind": _openapi_response_kind(openapi_operation),
|
||||
"productKind": runtime.stream_kind,
|
||||
},
|
||||
"transport": runtime.transport,
|
||||
}
|
||||
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
|
||||
expected_value = expected[field]
|
||||
if entry[field] != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {entry[field]!r}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_capability_alignment(
|
||||
operation_id: str,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
python_operation = python_capabilities.get(operation_id)
|
||||
guard_operation = guard_capabilities.get(operation_id)
|
||||
openapi_operation = openapi_operations.get(operation_id)
|
||||
if python_operation is None or guard_operation is None or openapi_operation is None:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
|
||||
)
|
||||
method, path, _ = openapi_operation
|
||||
expected = (
|
||||
python_operation.action,
|
||||
python_operation.method,
|
||||
python_operation.path,
|
||||
python_operation.resource,
|
||||
)
|
||||
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
|
||||
if received_guard != expected:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
|
||||
if (method, path) != (python_operation.method, python_operation.path):
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
|
||||
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
|
||||
|
||||
|
||||
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
|
||||
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
|
||||
duplicates: set[str] = set()
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
if not isinstance(path, str) or not isinstance(path_item, dict):
|
||||
continue
|
||||
for method in ("delete", "get", "patch", "post", "put"):
|
||||
operation = path_item.get(method)
|
||||
if not isinstance(operation, dict):
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if not isinstance(operation_id, str) or not operation_id:
|
||||
continue
|
||||
if operation_id in operations:
|
||||
duplicates.add(operation_id)
|
||||
operations[operation_id] = (method.upper(), path, operation)
|
||||
if duplicates:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
|
||||
return operations
|
||||
|
||||
|
||||
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
|
||||
media_types = {
|
||||
media_type
|
||||
for status, response in operation.get("responses", {}).items()
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
|
||||
for media_type in response.get("content", {})
|
||||
}
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
||||
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
|
||||
return value
|
||||
|
||||
|
||||
def _unique(values: Iterable[str], label: str) -> None:
|
||||
sequence = tuple(values)
|
||||
if len(sequence) != len(set(sequence)):
|
||||
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CapabilityOperationPolicy",
|
||||
"CapabilityOperationRuntimeContract",
|
||||
"ProductOperationGapManifest",
|
||||
"ProductOperationManifest",
|
||||
"ProductOperationRuntimeContract",
|
||||
"capability_operation_runtime_contracts",
|
||||
"parse_capability_operation_policy",
|
||||
"parse_product_operation_gap_manifest",
|
||||
"parse_product_operation_manifest",
|
||||
"product_operation_runtime_contracts",
|
||||
"validate_product_operation_contracts",
|
||||
]
|
||||
@@ -1,23 +1,10 @@
|
||||
"""Register API blueprints with their browser-facing CORS policies."""
|
||||
|
||||
from configs import dify_config
|
||||
from constants import (
|
||||
HEADER_NAME_APP_CODE,
|
||||
HEADER_NAME_CSRF_TOKEN,
|
||||
HEADER_NAME_IDEMPOTENCY_KEY,
|
||||
HEADER_NAME_PASSPORT,
|
||||
HEADER_NAME_REQUEST_ID,
|
||||
)
|
||||
from constants import HEADER_NAME_APP_CODE, HEADER_NAME_CSRF_TOKEN, HEADER_NAME_PASSPORT
|
||||
from dify_app import DifyApp
|
||||
|
||||
BASE_CORS_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE, HEADER_NAME_PASSPORT)
|
||||
SERVICE_API_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, "Authorization")
|
||||
AUTHENTICATED_HEADERS: tuple[str, ...] = (
|
||||
*SERVICE_API_HEADERS,
|
||||
HEADER_NAME_CSRF_TOKEN,
|
||||
HEADER_NAME_IDEMPOTENCY_KEY,
|
||||
HEADER_NAME_REQUEST_ID,
|
||||
)
|
||||
AUTHENTICATED_HEADERS: tuple[str, ...] = (*SERVICE_API_HEADERS, HEADER_NAME_CSRF_TOKEN)
|
||||
FILES_HEADERS: tuple[str, ...] = (*BASE_CORS_HEADERS, HEADER_NAME_CSRF_TOKEN)
|
||||
EMBED_HEADERS: tuple[str, ...] = ("Content-Type", HEADER_NAME_APP_CODE)
|
||||
EXPOSED_HEADERS: tuple[str, ...] = ("X-Version", "X-Env", "X-Trace-Id")
|
||||
|
||||
@@ -179,14 +179,6 @@ def init_app(app: DifyApp) -> Celery:
|
||||
|
||||
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
|
||||
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
|
||||
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
|
||||
|
||||
if get_configured_knowledge_fs_lifecycle_worker_readiness().ready:
|
||||
imports.append("tasks.knowledge_fs_lifecycle_tasks")
|
||||
beat_schedule["knowledge_fs_lifecycle_worker"] = {
|
||||
"task": "tasks.knowledge_fs_lifecycle_tasks.run_knowledge_fs_lifecycle_worker",
|
||||
"schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
|
||||
}
|
||||
if dify_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK:
|
||||
imports.append("schedule.clean_embedding_cache_task")
|
||||
beat_schedule["clean_embedding_cache_task"] = {
|
||||
|
||||
@@ -27,7 +27,6 @@ def init_app(app: DifyApp):
|
||||
import_migration_data,
|
||||
install_plugins,
|
||||
install_rag_pipeline_plugins,
|
||||
knowledge_fs_control_space,
|
||||
migrate_data_for_plugin,
|
||||
migrate_dataset_permissions_to_rbac,
|
||||
migrate_member_roles_to_rbac,
|
||||
@@ -89,7 +88,6 @@ def init_app(app: DifyApp):
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
knowledge_fs_control_space,
|
||||
]
|
||||
for cmd in cmds_to_register:
|
||||
app.cli.add_command(cmd)
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
"""Production assembly for KnowledgeFS database-backed metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
|
||||
)
|
||||
from services.knowledge_fs.observability import get_knowledge_fs_operational_metrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_app(app: Flask) -> None:
|
||||
del app
|
||||
try:
|
||||
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_factory.get_session_maker())
|
||||
get_knowledge_fs_operational_metrics().register_control_space_state_gauge(reader)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeFS operational metric registration failed; continuing application startup")
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user