Compare commits
27
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d74499c09d | ||
|
|
29858d0db0 | ||
|
|
77732c7bb1 | ||
|
|
dde79fc9bf | ||
|
|
4ee43b8afc | ||
|
|
873ef2b592 | ||
|
|
43d725a1ce | ||
|
|
c8ccfba960 | ||
|
|
eee8d6cf7b | ||
|
|
11016563b1 | ||
|
|
b7edc5ef0c | ||
|
|
f93a5b95c6 | ||
|
|
6022939bf0 | ||
|
|
30df1433cc | ||
|
|
aa7b65c602 | ||
|
|
ba24745c07 | ||
|
|
38ea0de907 | ||
|
|
bbc31a3403 | ||
|
|
605c0fe02b | ||
|
|
15674a9db6 | ||
|
|
9a2d039e93 | ||
|
|
58d1dd8873 | ||
|
|
5ea884f799 | ||
|
|
2ec34b2cfb | ||
|
|
fa65f45bff | ||
|
|
63ca2b94b5 | ||
|
|
a7ef41a8f5 |
@@ -9,12 +9,11 @@ Use this skill for Vitest work under `web/` and `packages/dify-ui/`. Do not use
|
||||
|
||||
## Required Source
|
||||
|
||||
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill defines the execution workflow and must not add requirements that conflict with or duplicate that guide.
|
||||
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill provides an execution checklist and must not redefine or extend that policy.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the source, its behavior owner, nearby specs, and relevant public dependencies.
|
||||
1. Identify whether the contract belongs in `web/`, Dify UI Browser Mode, or a styled Storybook test.
|
||||
1. Apply the canonical guide to decide whether a test is needed and choose its boundary.
|
||||
1. For a behavior change or bug fix, write or identify the failing scenario first when practical.
|
||||
1. Implement one coherent scenario at a time and run the focused spec before expanding scope.
|
||||
@@ -33,4 +32,4 @@ vp test run path/to/spec-or-directory
|
||||
vp test run --project unit src/path/to/spec
|
||||
```
|
||||
|
||||
For styled Dify UI behavior, run `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
|
||||
Run Dify UI Storybook tests with `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx -y block-no-verify@1.1.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
**/*.pyc
|
||||
**/.mypy_cache
|
||||
**/.ruff_cache
|
||||
knowledge-fs/
|
||||
.git
|
||||
.github
|
||||
*.md
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/knowledge-fs"
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
groups:
|
||||
knowledge-fs-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
- package-ecosystem: "uv"
|
||||
directory: "/api"
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
jobs:
|
||||
api-unit:
|
||||
name: API Unit Tests
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
env:
|
||||
COVERAGE_FILE: coverage-unit
|
||||
defaults:
|
||||
@@ -75,7 +75,7 @@ jobs:
|
||||
|
||||
api-integration:
|
||||
name: API Integration Tests
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
env:
|
||||
COVERAGE_FILE: coverage-integration
|
||||
STORAGE_TYPE: opendal
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
|
||||
api-coverage:
|
||||
name: API Coverage
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
needs:
|
||||
- api-unit
|
||||
- api-integration
|
||||
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
|
||||
create-manifest:
|
||||
needs: build
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: github.repository == 'langgenius/dify'
|
||||
strategy:
|
||||
matrix:
|
||||
|
||||
@@ -23,7 +23,7 @@ concurrency:
|
||||
jobs:
|
||||
validate:
|
||||
name: validate manifest + resolve target Dify release
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: github.repository == 'langgenius/dify'
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
release:
|
||||
name: build + attach standalone binaries (all targets)
|
||||
needs: validate
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
contents: write
|
||||
defaults:
|
||||
|
||||
@@ -9,7 +9,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
db-migration-test-postgres:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
run: uv run --directory api flask upgrade-db
|
||||
|
||||
db-migration-test-mysql:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/agent'
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/dev'
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/enterprise'
|
||||
|
||||
@@ -13,7 +13,7 @@ on:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/saas'
|
||||
|
||||
@@ -22,7 +22,7 @@ concurrency:
|
||||
jobs:
|
||||
check-cherry-pick-provenance:
|
||||
name: Require cherry-pick provenance
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
name: KnowledgeFS CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
merge_group:
|
||||
branches: ["main"]
|
||||
types: [checks_requested]
|
||||
push:
|
||||
branches: ["main"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CI: true
|
||||
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
|
||||
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
name: Check KnowledgeFS changes
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect KnowledgeFS changes
|
||||
id: changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
filters: |
|
||||
knowledge-fs:
|
||||
- 'knowledge-fs/**'
|
||||
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
|
||||
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
|
||||
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
|
||||
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
|
||||
- 'knowledge-fs/scripts/export-openapi.mjs'
|
||||
- 'api/dev/generate_knowledge_fs_contract.py'
|
||||
- 'api/dev/knowledge_fs_product_contract.py'
|
||||
- 'api/knowledge-fs-contract.lock.json'
|
||||
- 'api/knowledge-fs-product-operation-gaps.json'
|
||||
- 'api/knowledge-fs-product-operations.json'
|
||||
- 'api/**/knowledge_fs/**'
|
||||
- 'api/**/*knowledge_fs*'
|
||||
- 'api/**/*knowledge-fs*'
|
||||
- 'api/.env.example'
|
||||
- 'api/app_factory.py'
|
||||
- 'api/commands/__init__.py'
|
||||
- 'api/controllers/console/__init__.py'
|
||||
- 'api/controllers/console/workspace/rbac.py'
|
||||
- 'api/controllers/service_api/__init__.py'
|
||||
- 'api/core/agent/base_agent_runner.py'
|
||||
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
|
||||
- 'api/core/rbac/entities.py'
|
||||
- 'api/core/tools/__base/tool_runtime.py'
|
||||
- 'api/core/tools/builtin_tool/_position.yaml'
|
||||
- 'api/core/workflow/node_runtime.py'
|
||||
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
|
||||
- 'api/extensions/ext_celery.py'
|
||||
- 'api/extensions/ext_commands.py'
|
||||
- 'api/models/__init__.py'
|
||||
- 'api/services/account_service.py'
|
||||
- 'api/services/agent_tool_inner_service.py'
|
||||
- 'api/services/enterprise/rbac_service.py'
|
||||
- 'api/services/entities/agent_tool_inner.py'
|
||||
- 'api/services/knowledge_fs/**'
|
||||
- 'api/services/knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
|
||||
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
|
||||
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
|
||||
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
|
||||
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
|
||||
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
|
||||
- 'api/tests/unit_tests/services/test_account_service.py'
|
||||
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
|
||||
- 'api/pyproject.toml'
|
||||
- 'api/uv.lock'
|
||||
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
|
||||
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
|
||||
- 'packages/contracts/generated/api/console/**'
|
||||
- 'packages/contracts/generated/api/service/**'
|
||||
- 'docker/.env.example'
|
||||
- 'docker/README.md'
|
||||
- 'docker/dify-env-sync.py'
|
||||
- 'docker/dify-env-sync.sh'
|
||||
- 'docker/docker-compose-template.yaml'
|
||||
- 'docker/docker-compose.yaml'
|
||||
- 'docker/envs/core-services/api.env.example'
|
||||
- 'docker/envs/core-services/knowledge-fs.env.example'
|
||||
- 'docker/generate_docker_compose'
|
||||
- 'docs/design/knowledge-fs*'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/workflows/knowledge-fs-ci.yml'
|
||||
|
||||
build:
|
||||
name: Build KnowledgeFS API production image
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract KnowledgeFS image metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
type=ref,event=branch
|
||||
type=sha,format=long
|
||||
|
||||
- name: Build KnowledgeFS API image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: ./knowledge-fs
|
||||
file: ./knowledge-fs/apps/api/Dockerfile
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
|
||||
quality:
|
||||
name: Run KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: ./knowledge-fs
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
package_json_file: knowledge-fs/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
|
||||
|
||||
- name: Install KnowledgeFS dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Scan KnowledgeFS secrets
|
||||
run: pnpm security:secrets
|
||||
|
||||
- name: Audit KnowledgeFS production dependencies
|
||||
run: pnpm security:dependencies
|
||||
|
||||
- name: Run KnowledgeFS checks
|
||||
run: pnpm check
|
||||
|
||||
- name: Build KnowledgeFS
|
||||
run: pnpm build
|
||||
|
||||
- name: Lint KnowledgeFS
|
||||
run: pnpm lint
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.12"
|
||||
cache-dependency-glob: |
|
||||
api/uv.lock
|
||||
dify-agent/uv.lock
|
||||
|
||||
- name: Verify Dify dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install Dify contract dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project api --locked --dev
|
||||
|
||||
- name: Collect Dify KnowledgeFS gate targets
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
production_targets=()
|
||||
add_production_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS production target is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
production_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_production_target "$path"
|
||||
fi
|
||||
done < <(
|
||||
find api \
|
||||
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
|
||||
-o -type f -name '*.py' -print0
|
||||
)
|
||||
|
||||
production_touchpoints=(
|
||||
api/app_factory.py
|
||||
api/commands/__init__.py
|
||||
api/controllers/console/__init__.py
|
||||
api/controllers/console/workspace/rbac.py
|
||||
api/controllers/service_api/__init__.py
|
||||
api/core/agent/base_agent_runner.py
|
||||
api/core/app/apps/agent_app/runtime_request_builder.py
|
||||
api/core/rbac/entities.py
|
||||
api/core/tools/__base/tool_runtime.py
|
||||
api/core/workflow/node_runtime.py
|
||||
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
|
||||
api/extensions/ext_celery.py
|
||||
api/extensions/ext_commands.py
|
||||
api/models/__init__.py
|
||||
api/services/account_service.py
|
||||
api/services/agent_tool_inner_service.py
|
||||
api/services/enterprise/rbac_service.py
|
||||
api/services/entities/agent_tool_inner.py
|
||||
)
|
||||
for path in "${production_touchpoints[@]}"; do
|
||||
add_production_target "$path"
|
||||
done
|
||||
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
|
||||
|
||||
if ((${#production_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
|
||||
|
||||
test_targets=()
|
||||
add_test_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
test_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_test_target "$path"
|
||||
fi
|
||||
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
|
||||
|
||||
test_touchpoints=(
|
||||
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
|
||||
api/tests/unit_tests/core/agent/test_base_agent_runner.py
|
||||
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
|
||||
api/tests/unit_tests/core/workflow/test_node_runtime.py
|
||||
api/tests/unit_tests/services/enterprise/test_rbac_service.py
|
||||
api/tests/unit_tests/services/test_account_service.py
|
||||
api/tests/unit_tests/services/test_agent_tool_inner_service.py
|
||||
)
|
||||
for path in "${test_touchpoints[@]}"; do
|
||||
add_test_target "$path"
|
||||
done
|
||||
|
||||
required_test_scopes=(
|
||||
/commands/
|
||||
/configs/
|
||||
/controllers/
|
||||
/core/agent/
|
||||
/core/app/apps/agent_app/
|
||||
/core/tools/builtin_tool/providers/knowledge_fs/
|
||||
/core/workflow/
|
||||
/dev/
|
||||
/extensions/
|
||||
/migrations/
|
||||
/models/
|
||||
/repositories/
|
||||
/services/
|
||||
/tasks/
|
||||
)
|
||||
for required_scope in "${required_test_scopes[@]}"; do
|
||||
scope_found=false
|
||||
for path in "${test_targets[@]}"; do
|
||||
if [[ "$path" == *"$required_scope"* ]]; then
|
||||
scope_found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$scope_found" != true ]]; then
|
||||
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ((${#test_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
|
||||
|
||||
- name: Lint Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev ruff format --check "${targets[@]}"
|
||||
uv run --project api --dev ruff check "${targets[@]}"
|
||||
|
||||
- name: Type-check Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
|
||||
|
||||
mypy_targets=()
|
||||
for path in "${targets[@]}"; do
|
||||
if [[ "$path" != api/migrations/* ]]; then
|
||||
mypy_targets+=("${path#api/}")
|
||||
fi
|
||||
done
|
||||
if ((${#mypy_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS Mypy target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --directory api --dev mypy \
|
||||
--explicit-package-bases \
|
||||
--exclude-gitignore \
|
||||
--exclude '(^|/)conftest\.py$' \
|
||||
--exclude 'tests/' \
|
||||
--exclude 'migrations/' \
|
||||
--check-untyped-defs \
|
||||
--disable-error-code=import-untyped \
|
||||
"${mypy_targets[@]}"
|
||||
|
||||
- name: Test Dify KnowledgeFS unit surface
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
|
||||
|
||||
- name: Enforce Dify KnowledgeFS focused coverage
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
KNOWLEDGE_FS_COVERAGE_BASE: >-
|
||||
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
|
||||
uv run --project api --dev coverage json --show-contexts -o "$report"
|
||||
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
|
||||
--coverage-json "$report" \
|
||||
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
|
||||
--base "$KNOWLEDGE_FS_COVERAGE_BASE" \
|
||||
--minimum 90 \
|
||||
--glue-minimum 90
|
||||
|
||||
- name: Verify Dify KnowledgeFS contract
|
||||
working-directory: .
|
||||
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
|
||||
|
||||
- name: Verify Dify Agent dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project dify-agent --check
|
||||
|
||||
- name: Install Dify Agent gate dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project dify-agent --locked --dev
|
||||
|
||||
- name: Lint Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: |
|
||||
uv run --project . --dev ruff format --check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
uv run --project . --dev ruff check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Type-check Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev basedpyright --level error
|
||||
src/dify_agent/layers/dify_core_tools/client.py
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Test Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev python -m pytest
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
-q
|
||||
|
||||
skip:
|
||||
name: Skip KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped KnowledgeFS checks
|
||||
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
|
||||
|
||||
final:
|
||||
name: KnowledgeFS CI
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
- check-changes
|
||||
- build
|
||||
- quality
|
||||
- skip
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize KnowledgeFS CI status
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BUILD_RESULT: ${{ needs.build.result }}
|
||||
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
|
||||
QUALITY_RESULT: ${{ needs.quality.result }}
|
||||
SKIP_RESULT: ${{ needs.skip.result }}
|
||||
run: |
|
||||
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
|
||||
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS build and checks ran successfully."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS checks were skipped because no related files changed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
|
||||
exit 1
|
||||
@@ -7,7 +7,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0
|
||||
with:
|
||||
|
||||
@@ -21,13 +21,13 @@ concurrency:
|
||||
jobs:
|
||||
pre_job:
|
||||
name: Skip Duplicate Checks
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
should_skip: ${{ steps.skip_check.outputs.should_skip || 'false' }}
|
||||
steps:
|
||||
- id: skip_check
|
||||
continue-on-error: true
|
||||
uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
|
||||
uses: fkirc/skip-duplicate-actions@b974a9395958c231af965b70070979a577efa578 # v5.3.2
|
||||
with:
|
||||
cancel_others: 'true'
|
||||
concurrent_skipping: same_content_newer
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
name: Check Changed Files
|
||||
needs: pre_job
|
||||
if: needs.pre_job.outputs.should_skip != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
api-changed: ${{ steps.changes.outputs.api }}
|
||||
cli-changed: ${{ steps.changes.outputs.cli }}
|
||||
@@ -164,7 +164,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped API tests
|
||||
run: echo "No API-related changes detected; skipping API tests."
|
||||
@@ -177,7 +177,7 @@ jobs:
|
||||
- check-changes
|
||||
- api-tests-run
|
||||
- api-tests-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize API Tests status
|
||||
env:
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.cli-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped CLI tests
|
||||
run: echo "No CLI-related changes detected; skipping CLI tests."
|
||||
@@ -237,7 +237,7 @@ jobs:
|
||||
- check-changes
|
||||
- cli-tests-run
|
||||
- cli-tests-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize CLI Tests status
|
||||
env:
|
||||
@@ -284,7 +284,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped web tests
|
||||
run: echo "No web-related changes detected; skipping web tests."
|
||||
@@ -297,7 +297,7 @@ jobs:
|
||||
- check-changes
|
||||
- web-tests-run
|
||||
- web-tests-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize Web Tests status
|
||||
env:
|
||||
@@ -344,7 +344,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped web full-stack e2e
|
||||
run: echo "No E2E-related changes detected; skipping web full-stack E2E."
|
||||
@@ -357,7 +357,7 @@ jobs:
|
||||
- check-changes
|
||||
- web-e2e-run
|
||||
- web-e2e-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize Web Full-Stack E2E status
|
||||
env:
|
||||
@@ -411,7 +411,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped VDB tests
|
||||
run: echo "No VDB-related changes detected; skipping VDB tests."
|
||||
@@ -424,7 +424,7 @@ jobs:
|
||||
- check-changes
|
||||
- vdb-tests-run
|
||||
- vdb-tests-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize VDB Tests status
|
||||
env:
|
||||
@@ -470,7 +470,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped DB migration tests
|
||||
run: echo "No migration-related changes detected; skipping DB migration tests."
|
||||
@@ -483,7 +483,7 @@ jobs:
|
||||
- check-changes
|
||||
- db-migration-test-run
|
||||
- db-migration-test-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize DB Migration Test status
|
||||
env:
|
||||
@@ -530,7 +530,7 @@ jobs:
|
||||
- pre_job
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.sandbox-runtime-changed != 'true'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped sandbox runtime tests
|
||||
run: echo "No sandbox-runtime-related changes detected; skipping sandbox runtime tests."
|
||||
@@ -543,7 +543,7 @@ jobs:
|
||||
- check-changes
|
||||
- sandbox-runtime-tests-run
|
||||
- sandbox-runtime-tests-skip
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize Sandbox Runtime Tests status
|
||||
env:
|
||||
|
||||
@@ -14,7 +14,7 @@ concurrency:
|
||||
jobs:
|
||||
check-changes:
|
||||
name: Check Changed Files
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
external-e2e-changed: ${{ steps.changes.outputs.external_e2e }}
|
||||
steps:
|
||||
|
||||
@@ -12,7 +12,7 @@ permissions: {}
|
||||
jobs:
|
||||
comment:
|
||||
name: Comment PR with pyrefly diff
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
@@ -10,7 +10,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pyrefly-diff:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
@@ -12,7 +12,7 @@ permissions: {}
|
||||
jobs:
|
||||
comment:
|
||||
name: Comment PR with type coverage
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
@@ -10,7 +10,7 @@ permissions:
|
||||
|
||||
jobs:
|
||||
pyrefly-type-coverage:
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
@@ -13,7 +13,7 @@ concurrency:
|
||||
jobs:
|
||||
sandbox-runtime-unit:
|
||||
name: Sandbox Runtime Unit Tests
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
|
||||
sandbox-runtime-lint:
|
||||
name: Sandbox Runtime Lint
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -51,20 +51,20 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v6.5.0
|
||||
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6.5.0
|
||||
with:
|
||||
working-directory: dify-agent-runtime
|
||||
version: latest
|
||||
|
||||
sandbox-runtime-integration:
|
||||
name: Sandbox Runtime Integration Tests
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
name: Validate PR title
|
||||
permissions:
|
||||
pull-requests: read
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Complete merge group check
|
||||
if: github.event_name == 'merge_group'
|
||||
|
||||
@@ -12,7 +12,7 @@ on:
|
||||
jobs:
|
||||
stale:
|
||||
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
@@ -19,7 +19,7 @@ permissions:
|
||||
jobs:
|
||||
python-style:
|
||||
name: Python Style
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
|
||||
web-style:
|
||||
name: Web Style
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./web
|
||||
@@ -182,7 +182,7 @@ jobs:
|
||||
|
||||
superlinter:
|
||||
name: SuperLinter
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
|
||||
@@ -17,7 +17,7 @@ concurrency:
|
||||
jobs:
|
||||
build:
|
||||
name: unit test for Node.js SDK
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
|
||||
defaults:
|
||||
run:
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Use Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: ''
|
||||
|
||||
@@ -35,7 +35,7 @@ concurrency:
|
||||
jobs:
|
||||
translate:
|
||||
if: github.repository == 'langgenius/dify'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
timeout-minutes: 120
|
||||
|
||||
steps:
|
||||
@@ -158,7 +158,7 @@ jobs:
|
||||
|
||||
- name: Run Claude Code for Translation Sync
|
||||
if: steps.context.outputs.CHANGED_FILES != ''
|
||||
uses: anthropics/claude-code-action@e90deca47693f9457b72f2b53c17d7c445a87342 # v1.0.171
|
||||
uses: anthropics/claude-code-action@af0559ee4f514d1ef21826982bed13f7edc3c35e # v1.0.178
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -16,7 +16,7 @@ concurrency:
|
||||
jobs:
|
||||
trigger:
|
||||
if: github.repository == 'langgenius/dify'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
test:
|
||||
name: Full VDB Tests
|
||||
if: github.repository == 'langgenius/dify'
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
|
||||
@@ -13,7 +13,7 @@ concurrency:
|
||||
jobs:
|
||||
test:
|
||||
name: VDB Smoke Tests
|
||||
runs-on: depot-ubuntu-24.04
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
|
||||
@@ -30,6 +30,11 @@ share/python-wheels/
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
|
||||
# directory contains source files rather than Python build output.
|
||||
!/knowledge-fs/apps/admin/lib/
|
||||
!/knowledge-fs/apps/admin/lib/**
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
|
||||
@@ -681,6 +681,31 @@ AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
|
||||
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
|
||||
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
|
||||
|
||||
# KnowledgeFS (Dataset 2.0)
|
||||
KNOWLEDGE_FS_ENABLED=false
|
||||
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
|
||||
KNOWLEDGE_FS_BASE_URL=
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN=
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
|
||||
# Legacy rollback-only HMAC; Capability v2 deployments leave this blank.
|
||||
KNOWLEDGE_FS_JWT_SECRET=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
|
||||
|
||||
# Marketplace configuration
|
||||
MARKETPLACE_ENABLED=true
|
||||
MARKETPLACE_API_URL=https://marketplace.dify.ai
|
||||
|
||||
@@ -154,6 +154,7 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_forward_refs,
|
||||
ext_hosting_provider,
|
||||
ext_import_modules,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_logging,
|
||||
ext_login,
|
||||
ext_logstore,
|
||||
@@ -204,6 +205,7 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_enterprise_telemetry,
|
||||
ext_request_logging,
|
||||
ext_session_factory,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_oauth_bearer,
|
||||
]
|
||||
for ext in extensions:
|
||||
|
||||
@@ -10,6 +10,7 @@ from .data_migration import (
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
)
|
||||
from .knowledge_fs import knowledge_fs_control_space
|
||||
from .plugin import (
|
||||
backfill_plugin_auto_upgrade,
|
||||
extract_plugins,
|
||||
@@ -75,6 +76,7 @@ __all__ = [
|
||||
"import_migration_data",
|
||||
"install_plugins",
|
||||
"install_rag_pipeline_plugins",
|
||||
"knowledge_fs_control_space",
|
||||
"legacy_model_types",
|
||||
"migrate_annotation_vector_database",
|
||||
"migrate_data_for_plugin",
|
||||
|
||||
@@ -0,0 +1,604 @@
|
||||
"""Operator commands for the independent KnowledgeFS control-plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from services.knowledge_fs.cleanup import (
|
||||
CleanupApprovalInput,
|
||||
CleanupCompletionEvidenceInput,
|
||||
CleanupReadinessEvidenceInput,
|
||||
CleanupStartInput,
|
||||
KnowledgeFSCleanupError,
|
||||
KnowledgeFSCleanupService,
|
||||
)
|
||||
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
|
||||
from services.knowledge_fs.control_space_management import (
|
||||
KnowledgeFSControlSpaceManagementService,
|
||||
KnowledgeFSControlSpaceRegistration,
|
||||
)
|
||||
from services.knowledge_fs.cutover import (
|
||||
CutoverSmokeResultsInput,
|
||||
FinalDeltaInput,
|
||||
KnowledgeFSCutoverError,
|
||||
KnowledgeFSWorkspaceCutoverService,
|
||||
LegacyDependencyInput,
|
||||
QuarantineResolutionInput,
|
||||
ShadowAuthorizationObservationInput,
|
||||
ShadowCompletionInput,
|
||||
WorkspaceInventoryInput,
|
||||
)
|
||||
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
|
||||
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
|
||||
|
||||
|
||||
@click.group("knowledge-fs-control-space")
|
||||
def knowledge_fs_control_space() -> None:
|
||||
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("dry-run")
|
||||
@click.option("--tenant-id", default=None)
|
||||
def dry_run(tenant_id: str | None) -> None:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("inventory")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
|
||||
def inventory(input_path: Path, apply: bool) -> None:
|
||||
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.inventory, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("register")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--owner-account-id", required=True)
|
||||
@click.option("--provisioning-key", required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def register(
|
||||
tenant_id: str,
|
||||
owner_account_id: str,
|
||||
provisioning_key: str,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space, replayed = _management_service().register(
|
||||
KnowledgeFSControlSpaceRegistration(
|
||||
tenant_id,
|
||||
owner_account_id,
|
||||
provisioning_key,
|
||||
knowledge_space_id,
|
||||
knowledge_space_revision,
|
||||
)
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("backfill")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
|
||||
def backfill(input_path: Path, apply: bool) -> None:
|
||||
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.backfill, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("quarantine-resolve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
|
||||
def quarantine_resolve(input_path: Path, apply: bool) -> None:
|
||||
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
|
||||
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-start")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
|
||||
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.begin_shadow(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at) if started_at is not None else None,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-report")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
|
||||
def shadow_report(input_path: Path, apply: bool) -> None:
|
||||
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
|
||||
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
|
||||
def shadow_complete(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
|
||||
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_issue_fail_closed(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_issue(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-dashboard")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--checked-at", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
|
||||
def legacy_dashboard(
|
||||
tenant_id: str,
|
||||
input_path: Path,
|
||||
checked_at: str,
|
||||
expected_cas_version: int | None,
|
||||
apply: bool,
|
||||
) -> None:
|
||||
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
|
||||
report = _operator_call(
|
||||
lambda: _cutover_service().legacy_dependency_dashboard(
|
||||
tenant_id=tenant_id,
|
||||
dependencies=dependencies,
|
||||
expected_cas_version=expected_cas_version,
|
||||
checked_at=_parse_timestamp(checked_at),
|
||||
apply=apply,
|
||||
)
|
||||
)
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-check")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def legacy_check(tenant_id: str) -> None:
|
||||
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
|
||||
passed = (
|
||||
bool(status_report["legacy_dependency_ready"])
|
||||
and status_report["open_issues"] == 0
|
||||
and status_report["unresolved_cutover_quarantine"] == 0
|
||||
)
|
||||
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
|
||||
if not passed:
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("freeze")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "freeze_at", required=True)
|
||||
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.freeze(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
freeze_at=_parse_timestamp(freeze_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("final-delta")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def final_delta(input_path: Path) -> None:
|
||||
payload = _read_one_jsonl(input_path, FinalDeltaInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(lambda: service.apply_final_delta(payload))
|
||||
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cutover")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "cutover_at", required=True)
|
||||
@click.option("--rollback-cutoff-at", required=True)
|
||||
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.cutover(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
cutover_at=_parse_timestamp(cutover_at),
|
||||
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("smoke")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
|
||||
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.record_smoke_results(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
results=results,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("observe")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--started-at", default=None)
|
||||
@click.option("--window-ends-at", default=None)
|
||||
@click.option("--maximum-task-expires-at", default=None)
|
||||
@click.option("--observed-at", default=None)
|
||||
def observe(
|
||||
tenant_id: str,
|
||||
expected_cas_version: int,
|
||||
started_at: str | None,
|
||||
window_ends_at: str | None,
|
||||
maximum_task_expires_at: str | None,
|
||||
observed_at: str | None,
|
||||
) -> None:
|
||||
service = _cutover_service()
|
||||
if observed_at is not None:
|
||||
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
|
||||
raise click.UsageError("--observed-at cannot be combined with observation start options")
|
||||
_operator_call(
|
||||
lambda: service.complete_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
observed_at=_parse_timestamp(observed_at),
|
||||
)
|
||||
)
|
||||
else:
|
||||
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
|
||||
raise click.UsageError(
|
||||
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
|
||||
)
|
||||
_operator_call(
|
||||
lambda: service.begin_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at),
|
||||
window_ends_at=_parse_timestamp(window_ends_at),
|
||||
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("rollback")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "rolled_back_at", required=True)
|
||||
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.rollback(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
rolled_back_at=_parse_timestamp(rolled_back_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def status(tenant_id: str) -> None:
|
||||
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-request")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
|
||||
def cleanup_request(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-approve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
|
||||
def cleanup_approve(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-start")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
|
||||
@click.option(
|
||||
"--acknowledge-irreversible",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms rollback will be permanently closed.",
|
||||
)
|
||||
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupStartInput)
|
||||
if apply and not acknowledge_irreversible:
|
||||
raise click.UsageError("--apply requires --acknowledge-irreversible")
|
||||
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
|
||||
@click.option(
|
||||
"--acknowledge-executed",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
|
||||
)
|
||||
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
|
||||
if apply and not acknowledge_executed:
|
||||
raise click.UsageError("--apply requires --acknowledge-executed")
|
||||
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--request-id", required=True)
|
||||
def cleanup_status(tenant_id: str, request_id: str) -> None:
|
||||
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("repair")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--control-space-id", required=True)
|
||||
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def repair(
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
expected_resource_version: int,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space = _management_service().repair_registration(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=expected_resource_version,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("orphan-report")
|
||||
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
|
||||
def orphan_report(limit: int) -> None:
|
||||
report = KnowledgeFSOrphanReconciler(
|
||||
session_factory.get_session_maker(),
|
||||
get_knowledge_fs_lifecycle_remote(),
|
||||
).reconcile(limit=limit, apply_repairs=False)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-request")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
|
||||
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
|
||||
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
|
||||
|
||||
if not apply:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
_echo_json(
|
||||
{
|
||||
"apply": False,
|
||||
"by_state": report.by_state,
|
||||
"tenant_id": tenant_id,
|
||||
"total": report.total,
|
||||
}
|
||||
)
|
||||
return
|
||||
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
|
||||
_echo_json(
|
||||
{
|
||||
"apply": True,
|
||||
"control_space_ids": [result.control_space.id for result in results],
|
||||
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-finalize")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
|
||||
@click.option(
|
||||
"--acknowledge-control-plane-purge",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply after every remote Space has reached deleted.",
|
||||
)
|
||||
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
|
||||
"""Release the Workspace FK only after all remote deletions are terminal."""
|
||||
|
||||
service = _lifecycle_service()
|
||||
if not apply:
|
||||
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
|
||||
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
|
||||
return
|
||||
if not acknowledge_control_plane_purge:
|
||||
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
|
||||
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
|
||||
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
|
||||
|
||||
|
||||
def _read_jsonl[InputT: BaseModel](
|
||||
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
|
||||
) -> tuple[InputT, ...]:
|
||||
records: list[InputT] = []
|
||||
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
records.append(input_type.model_validate_json(line))
|
||||
except ValidationError as exc:
|
||||
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
|
||||
if not records and not allow_empty:
|
||||
raise click.ClickException("strict JSONL input must contain at least one record")
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
|
||||
records = _read_jsonl(input_path, input_type)
|
||||
if len(records) != 1:
|
||||
raise click.ClickException("this command requires exactly one JSONL record")
|
||||
return records[0]
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise click.ClickException("operator timestamps must include an explicit timezone")
|
||||
return parsed
|
||||
|
||||
|
||||
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCutoverError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCleanupError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSControlSpaceLifecycleError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _echo_json(payload: object) -> None:
|
||||
click.echo(json.dumps(payload, default=str, sort_keys=True))
|
||||
|
||||
|
||||
def _management_service() -> KnowledgeFSControlSpaceManagementService:
|
||||
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
|
||||
return KnowledgeFSWorkspaceCutoverService(
|
||||
session_factory.get_session_maker(),
|
||||
remote_factory=get_knowledge_fs_lifecycle_remote,
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_service() -> KnowledgeFSCleanupService:
|
||||
return KnowledgeFSCleanupService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
|
||||
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
__all__ = ["knowledge_fs_control_space"]
|
||||
+209
-63
@@ -1,6 +1,8 @@
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import TypedDict
|
||||
|
||||
@@ -21,6 +23,7 @@ from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEX_PREFIXES = tuple("0123456789abcdef")
|
||||
_TARGET_MONTH_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
|
||||
|
||||
|
||||
class WorkflowRunArchivePlanRow(TypedDict):
|
||||
@@ -66,6 +69,7 @@ def _parse_tenant_prefixes(prefixes: str | None) -> list[str]:
|
||||
|
||||
|
||||
def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[str] | None:
|
||||
"""Keep an omitted scope unset while rejecting an explicitly empty scope."""
|
||||
if raw_ids is None:
|
||||
return None
|
||||
parsed = sorted({raw_id.strip() for raw_id in raw_ids.split(",") if raw_id.strip()})
|
||||
@@ -74,6 +78,27 @@ def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[
|
||||
return parsed
|
||||
|
||||
|
||||
def _parse_archive_target_month(target_month: str) -> tuple[int, int]:
|
||||
"""Validate the V2 catalog month selector and return its numeric components."""
|
||||
if not _TARGET_MONTH_PATTERN.fullmatch(target_month):
|
||||
raise click.BadParameter("target-month must use YYYY-MM format", param_hint="--target-month")
|
||||
year_text, month_text = target_month.split("-", maxsplit=1)
|
||||
return int(year_text), int(month_text)
|
||||
|
||||
|
||||
def _parse_archive_catalog_cursor(after_catalog_id: str | None) -> str | None:
|
||||
"""Normalize the exclusive V2 catalog keyset cursor when one is provided."""
|
||||
if after_catalog_id is None:
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(after_catalog_id))
|
||||
except ValueError as exc:
|
||||
raise click.BadParameter(
|
||||
"after-catalog-id must be a UUID returned by the same V2 operation and scope",
|
||||
param_hint="--after-catalog-id",
|
||||
) from exc
|
||||
|
||||
|
||||
def _get_archive_candidate_tenant_ids_by_prefix(
|
||||
session: Session,
|
||||
prefix: str,
|
||||
@@ -810,9 +835,11 @@ def backfill_workflow_run_archive_bundles(
|
||||
click.echo(click.style(f" ... and {len(summary.errors) - 10} more failures", fg="red"))
|
||||
|
||||
|
||||
def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
def _echo_bundle_archive_operation_summary(summary, *, dry_run: bool) -> None:
|
||||
status = "completed successfully" if summary.bundles_failed == 0 else "completed with failures"
|
||||
fg = "green" if summary.bundles_failed == 0 else "red"
|
||||
cursor_label = "preview_next_catalog_id" if dry_run else "next_catalog_id"
|
||||
cursor_value = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
click.echo(
|
||||
click.style(
|
||||
f"{summary.operation} {status}. "
|
||||
@@ -821,10 +848,12 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
f"archive_bytes={summary.archive_bytes} duration={summary.elapsed_time:.2f}s "
|
||||
f"validation_time={summary.validation_time:.2f}s "
|
||||
f"runs_per_second={summary.runs_per_second:.2f} rows_per_second={summary.rows_per_second:.2f} "
|
||||
f"bytes_per_second={summary.bytes_per_second:.2f}",
|
||||
f"bytes_per_second={summary.bytes_per_second:.2f} {cursor_label}={cursor_value or 'none'}",
|
||||
fg=fg,
|
||||
)
|
||||
)
|
||||
if dry_run:
|
||||
click.echo(click.style("Dry-run cursor is preview-only; do not persist it for a destructive run.", fg="yellow"))
|
||||
click.echo(click.style("table,row_count", fg="white"))
|
||||
for table_name in [
|
||||
"workflow_runs",
|
||||
@@ -842,7 +871,8 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
click.style(
|
||||
f" bundle={result.bundle_id} tenant={result.tenant_id} runs={result.run_count} "
|
||||
f"rows={result.row_count} archive_bytes={result.archive_bytes} "
|
||||
f"time={result.elapsed_time:.2f}s validation={result.validation_time:.2f}s",
|
||||
f"catalog_id={result.catalog_id} time={result.elapsed_time:.2f}s "
|
||||
f"validation={result.validation_time:.2f}s",
|
||||
fg="white",
|
||||
)
|
||||
)
|
||||
@@ -850,7 +880,7 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
click.echo(
|
||||
click.style(
|
||||
f" failed bundle={result.bundle_id} tenant={result.tenant_id} "
|
||||
f"object_prefix={result.object_prefix} error={result.error}",
|
||||
f"catalog_id={result.catalog_id} object_prefix={result.object_prefix} error={result.error}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
@@ -867,25 +897,24 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
)
|
||||
@click.option("--run-id", required=False, help="Workflow run ID to restore.")
|
||||
@click.option(
|
||||
"--start-from",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--target-month",
|
||||
metavar="YYYY-MM",
|
||||
default=None,
|
||||
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
|
||||
help="V2 catalog month to restore; required unless --run-id is used.",
|
||||
)
|
||||
@click.option(
|
||||
"--end-before",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--after-catalog-id",
|
||||
default=None,
|
||||
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
|
||||
help="Exclusive V2 cursor from the same restore month and tenant scope.",
|
||||
)
|
||||
@click.option("--workers", default=1, show_default=True, type=int, help="V1 --run-id compatibility only.")
|
||||
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to restore.")
|
||||
@click.option("--limit", type=click.IntRange(min=1), default=100, show_default=True, help="Maximum V2 catalog rows.")
|
||||
@click.option("--dry-run", is_flag=True, help="Preview without restoring.")
|
||||
def restore_workflow_runs(
|
||||
tenant_ids: str | None,
|
||||
run_id: str | None,
|
||||
start_from: datetime.datetime | None,
|
||||
end_before: datetime.datetime | None,
|
||||
target_month: str | None,
|
||||
after_catalog_id: str | None,
|
||||
workers: int,
|
||||
limit: int,
|
||||
dry_run: bool,
|
||||
@@ -905,23 +934,20 @@ def restore_workflow_runs(
|
||||
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
|
||||
from services.retention.workflow_run.restore_archived_workflow_run import WorkflowRunRestore
|
||||
|
||||
parsed_tenant_ids = None
|
||||
if tenant_ids:
|
||||
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
|
||||
if not parsed_tenant_ids:
|
||||
raise click.BadParameter("tenant-ids must not be empty")
|
||||
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
|
||||
|
||||
if (start_from is None) ^ (end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before must be provided together.")
|
||||
if run_id is None and (start_from is None or end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before are required for batch restore.")
|
||||
if workers < 1:
|
||||
raise click.BadParameter("workers must be at least 1")
|
||||
if run_id is not None and (target_month is not None or after_catalog_id is not None):
|
||||
raise click.UsageError("--target-month and --after-catalog-id are only valid for V2 batch restore.")
|
||||
if run_id is None and target_month is None:
|
||||
raise click.UsageError("--target-month is required for V2 batch restore.")
|
||||
|
||||
start_time = datetime.datetime.now(datetime.UTC)
|
||||
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Starting restore of workflow run {run_id} at {start_time.isoformat()}.",
|
||||
f"Starting restore of {target_desc} at {start_time.isoformat()}.",
|
||||
fg="white",
|
||||
)
|
||||
)
|
||||
@@ -955,17 +981,20 @@ def restore_workflow_runs(
|
||||
click.echo(
|
||||
click.style("--workers is ignored for V2 bundle restore; bundles are processed serially.", fg="yellow")
|
||||
)
|
||||
assert start_from is not None
|
||||
assert end_before is not None
|
||||
assert target_month is not None
|
||||
target_year, target_month_number = _parse_archive_target_month(target_month)
|
||||
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
|
||||
bundle_restorer = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
|
||||
summary = bundle_restorer.restore_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
start_date=start_from,
|
||||
end_date=end_before,
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
after_catalog_id=catalog_cursor,
|
||||
limit=limit,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary)
|
||||
return
|
||||
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
|
||||
if summary.bundles_failed:
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
|
||||
@click.command(
|
||||
@@ -979,23 +1008,41 @@ def restore_workflow_runs(
|
||||
)
|
||||
@click.option("--run-id", required=False, help="Workflow run ID to delete.")
|
||||
@click.option(
|
||||
"--start-from",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--target-month",
|
||||
metavar="YYYY-MM",
|
||||
default=None,
|
||||
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
|
||||
help="V2 catalog month to delete; required unless --run-id is used.",
|
||||
)
|
||||
@click.option(
|
||||
"--end-before",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--after-catalog-id",
|
||||
default=None,
|
||||
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
|
||||
help="Exclusive V2 cursor from the same delete month and tenant scope.",
|
||||
)
|
||||
@click.option(
|
||||
"--run-shard-index",
|
||||
default=None,
|
||||
type=click.IntRange(min=0),
|
||||
help="Zero-based archive shard index. Must be paired with --run-shard-total.",
|
||||
)
|
||||
@click.option(
|
||||
"--run-shard-total",
|
||||
default=None,
|
||||
type=click.IntRange(min=1, max=16),
|
||||
help="Total archive shard count. Must be paired with --run-shard-index.",
|
||||
)
|
||||
@click.option("--all-pages", is_flag=True, help="Process catalog pages until an empty page is reached.")
|
||||
@click.option(
|
||||
"--limit",
|
||||
type=click.IntRange(min=1),
|
||||
default=100,
|
||||
show_default=True,
|
||||
help="Maximum V2 catalog rows per page.",
|
||||
)
|
||||
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to delete.")
|
||||
@click.option("--dry-run", is_flag=True, help="Preview without deleting.")
|
||||
@click.option(
|
||||
"--skip-bad-archives",
|
||||
is_flag=True,
|
||||
help="Continue batch deletion when one archive object fails validation.",
|
||||
help="V1 --run-id only: continue when one archive object fails validation.",
|
||||
)
|
||||
@click.option(
|
||||
"--restore-sample-interval",
|
||||
@@ -1007,8 +1054,11 @@ def restore_workflow_runs(
|
||||
def delete_archived_workflow_runs(
|
||||
tenant_ids: str | None,
|
||||
run_id: str | None,
|
||||
start_from: datetime.datetime | None,
|
||||
end_before: datetime.datetime | None,
|
||||
target_month: str | None,
|
||||
after_catalog_id: str | None,
|
||||
run_shard_index: int | None,
|
||||
run_shard_total: int | None,
|
||||
all_pages: bool,
|
||||
limit: int,
|
||||
dry_run: bool,
|
||||
skip_bad_archives: bool,
|
||||
@@ -1018,26 +1068,38 @@ def delete_archived_workflow_runs(
|
||||
Delete archived workflow runs from the database.
|
||||
|
||||
Batch delete uses V2 bundle metadata and validates object existence, manifest schema, object size, checksum, row
|
||||
counts, and source/archive content checksums before deleting source rows. `--run-id` keeps the V1 per-run path.
|
||||
counts, and source/archive content checksums before deleting source rows. Parallel workers may select one exact
|
||||
archive shard; all-pages mode keeps only the current bounded page in memory. `--run-id` keeps the V1 per-run path.
|
||||
"""
|
||||
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
|
||||
from services.retention.workflow_run.delete_archived_workflow_run import ArchivedWorkflowRunDeletion
|
||||
|
||||
parsed_tenant_ids = None
|
||||
if tenant_ids:
|
||||
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
|
||||
if not parsed_tenant_ids:
|
||||
raise click.BadParameter("tenant-ids must not be empty")
|
||||
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
|
||||
|
||||
if (start_from is None) ^ (end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before must be provided together.")
|
||||
if run_id is None and (start_from is None or end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before are required for batch delete.")
|
||||
if restore_sample_interval < 0:
|
||||
raise click.BadParameter("restore-sample-interval must be >= 0")
|
||||
if run_id is not None and (
|
||||
target_month is not None
|
||||
or after_catalog_id is not None
|
||||
or run_shard_index is not None
|
||||
or run_shard_total is not None
|
||||
or all_pages
|
||||
):
|
||||
raise click.UsageError(
|
||||
"--target-month, --after-catalog-id, --run-shard-index, --run-shard-total, and --all-pages "
|
||||
"are only valid for V2 batch delete."
|
||||
)
|
||||
if run_id is None and target_month is None:
|
||||
raise click.UsageError("--target-month is required for V2 batch delete.")
|
||||
if run_id is None and skip_bad_archives:
|
||||
raise click.UsageError("--skip-bad-archives is not supported for V2 catalog batches; they fail fast.")
|
||||
if (run_shard_index is None) ^ (run_shard_total is None):
|
||||
raise click.UsageError("--run-shard-index and --run-shard-total must be provided together.")
|
||||
if run_shard_index is not None and run_shard_total is not None and run_shard_index >= run_shard_total:
|
||||
raise click.UsageError("--run-shard-index must be less than --run-shard-total.")
|
||||
|
||||
start_time = datetime.datetime.now(datetime.UTC)
|
||||
target_desc = f"workflow run {run_id}" if run_id else "workflow runs"
|
||||
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Starting delete of {target_desc} at {start_time.isoformat()}.",
|
||||
@@ -1110,20 +1172,104 @@ def delete_archived_workflow_runs(
|
||||
|
||||
if restore_sample_interval:
|
||||
click.echo(click.style("--restore-sample-interval is ignored for V2 bundle delete.", fg="yellow"))
|
||||
assert start_from is not None
|
||||
assert end_before is not None
|
||||
bundle_deleter = WorkflowRunBundleArchiveMaintenance(
|
||||
dry_run=dry_run,
|
||||
strict_content_validation=True,
|
||||
stop_on_error=not skip_bad_archives,
|
||||
assert target_month is not None
|
||||
target_year, target_month_number = _parse_archive_target_month(target_month)
|
||||
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
|
||||
shard = (
|
||||
f"{run_shard_index:02d}-of-{run_shard_total:02d}"
|
||||
if run_shard_index is not None and run_shard_total is not None
|
||||
else None
|
||||
)
|
||||
summary = bundle_deleter.delete_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
start_date=start_from,
|
||||
end_date=end_before,
|
||||
limit=limit,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary)
|
||||
bundle_deleter = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
|
||||
if run_shard_total is not None:
|
||||
try:
|
||||
bundle_deleter.validate_catalog_shards(
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
shard_total=run_shard_total,
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.exception(
|
||||
"Archive catalog shard preflight failed: target_month=%s shard=%s",
|
||||
target_month,
|
||||
shard,
|
||||
)
|
||||
raise click.ClickException(
|
||||
f"Archive catalog shard preflight failed for target_month={target_month} shard={shard}: {exc}"
|
||||
) from exc
|
||||
|
||||
initial_catalog_cursor = catalog_cursor
|
||||
pages_processed = 0
|
||||
bundles_succeeded = 0
|
||||
runs_processed = 0
|
||||
rows_processed = 0
|
||||
archive_bytes = 0
|
||||
while True:
|
||||
summary = bundle_deleter.delete_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
after_catalog_id=catalog_cursor,
|
||||
limit=limit,
|
||||
shard=shard,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
|
||||
if summary.bundles_failed:
|
||||
failed_result = next((result for result in summary.results if not result.success), None)
|
||||
failed_catalog_id = failed_result.catalog_id if failed_result is not None else "unknown"
|
||||
page_resume_cursor = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
resume_cursor = page_resume_cursor or catalog_cursor
|
||||
if dry_run:
|
||||
cursor_details = (
|
||||
f"preview_after_catalog_id={resume_cursor or 'none'} "
|
||||
f"destructive_retry_after_catalog_id={initial_catalog_cursor or 'none'}"
|
||||
)
|
||||
else:
|
||||
cursor_details = f"resume_after_catalog_id={resume_cursor or 'none'}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete stopped: target_month={target_month} shard={shard or 'all'} "
|
||||
f"failed_catalog_id={failed_catalog_id} "
|
||||
f"{cursor_details}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
if not all_pages:
|
||||
break
|
||||
if summary.bundles_processed == 0:
|
||||
break
|
||||
|
||||
pages_processed += 1
|
||||
bundles_succeeded += summary.bundles_succeeded
|
||||
runs_processed += summary.runs_processed
|
||||
rows_processed += summary.rows_processed
|
||||
archive_bytes += summary.archive_bytes
|
||||
next_catalog_id = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
if next_catalog_id is None or (catalog_cursor is not None and next_catalog_id <= catalog_cursor):
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete cursor did not advance: target_month={target_month} shard={shard or 'all'} "
|
||||
f"after_catalog_id={catalog_cursor or 'none'} next_catalog_id={next_catalog_id or 'none'}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
raise click.exceptions.Exit(1)
|
||||
catalog_cursor = next_catalog_id
|
||||
|
||||
if all_pages:
|
||||
final_cursor_label = "preview_final_catalog_id" if dry_run else "final_catalog_id"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete all-pages completed successfully. target_month={target_month} shard={shard or 'all'} "
|
||||
f"pages={pages_processed} bundles_success={bundles_succeeded} runs={runs_processed} "
|
||||
f"rows={rows_processed} archive_bytes={archive_bytes} "
|
||||
f"{final_cursor_label}={catalog_cursor or 'none'}",
|
||||
fg="green",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _find_orphaned_draft_variables(batch_size: int = 1000) -> list[str]:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from configs.extra.agent_backend_config import AgentBackendConfig
|
||||
from configs.extra.archive_config import ArchiveStorageConfig
|
||||
from configs.extra.knowledge_fs_config import KnowledgeFSConfig
|
||||
from configs.extra.notion_config import NotionConfig
|
||||
from configs.extra.sentry_config import SentryConfig
|
||||
|
||||
@@ -8,6 +9,7 @@ class ExtraServiceConfig(
|
||||
# place the configs in alphabet order
|
||||
AgentBackendConfig,
|
||||
ArchiveStorageConfig,
|
||||
KnowledgeFSConfig,
|
||||
NotionConfig,
|
||||
SentryConfig,
|
||||
):
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Configuration for the optional KnowledgeFS control-plane integration."""
|
||||
|
||||
from ipaddress import ip_address
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class KnowledgeFSConfig(BaseSettings):
|
||||
"""Server-only KnowledgeFS connection and rollout settings."""
|
||||
|
||||
KNOWLEDGE_FS_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable the private KnowledgeFS Console bridge.",
|
||||
)
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
|
||||
)
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
|
||||
)
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
|
||||
)
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
|
||||
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN: str | None = Field(
|
||||
default=None,
|
||||
description="Public KnowledgeFS origin returned with direct upload capabilities.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
|
||||
default=None,
|
||||
description="Identifier for the current asymmetric Capability v2 signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
|
||||
default=None,
|
||||
description="Server-only PEM for the current Capability v2 RSA signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
|
||||
default=None,
|
||||
description="Optional public-only JWKS JSON retained during key rotation overlap.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
|
||||
|
||||
@field_validator(
|
||||
"KNOWLEDGE_FS_BASE_URL",
|
||||
"KNOWLEDGE_FS_DIRECT_ORIGIN",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def normalize_optional_string(cls, value: object) -> object:
|
||||
if isinstance(value, SecretStr):
|
||||
normalized = value.get_secret_value().strip()
|
||||
return SecretStr(normalized) if normalized else None
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
return value
|
||||
|
||||
@field_validator("KNOWLEDGE_FS_BASE_URL")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
|
||||
|
||||
@field_validator("KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
@classmethod
|
||||
def validate_direct_origin(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
|
||||
@classmethod
|
||||
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
|
||||
try:
|
||||
_ = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{name} must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
|
||||
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
|
||||
return value.rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
|
||||
if str(getattr(self, "DEPLOY_ENV", "")).strip().upper() == "PRODUCTION":
|
||||
for name, value in (
|
||||
("KNOWLEDGE_FS_BASE_URL", self.KNOWLEDGE_FS_BASE_URL),
|
||||
("KNOWLEDGE_FS_DIRECT_ORIGIN", self.KNOWLEDGE_FS_DIRECT_ORIGIN),
|
||||
):
|
||||
if value and not self._is_secure_or_loopback_origin(value):
|
||||
raise ValueError(f"{name} must use HTTPS in production unless it targets loopback")
|
||||
if self.KNOWLEDGE_FS_ENABLED:
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
|
||||
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
|
||||
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
|
||||
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
|
||||
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
|
||||
):
|
||||
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _is_secure_or_loopback_origin(value: str) -> bool:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme == "https":
|
||||
return True
|
||||
hostname = (parsed.hostname or "").rstrip(".").lower()
|
||||
if hostname == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ip_address(hostname).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
@@ -125,6 +125,7 @@ from .explore import (
|
||||
saved_message,
|
||||
trial,
|
||||
)
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .snippets import snippet_workflow, snippet_workflow_draft_variable
|
||||
from .socketio import workflow as socketio_workflow
|
||||
|
||||
@@ -195,6 +196,7 @@ __all__ = [
|
||||
"human_input_form",
|
||||
"init_validate",
|
||||
"installed_app",
|
||||
"knowledge_fs_resources",
|
||||
"load_balancing_config",
|
||||
"login",
|
||||
"mcp_server",
|
||||
|
||||
@@ -105,6 +105,7 @@ class SyncDraftWorkflowPayload(BaseModel):
|
||||
graph: dict[str, Any]
|
||||
features: dict[str, Any]
|
||||
hash: str | None = None
|
||||
is_collaborative: bool = Field(default=False, alias="_is_collaborative")
|
||||
environment_variables: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
)
|
||||
@@ -610,6 +611,7 @@ class DraftWorkflowApi(Resource):
|
||||
environment_variables=environment_variables,
|
||||
conversation_variables=conversation_variables,
|
||||
session=db.session(),
|
||||
graph_only=args["is_collaborative"],
|
||||
)
|
||||
except WorkflowHashNotEqualError:
|
||||
raise DraftWorkflowNotSync()
|
||||
|
||||
@@ -47,7 +47,9 @@ from controllers.console.explore.error import (
|
||||
NotWorkflowAppError,
|
||||
)
|
||||
from controllers.console.explore.wraps import TrialAppResource, trial_feature_enable
|
||||
from controllers.console.wraps import with_current_user
|
||||
from controllers.console.files import FILE_UPLOAD_PARAMS, upload_file_from_request
|
||||
from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file_from_request
|
||||
from controllers.console.wraps import cloud_edition_billing_resource_check, with_current_user
|
||||
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
|
||||
from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
|
||||
from core.app.apps.base_app_queue_manager import AppQueueManager
|
||||
@@ -61,12 +63,13 @@ from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from fields.base import ResponseModel
|
||||
from fields.conversation_variable_fields import WorkflowConversationVariableResponse
|
||||
from fields.file_fields import FileResponse, FileWithSignedUrl
|
||||
from fields.message_fields import SuggestedQuestionsResponse
|
||||
from graphon.graph_engine.manager import GraphEngineManager
|
||||
from graphon.model_runtime.errors.invoke import InvokeError
|
||||
from libs import helper
|
||||
from libs.helper import dump_response, to_timestamp, uuid_value
|
||||
from models import Account
|
||||
from models import Account, App
|
||||
from models.account import TenantStatus
|
||||
from models.model import AppMode, Site, load_annotation_reply_config
|
||||
from models.workflow import Workflow
|
||||
@@ -428,6 +431,36 @@ register_response_schema_models(
|
||||
simple_account_model = console_ns.models[TrialSimpleAccount.__name__]
|
||||
|
||||
|
||||
class TrialAppFileUploadApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account, app_model: App):
|
||||
"""Upload a file into the tenant that owns the trial app."""
|
||||
upload_file = upload_file_from_request(
|
||||
current_user=current_user,
|
||||
resource_tenant_id=app_model.tenant_id,
|
||||
)
|
||||
return dump_response(FileResponse, upload_file), 201
|
||||
|
||||
|
||||
class TrialAppRemoteFileUploadApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account, app_model: App):
|
||||
"""Upload a remote file into the tenant that owns the trial app."""
|
||||
remote_file = upload_remote_file_from_request(
|
||||
current_user=current_user,
|
||||
resource_tenant_id=app_model.tenant_id,
|
||||
)
|
||||
return remote_file.model_dump(mode="json"), 201
|
||||
|
||||
|
||||
class TrialAppWorkflowRunApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@console_ns.expect(console_ns.models[WorkflowRunRequest.__name__])
|
||||
@@ -887,6 +920,18 @@ class DatasetListApi(Resource):
|
||||
|
||||
console_ns.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialAppFileUploadApi,
|
||||
"/trial-apps/<uuid:app_id>/files/upload",
|
||||
endpoint="trial_app_file_upload",
|
||||
)
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialAppRemoteFileUploadApi,
|
||||
"/trial-apps/<uuid:app_id>/remote-files/upload",
|
||||
endpoint="trial_app_remote_file_upload",
|
||||
)
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialMessageSuggestedQuestionApi,
|
||||
"/trial-apps/<uuid:app_id>/messages/<uuid:message_id>/suggested-questions",
|
||||
|
||||
@@ -29,7 +29,7 @@ from extensions.ext_database import db
|
||||
from fields.file_fields import FileResponse, UploadConfig
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from models import Account, UploadFile
|
||||
from services.file_service import FileService
|
||||
|
||||
from . import console_ns
|
||||
@@ -39,7 +39,7 @@ register_response_schema_models(console_ns, AllowedExtensionsResponse, TextConte
|
||||
|
||||
PREVIEW_WORDS_LIMIT = 3000
|
||||
|
||||
_FILE_UPLOAD_PARAMS = {
|
||||
FILE_UPLOAD_PARAMS = {
|
||||
"file": {
|
||||
"description": "File to upload",
|
||||
"in": "formData",
|
||||
@@ -56,6 +56,43 @@ _FILE_UPLOAD_PARAMS = {
|
||||
}
|
||||
|
||||
|
||||
def upload_file_from_request(*, current_user: Account, resource_tenant_id: str | None = None) -> UploadFile:
|
||||
"""Validate the multipart request and persist the file under the requested resource tenant."""
|
||||
source_str = request.form.get("source")
|
||||
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
|
||||
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
|
||||
if len(request.files) > 1:
|
||||
raise TooManyFilesError()
|
||||
file = request.files["file"]
|
||||
|
||||
if not file.filename:
|
||||
raise FilenameNotExistsError
|
||||
if source == "datasets" and not current_user.is_dataset_editor:
|
||||
raise Forbidden()
|
||||
|
||||
if source not in ("datasets", None):
|
||||
source = None
|
||||
|
||||
try:
|
||||
return FileService(db.engine).upload_file(
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mimetype=file.mimetype,
|
||||
user=current_user,
|
||||
tenant_id=resource_tenant_id,
|
||||
source=source,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
|
||||
raise BlockedFileExtensionError(blocked_extension_error.description)
|
||||
|
||||
|
||||
@console_ns.route("/files/upload")
|
||||
class FileApi(Resource):
|
||||
@setup_required
|
||||
@@ -81,42 +118,11 @@ class FileApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account):
|
||||
source_str = request.form.get("source")
|
||||
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
|
||||
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
|
||||
if len(request.files) > 1:
|
||||
raise TooManyFilesError()
|
||||
file = request.files["file"]
|
||||
|
||||
if not file.filename:
|
||||
raise FilenameNotExistsError
|
||||
if source == "datasets" and not current_user.is_dataset_editor:
|
||||
raise Forbidden()
|
||||
|
||||
if source not in ("datasets", None):
|
||||
source = None
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mimetype=file.mimetype,
|
||||
user=current_user,
|
||||
source=source,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
|
||||
raise BlockedFileExtensionError(blocked_extension_error.description)
|
||||
upload_file = upload_file_from_request(current_user=current_user)
|
||||
|
||||
return dump_response(FileResponse, upload_file), 201
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Typed Dify-owned KnowledgeFS Console product API."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Stable, non-enumerating Console error contract for KnowledgeFS."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSSpaceNotFoundHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_space_not_found"
|
||||
description = "KnowledgeFS space was not found."
|
||||
code = 404
|
||||
|
||||
|
||||
class KnowledgeFSOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSAccessDeniedHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_access_denied"
|
||||
description = "KnowledgeFS operation is not allowed."
|
||||
code = 403
|
||||
|
||||
|
||||
class KnowledgeFSRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSAccessDeniedHTTPError",
|
||||
"KnowledgeFSInvalidRequestHTTPError",
|
||||
"KnowledgeFSOperationUnavailableHTTPError",
|
||||
"KnowledgeFSQuotaExceededHTTPError",
|
||||
"KnowledgeFSRateLimitHTTPError",
|
||||
"KnowledgeFSSpaceNotFoundHTTPError",
|
||||
"KnowledgeFSUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,6 +46,61 @@ class GetRemoteFileInfo(Resource):
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
def upload_remote_file_from_request(
|
||||
*,
|
||||
current_user: Account,
|
||||
resource_tenant_id: str | None = None,
|
||||
) -> FileWithSignedUrl:
|
||||
"""Validate the JSON request, fetch its remote file, and persist it under the requested tenant."""
|
||||
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
|
||||
url = payload.url
|
||||
|
||||
# Try to fetch remote file metadata/content first
|
||||
try:
|
||||
resp = remote_fetcher.make_request("HEAD", url=url)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
# Normalize into a user-friendly error message expected by tests
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
|
||||
except httpx.RequestError as e:
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
|
||||
|
||||
file_info = helpers.guess_file_info_from_response(resp)
|
||||
|
||||
# Enforce file size limit with 400 (Bad Request) per tests' expectation
|
||||
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
|
||||
raise FileTooLargeError()
|
||||
|
||||
# Load content if needed
|
||||
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file_info.filename,
|
||||
content=content,
|
||||
mimetype=file_info.mimetype,
|
||||
user=current_user,
|
||||
tenant_id=resource_tenant_id,
|
||||
source_url=url,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
return FileWithSignedUrl(
|
||||
id=upload_file.id,
|
||||
name=upload_file.name,
|
||||
size=upload_file.size,
|
||||
extension=upload_file.extension,
|
||||
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
|
||||
mime_type=upload_file.mime_type,
|
||||
created_by=upload_file.created_by,
|
||||
created_at=int(upload_file.created_at.timestamp()),
|
||||
)
|
||||
|
||||
|
||||
@console_ns.route("/remote-files/upload")
|
||||
class RemoteFileUpload(Resource):
|
||||
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
|
||||
@@ -53,53 +108,8 @@ class RemoteFileUpload(Resource):
|
||||
@login_required
|
||||
@with_current_user
|
||||
def post(self, current_user: Account):
|
||||
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
|
||||
url = payload.url
|
||||
|
||||
# Try to fetch remote file metadata/content first
|
||||
try:
|
||||
resp = remote_fetcher.make_request("HEAD", url=url)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
# Normalize into a user-friendly error message expected by tests
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
|
||||
except httpx.RequestError as e:
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
|
||||
|
||||
file_info = helpers.guess_file_info_from_response(resp)
|
||||
|
||||
# Enforce file size limit with 400 (Bad Request) per tests' expectation
|
||||
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
|
||||
raise FileTooLargeError()
|
||||
|
||||
# Load content if needed
|
||||
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file_info.filename,
|
||||
content=content,
|
||||
mimetype=file_info.mimetype,
|
||||
user=current_user,
|
||||
source_url=url,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
# Success: return created resource with 201 status
|
||||
remote_file = upload_remote_file_from_request(current_user=current_user)
|
||||
return (
|
||||
FileWithSignedUrl(
|
||||
id=upload_file.id,
|
||||
name=upload_file.name,
|
||||
size=upload_file.size,
|
||||
extension=upload_file.extension,
|
||||
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
|
||||
mime_type=upload_file.mime_type,
|
||||
created_by=upload_file.created_by,
|
||||
created_at=int(upload_file.created_at.timestamp()),
|
||||
).model_dump(mode="json"),
|
||||
remote_file.model_dump(mode="json"),
|
||||
201,
|
||||
)
|
||||
|
||||
@@ -98,6 +98,7 @@ def handle_collaboration_event(sid, data):
|
||||
6. workflow_update
|
||||
7. comments_update
|
||||
8. node_panel_presence
|
||||
9. graph_view_state (session reports tab visibility; drives leader election)
|
||||
"""
|
||||
return collaboration_service.relay_collaboration_event(sid, data)
|
||||
|
||||
|
||||
@@ -346,7 +346,13 @@ class RBACRoleItemApi(Resource):
|
||||
def put(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_RoleUpsertRequest)
|
||||
role = svc.RBACService.Roles.update(tenant_id, account_id, str(role_id), request.to_mutation())
|
||||
role = svc.RBACService.KnowledgeFSRoleMutations.update_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
request.to_mutation(),
|
||||
session=db.session(),
|
||||
)
|
||||
return _dump(role)
|
||||
|
||||
@login_required
|
||||
@@ -356,7 +362,12 @@ class RBACRoleItemApi(Resource):
|
||||
@console_ns.response(200, "Success", console_ns.models[svc.RBACRole.__name__])
|
||||
def delete(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
svc.RBACService.Roles.delete(tenant_id, account_id, str(role_id))
|
||||
svc.RBACService.KnowledgeFSRoleMutations.delete_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
session=db.session(),
|
||||
)
|
||||
return {"result": "success"}
|
||||
|
||||
|
||||
@@ -915,7 +926,7 @@ class RBACMemberRolesApi(Resource):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_ReplaceMemberRolesRequest)
|
||||
return _dump(
|
||||
svc.RBACService.MemberRoles.replace(
|
||||
svc.RBACService.KnowledgeFSRoleMutations.replace_member_roles(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(member_id),
|
||||
|
||||
@@ -8,6 +8,7 @@ from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
|
||||
from controllers.inner_api.wraps import plugin_inner_api_only
|
||||
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
|
||||
from core.plugin.backwards_invocation.datasource import PluginDatasourceBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.encrypt import PluginEncrypter
|
||||
from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
|
||||
@@ -15,10 +16,12 @@ from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
RequestFetchAppInfo,
|
||||
RequestInvokeApp,
|
||||
RequestInvokeDatasource,
|
||||
RequestInvokeEncrypt,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeParameterExtractorNode,
|
||||
RequestInvokeQuestionClassifierNode,
|
||||
RequestInvokeRerank,
|
||||
@@ -27,6 +30,7 @@ from core.plugin.entities.request import (
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTool,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
RequestRequestDownloadFile,
|
||||
RequestRequestUploadFile,
|
||||
)
|
||||
@@ -118,6 +122,36 @@ class PluginInvokeTextEmbeddingApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/multimodal-embedding")
|
||||
class PluginInvokeMultimodalEmbeddingApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeMultimodalEmbedding)
|
||||
@inner_api_ns.doc("plugin_invoke_multimodal_embedding")
|
||||
@inner_api_ns.doc(description="Invoke multimodal embedding models through Dify model management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Multimodal embedding successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeMultimodalEmbedding):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.invoke_multimodal_embedding(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/rerank")
|
||||
class PluginInvokeRerankApi(Resource):
|
||||
@get_user_tenant
|
||||
@@ -144,6 +178,70 @@ class PluginInvokeRerankApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/model-catalog")
|
||||
class PluginModelCatalogApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestListModels)
|
||||
@inner_api_ns.doc("plugin_model_catalog")
|
||||
@inner_api_ns.doc(description="List tenant-active models managed by Dify")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Model catalog lookup successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestListModels):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.list_models(
|
||||
tenant_id=tenant_model.id,
|
||||
user_id=user_model.id,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/datasource")
|
||||
class PluginInvokeDatasourceApi(Resource):
|
||||
"""Invoke an installed datasource with credentials resolved inside Dify."""
|
||||
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeDatasource)
|
||||
@inner_api_ns.doc("plugin_invoke_datasource")
|
||||
@inner_api_ns.doc(description="Invoke datasource plugins through Dify credential management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Datasource invocation successful (streaming response)",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Datasource provider, datasource, or credential not found",
|
||||
}
|
||||
)
|
||||
def post(
|
||||
self,
|
||||
user_model: Account | EndUser,
|
||||
tenant_model: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
):
|
||||
response = PluginDatasourceBackwardsInvocation.invoke(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
return length_prefixed_response(
|
||||
0xF,
|
||||
PluginDatasourceBackwardsInvocation.convert_to_event_stream(response),
|
||||
)
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/tts")
|
||||
class PluginInvokeTTSApi(Resource):
|
||||
@get_user_tenant
|
||||
|
||||
@@ -38,6 +38,7 @@ from .dataset import (
|
||||
)
|
||||
from .dataset.rag_pipeline import rag_pipeline_workflow
|
||||
from .end_user import end_user
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .workspace import models
|
||||
|
||||
__all__ = [
|
||||
@@ -54,6 +55,7 @@ __all__ = [
|
||||
"hit_testing",
|
||||
"human_input_form",
|
||||
"index",
|
||||
"knowledge_fs_resources",
|
||||
"message",
|
||||
"metadata",
|
||||
"models",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""KnowledgeFS-specific Service API authenticated by resource credentials."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Stable KnowledgeFS Service API error contract."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSInvalidCredentialHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_credential"
|
||||
description = "Invalid KnowledgeFS service credential."
|
||||
code = 401
|
||||
|
||||
|
||||
class KnowledgeFSServiceOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSServiceUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSServiceInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSServiceRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSServiceQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSInvalidCredentialHTTPError",
|
||||
"KnowledgeFSServiceInvalidRequestHTTPError",
|
||||
"KnowledgeFSServiceOperationUnavailableHTTPError",
|
||||
"KnowledgeFSServiceQuotaExceededHTTPError",
|
||||
"KnowledgeFSServiceRateLimitHTTPError",
|
||||
"KnowledgeFSServiceUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,10 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
|
||||
from core.app.apps.base_app_runner import AppRunner
|
||||
from core.app.entities.app_invoke_entities import (
|
||||
AgentChatAppGenerateEntity,
|
||||
DifyRunContext,
|
||||
InvokeFrom,
|
||||
ModelConfigWithCredentialsEntity,
|
||||
UserFrom,
|
||||
)
|
||||
from core.app.file_access import DatabaseFileAccessController
|
||||
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
|
||||
@@ -148,6 +151,15 @@ class BaseAgentRunner(AppRunner):
|
||||
user_id=self.user_id,
|
||||
invoke_from=self.application_generate_entity.invoke_from,
|
||||
)
|
||||
invoke_from = self.application_generate_entity.invoke_from
|
||||
if isinstance(invoke_from, InvokeFrom):
|
||||
tool_entity.runtime.dify_run_context = DifyRunContext(
|
||||
tenant_id=self.tenant_id,
|
||||
app_id=self.app_config.app_id,
|
||||
user_id=self.user_id,
|
||||
user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER),
|
||||
invoke_from=invoke_from,
|
||||
)
|
||||
assert tool_entity.entity.description
|
||||
message_tool = PromptMessageTool(
|
||||
name=tool.tool_name,
|
||||
|
||||
@@ -159,6 +159,7 @@ class AgentAppRuntimeRequestBuilder:
|
||||
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
agent_mode="agent_app",
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
# ENG-616: expand slash-menu mention tokens to canonical names so
|
||||
# no frontend-internal {{#…#}} marker ever reaches the model.
|
||||
|
||||
@@ -38,6 +38,11 @@ class DatasourcePluginProviderController(ABC):
|
||||
):
|
||||
raise ToolProviderCredentialValidationError("Invalid credentials")
|
||||
|
||||
def validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
"""Validate credential shape and value against this installed provider declaration."""
|
||||
self.validate_credentials_format(credentials)
|
||||
self._validate_credentials(user_id, credentials)
|
||||
|
||||
@property
|
||||
def provider_type(self) -> DatasourceProviderType:
|
||||
"""
|
||||
|
||||
@@ -47,6 +47,24 @@ class MaxRetriesExceededError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ResponseLimitError(ValueError):
|
||||
"""Base error for responses that cannot be safely bounded."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ResponseTooLargeError(ResponseLimitError):
|
||||
"""Raised when an identity response exceeds the configured byte limit."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedResponseEncodingError(ResponseLimitError):
|
||||
"""Raised when response encoding prevents safe decoded-size enforcement."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
request_error = httpx.RequestError
|
||||
max_retries_exceeded_error = MaxRetriesExceededError
|
||||
|
||||
@@ -142,7 +160,31 @@ def _inject_trace_headers(headers: Headers | None) -> Headers:
|
||||
return headers
|
||||
|
||||
|
||||
def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
|
||||
def make_request(
|
||||
method: str,
|
||||
url: str,
|
||||
max_retries: int = SSRF_DEFAULT_MAX_RETRIES,
|
||||
stream_response: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> httpx.Response:
|
||||
"""Send one SSRF-protected request with optional streaming.
|
||||
|
||||
Args:
|
||||
method: HTTP method sent through the configured SSRF client.
|
||||
url: Absolute request URL.
|
||||
max_retries: Number of retry attempts after the initial request.
|
||||
stream_response: Return an open streaming response that the caller must close.
|
||||
**kwargs: Additional keyword arguments forwarded to ``httpx.Client``.
|
||||
|
||||
Returns:
|
||||
A buffered response, or an open response when ``stream_response`` is true.
|
||||
|
||||
Raises:
|
||||
ToolSSRFError: The configured SSRF proxy rejects the destination.
|
||||
MaxRetriesExceededError: All configured request attempts fail.
|
||||
httpx.RequestError: A request fails while retries are disabled.
|
||||
ValueError: The SSL verification option or request headers are invalid.
|
||||
"""
|
||||
# Convert requests-style allow_redirects to httpx-style follow_redirects
|
||||
if "allow_redirects" in kwargs:
|
||||
allow_redirects = kwargs.pop("allow_redirects")
|
||||
@@ -175,6 +217,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
# When using a forward proxy, httpx may override the Host header based on the URL.
|
||||
# We extract and preserve any explicitly set Host header to support virtual hosting.
|
||||
user_provided_host = _get_user_provided_host_header(headers)
|
||||
send_kwargs: dict[str, Any] = {}
|
||||
if "auth" in kwargs:
|
||||
send_kwargs["auth"] = kwargs.pop("auth")
|
||||
if "follow_redirects" in kwargs:
|
||||
send_kwargs["follow_redirects"] = kwargs.pop("follow_redirects")
|
||||
|
||||
retries = 0
|
||||
while retries <= max_retries:
|
||||
@@ -185,7 +232,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
if user_provided_host is not None:
|
||||
headers["host"] = user_provided_host
|
||||
kwargs["headers"] = headers
|
||||
response = client.request(method=method, url=url, **kwargs)
|
||||
request = client.build_request(method=method, url=url, **kwargs)
|
||||
if stream_response:
|
||||
response = client.send(request, stream=True, **send_kwargs)
|
||||
else:
|
||||
response = client.send(request, **send_kwargs)
|
||||
|
||||
# Check for SSRF protection by Squid proxy
|
||||
if response.status_code in (401, 403):
|
||||
@@ -195,6 +246,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
|
||||
# Squid typically identifies itself in Server or Via headers
|
||||
if "squid" in server_header or "squid" in via_header:
|
||||
response.close()
|
||||
raise ToolSSRFError(
|
||||
f"Access to '{url}' was blocked by SSRF protection. "
|
||||
f"The URL may point to a private or local network address. "
|
||||
@@ -208,6 +260,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
response.status_code,
|
||||
url,
|
||||
)
|
||||
response.close()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
logger.warning("Request to URL %s failed on attempt %s: %s", url, retries + 1, e)
|
||||
@@ -220,6 +273,42 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
raise MaxRetriesExceededError(f"Reached maximum retries ({max_retries}) for URL {url}")
|
||||
|
||||
|
||||
def buffer_response(response: httpx.Response, *, max_response_bytes: int) -> httpx.Response:
|
||||
"""Consume one open identity response under a decoded byte limit and close its stream."""
|
||||
if max_response_bytes <= 0:
|
||||
raise ValueError("max_response_bytes must be positive")
|
||||
|
||||
try:
|
||||
content_encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
||||
if content_encoding not in {"", "identity"}:
|
||||
raise UnsupportedResponseEncodingError(f"content encoding {content_encoding} cannot be safely bounded")
|
||||
content = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
if len(content) + len(chunk) > max_response_bytes:
|
||||
raise ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
|
||||
content.extend(chunk)
|
||||
decoded_headers = {
|
||||
name: value
|
||||
for name, value in response.headers.items()
|
||||
if name.lower() not in {"content-encoding", "content-length", "transfer-encoding"}
|
||||
}
|
||||
try:
|
||||
request = response.request
|
||||
except RuntimeError:
|
||||
request = None
|
||||
return httpx.Response(
|
||||
response.status_code,
|
||||
headers=decoded_headers,
|
||||
content=bytes(content),
|
||||
request=request,
|
||||
extensions=response.extensions,
|
||||
history=response.history,
|
||||
default_encoding=response.default_encoding,
|
||||
)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def get(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
|
||||
return make_request("GET", url, max_retries=max_retries, **kwargs)
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Datasource backward invocation through Dify's tenant-bound runtime.
|
||||
|
||||
This module is the internal service boundary used by trusted callers such as
|
||||
KnowledgeFS. It resolves installed provider declarations and Dify-owned
|
||||
credential references before reaching ``PluginDatasourceManager``; callers
|
||||
must never provide raw datasource credentials or a plugin-daemon API key.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.datasource.datasource_manager import DatasourceManager
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
OnlineDriveBrowseFilesRequest,
|
||||
OnlineDriveDownloadFileRequest,
|
||||
)
|
||||
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
|
||||
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
|
||||
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import RequestInvokeDatasource
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import DatasourceProviderID
|
||||
from services.datasource_provider_service import DatasourceProviderService
|
||||
|
||||
|
||||
class PluginDatasourceBackwardsInvocation(BaseBackwardsInvocation):
|
||||
"""Resolve and invoke a datasource without exposing credential material to the caller."""
|
||||
|
||||
@classmethod
|
||||
def invoke(
|
||||
cls,
|
||||
*,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
) -> Generator[BaseModel | dict[str, Any], None, None]:
|
||||
"""Yield datasource messages for one validated inner-runtime request."""
|
||||
provider_id = DatasourceProviderID(payload.provider)
|
||||
canonical_provider_id = str(provider_id)
|
||||
controller = DatasourceManager.get_datasource_plugin_provider(
|
||||
provider_id=canonical_provider_id,
|
||||
tenant_id=tenant.id,
|
||||
datasource_type=payload.datasource_type,
|
||||
)
|
||||
if controller.entity.provider_type != payload.datasource_type:
|
||||
raise ValueError("Datasource provider type mismatch")
|
||||
|
||||
# Resolving the datasource from the installed declaration prevents a caller
|
||||
# from dispatching an arbitrary datasource name under a valid plugin ID.
|
||||
runtime = controller.get_datasource(payload.datasource)
|
||||
credentials = DatasourceProviderService().get_datasource_credentials(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider_id.provider_name,
|
||||
plugin_id=provider_id.plugin_id,
|
||||
credential_id=payload.credential_id,
|
||||
)
|
||||
if controller.need_credentials and not credentials:
|
||||
raise ValueError("Datasource credential not found")
|
||||
|
||||
if payload.operation == "validate_credentials":
|
||||
controller.validate_credentials(user_id=user_id, credentials=credentials)
|
||||
yield {"result": True}
|
||||
return
|
||||
|
||||
runtime.runtime.credentials = credentials
|
||||
provider_type = runtime.datasource_provider_type()
|
||||
|
||||
match payload.operation:
|
||||
case "get_website_crawl":
|
||||
website = cast(WebsiteCrawlDatasourcePlugin, runtime)
|
||||
yield from website.get_website_crawl(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_pages":
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_pages(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_page_content":
|
||||
if payload.page is None:
|
||||
raise ValueError("Online-document page input is required")
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_page_content(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.page,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_browse_files":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_browse_files(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveBrowseFilesRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_download_file":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_download_file(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveDownloadFileRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case _:
|
||||
raise ValueError(f"Unsupported datasource operation: {payload.operation}")
|
||||
@@ -1,22 +1,31 @@
|
||||
import tempfile
|
||||
from binascii import hexlify, unhexlify
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Generator, Mapping
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.app.llm import deduct_llm_quota
|
||||
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
|
||||
from core.model_manager import ModelManager
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
InvokableModelCatalogItem,
|
||||
InvokableModelCatalogPage,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeRerank,
|
||||
RequestInvokeSpeech2Text,
|
||||
RequestInvokeSummary,
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
)
|
||||
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
|
||||
from core.plugin.plugin_service import PluginService
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
from core.tools.utils.model_invocation_utils import ModelInvocationUtils
|
||||
from graphon.model_runtime.entities.llm_entities import (
|
||||
@@ -33,6 +42,20 @@ from graphon.model_runtime.entities.message_entities import (
|
||||
)
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import ModelProviderID
|
||||
|
||||
|
||||
def _json_compatible(value: Any) -> Any:
|
||||
"""Convert model-runtime metadata into stable JSON-compatible values."""
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(mode="json")
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Mapping):
|
||||
return {str(_json_compatible(key)): _json_compatible(child) for key, child in value.items()}
|
||||
if isinstance(value, list | tuple | set):
|
||||
return [_json_compatible(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
@@ -183,7 +206,30 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
)
|
||||
|
||||
# invoke model
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts)
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts, input_type=payload.input_type)
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def invoke_multimodal_embedding(
|
||||
cls,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeMultimodalEmbedding,
|
||||
):
|
||||
"""Invoke multimodal embedding through the tenant-bound model instance."""
|
||||
model_instance = cls._get_bound_model_instance(
|
||||
tenant_id=tenant.id,
|
||||
user_id=user_id,
|
||||
provider=payload.provider,
|
||||
model_type=payload.model_type,
|
||||
model=payload.model,
|
||||
)
|
||||
|
||||
response = model_instance.invoke_multimodal_embedding(
|
||||
multimodel_documents=[document.model_dump(exclude_none=True) for document in payload.documents],
|
||||
input_type=payload.input_type,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
@@ -210,6 +256,67 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def list_models(
|
||||
cls,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
payload: RequestListModels,
|
||||
) -> InvokableModelCatalogPage:
|
||||
"""List only models that are active for the tenant's Dify configuration."""
|
||||
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id, user_id=user_id)
|
||||
active_models = provider_manager.get_configurations(tenant_id).get_models(
|
||||
model_type=payload.model_type,
|
||||
only_active=True,
|
||||
)
|
||||
|
||||
installed_identities: dict[str, str] = {}
|
||||
for plugin in PluginService.list(tenant_id):
|
||||
existing = installed_identities.get(plugin.plugin_id)
|
||||
if existing is not None and existing != plugin.plugin_unique_identifier:
|
||||
raise ValueError(f"Ambiguous installed identity for model plugin {plugin.plugin_id}")
|
||||
installed_identities[plugin.plugin_id] = plugin.plugin_unique_identifier
|
||||
|
||||
requested_provider = str(ModelProviderID(payload.provider)) if payload.provider else None
|
||||
matched_models = [
|
||||
model
|
||||
for model in active_models
|
||||
if (requested_provider is None or model.provider.provider == requested_provider)
|
||||
and (payload.model is None or model.model == payload.model)
|
||||
]
|
||||
matched_models.sort(key=lambda model: (model.provider.provider, model.model))
|
||||
|
||||
page_models = matched_models[payload.offset : payload.offset + payload.limit]
|
||||
items: list[InvokableModelCatalogItem] = []
|
||||
for model in page_models:
|
||||
provider_id = ModelProviderID(model.provider.provider)
|
||||
unique_identifier = installed_identities.get(provider_id.plugin_id)
|
||||
if unique_identifier is None:
|
||||
raise ValueError(f"Installed identity not found for active model plugin {provider_id.plugin_id}")
|
||||
items.append(
|
||||
InvokableModelCatalogItem(
|
||||
plugin_id=provider_id.plugin_id,
|
||||
plugin_unique_identifier=unique_identifier,
|
||||
provider=provider_id.provider_name,
|
||||
model=model.model,
|
||||
model_type=model.model_type,
|
||||
capabilities={
|
||||
"deprecated": model.deprecated,
|
||||
"features": _json_compatible(model.features or []),
|
||||
"fetchFrom": _json_compatible(model.fetch_from),
|
||||
"modelProperties": _json_compatible(model.model_properties),
|
||||
"modelType": model.model_type.value,
|
||||
"status": _json_compatible(model.status),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
next_offset = payload.offset + len(page_models)
|
||||
return InvokableModelCatalogPage(
|
||||
items=items,
|
||||
next_offset=next_offset if next_offset < len(matched_models) else None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def invoke_tts(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTTS):
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,11 @@ from typing import Any, Literal
|
||||
from flask import Response
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
DatasourceProviderType,
|
||||
GetOnlineDocumentPageContentRequest,
|
||||
)
|
||||
from core.entities.embedding_type import EmbeddingInputType
|
||||
from core.entities.provider_entities import BasicProviderConfig
|
||||
from core.plugin.utils.http_parser import deserialize_response
|
||||
from core.workflow.file_reference import is_canonical_file_reference
|
||||
@@ -54,6 +59,61 @@ class RequestInvokeTool(BaseModel):
|
||||
credential_id: str | None = None
|
||||
|
||||
|
||||
DatasourceInvocationOperation = Literal[
|
||||
"get_online_document_page_content",
|
||||
"get_online_document_pages",
|
||||
"get_website_crawl",
|
||||
"online_drive_browse_files",
|
||||
"online_drive_download_file",
|
||||
"validate_credentials",
|
||||
]
|
||||
|
||||
|
||||
class RequestInvokeDatasource(BaseModel):
|
||||
"""Invoke one installed datasource using a Dify-owned credential reference.
|
||||
|
||||
Raw credentials are intentionally not part of this contract. ``tenant_id`` and
|
||||
``user_id`` are consumed by the inner-API request context, while the remaining
|
||||
fields select an installed provider declaration and an operation-specific input.
|
||||
"""
|
||||
|
||||
tenant_id: str = Field(min_length=1, max_length=512)
|
||||
user_id: str = Field(min_length=1, max_length=512)
|
||||
provider: str = Field(min_length=1, max_length=768)
|
||||
datasource: str = Field(min_length=1, max_length=256)
|
||||
datasource_type: DatasourceProviderType
|
||||
credential_id: str = Field(min_length=1, max_length=512)
|
||||
operation: DatasourceInvocationOperation
|
||||
datasource_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
page: GetOnlineDocumentPageContentRequest | None = None
|
||||
request: dict[str, Any] | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_operation_payload(self) -> "RequestInvokeDatasource":
|
||||
expected_type = {
|
||||
"get_online_document_page_content": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_online_document_pages": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_website_crawl": DatasourceProviderType.WEBSITE_CRAWL,
|
||||
"online_drive_browse_files": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"online_drive_download_file": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"validate_credentials": self.datasource_type,
|
||||
}[self.operation]
|
||||
if self.datasource_type != expected_type:
|
||||
raise ValueError(f"{self.operation} requires datasource_type {expected_type.value}")
|
||||
|
||||
page_required = self.operation == "get_online_document_page_content"
|
||||
if page_required != (self.page is not None):
|
||||
raise ValueError("page is required only for get_online_document_page_content")
|
||||
|
||||
request_required = self.operation in {"online_drive_browse_files", "online_drive_download_file"}
|
||||
if request_required != (self.request is not None):
|
||||
raise ValueError("request is required only for online-drive operations")
|
||||
|
||||
return self
|
||||
|
||||
|
||||
class BaseRequestInvokeModel(BaseModel):
|
||||
provider: str
|
||||
model: str
|
||||
@@ -115,6 +175,25 @@ class RequestInvokeTextEmbedding(BaseRequestInvokeModel):
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
texts: list[str]
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class MultimodalEmbeddingDocument(BaseModel):
|
||||
"""A document accepted by a multimodal text-embedding model."""
|
||||
|
||||
content: str
|
||||
content_type: str
|
||||
file_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RequestInvokeMultimodalEmbedding(BaseRequestInvokeModel):
|
||||
"""Request to invoke a multimodal text-embedding model."""
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
documents: list[MultimodalEmbeddingDocument] = Field(min_length=1)
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
@@ -125,8 +204,40 @@ class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
model_type: ModelType = ModelType.RERANK
|
||||
query: str
|
||||
docs: list[str]
|
||||
score_threshold: float
|
||||
top_n: int
|
||||
score_threshold: float | None = None
|
||||
top_n: int | None = None
|
||||
|
||||
|
||||
class RequestListModels(BaseModel):
|
||||
"""Tenant-scoped query for models that Dify can invoke."""
|
||||
|
||||
model_type: Literal[ModelType.LLM, ModelType.TEXT_EMBEDDING, ModelType.RERANK]
|
||||
provider: str | None = None
|
||||
model: str | None = None
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=50, ge=1, le=100)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogItem(BaseModel):
|
||||
"""Installed identity and active Dify capability metadata for one model."""
|
||||
|
||||
plugin_id: str
|
||||
plugin_unique_identifier: str
|
||||
provider: str
|
||||
model: str
|
||||
model_type: ModelType
|
||||
capabilities: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogPage(BaseModel):
|
||||
"""Offset page returned by the internal model catalog endpoint."""
|
||||
|
||||
items: list[InvokableModelCatalogItem] = Field(default_factory=list)
|
||||
next_offset: int | None = None
|
||||
|
||||
|
||||
class RequestInvokeTTS(BaseRequestInvokeModel):
|
||||
|
||||
@@ -10,6 +10,7 @@ class RBACResourceScope(StrEnum):
|
||||
|
||||
APP = "app"
|
||||
DATASET = "dataset"
|
||||
KNOWLEDGE_FS = "knowledge_space"
|
||||
WORKSPACE = "workspace"
|
||||
|
||||
|
||||
@@ -57,6 +58,15 @@ class RBACPermission(StrEnum):
|
||||
DATASET_EXTERNAL_CONNECT = "dataset_external_connect"
|
||||
DATASET_IMPORT_EXPORT_DSL = "dataset_import_export_dsl"
|
||||
|
||||
KNOWLEDGE_FS_READ = "knowledge_space_read"
|
||||
KNOWLEDGE_FS_CREATE = "knowledge_space_create"
|
||||
KNOWLEDGE_FS_EDIT = "knowledge_space_edit"
|
||||
KNOWLEDGE_FS_DELETE = "knowledge_space_delete"
|
||||
KNOWLEDGE_FS_ACCESS_CONFIG = "knowledge_space_access_config"
|
||||
KNOWLEDGE_FS_API_KEY_MANAGE = "knowledge_space_api_key_manage"
|
||||
KNOWLEDGE_FS_DOCUMENT_WRITE = "knowledge_space_document_write"
|
||||
KNOWLEDGE_FS_QUERY = "knowledge_space_query"
|
||||
|
||||
WORKSPACE_MEMBER_MANAGE = "workspace_member_manage"
|
||||
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
|
||||
API_EXTENSION_MANAGE = "api_extension_manage"
|
||||
|
||||
@@ -2,7 +2,7 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
|
||||
from core.plugin.entities.plugin_daemon import CredentialType
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom
|
||||
|
||||
@@ -20,6 +20,7 @@ class ToolRuntime(BaseModel):
|
||||
tool_id: str | None = None
|
||||
invoke_from: InvokeFrom | None = None
|
||||
tool_invoke_from: ToolInvokeFrom | None = None
|
||||
dify_run_context: DifyRunContext | None = Field(default=None, exclude=True, repr=False)
|
||||
credentials: dict[str, Any] = Field(default_factory=dict)
|
||||
credential_type: CredentialType = Field(default=CredentialType.API_KEY)
|
||||
runtime_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
- audio
|
||||
- code
|
||||
- knowledge_fs
|
||||
- time
|
||||
- webscraper
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="none">
|
||||
<rect width="64" height="64" rx="14" fill="#155EEF"/>
|
||||
<path d="M17 17h20c5.5 0 10 4.5 10 10v20H27c-5.5 0-10-4.5-10-10V17Z" fill="white" fill-opacity=".96"/>
|
||||
<path d="M26 27h12M26 34h12M26 41h7" stroke="#155EEF" stroke-width="4" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
||||
|
After Width: | Height: | Size: 340 B |
@@ -0,0 +1,9 @@
|
||||
from typing import Any, override
|
||||
|
||||
from core.tools.builtin_tool.provider import BuiltinToolProviderController
|
||||
|
||||
|
||||
class KnowledgeFSProvider(BuiltinToolProviderController):
|
||||
@override
|
||||
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
_ = (user_id, credentials)
|
||||
@@ -0,0 +1,13 @@
|
||||
identity:
|
||||
author: Dify
|
||||
name: knowledge_fs
|
||||
label:
|
||||
en_US: KnowledgeFS
|
||||
zh_Hans: KnowledgeFS
|
||||
description:
|
||||
en_US: Run explicitly bound KnowledgeFS operations from an Agent or Workflow.
|
||||
zh_Hans: 从 Agent 或 Workflow 执行显式绑定的 KnowledgeFS 操作。
|
||||
icon: icon.svg
|
||||
tags:
|
||||
- rag
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any, override
|
||||
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.tools.builtin_tool.tool import BuiltinTool
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage
|
||||
from core.tools.errors import ToolInvokeError
|
||||
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
|
||||
from services.knowledge_fs.app_execution_capability import KnowledgeResourceRef
|
||||
from services.knowledge_fs.product_dto import KnowledgeFSResearchTaskCreatePayload
|
||||
from services.knowledge_fs.runtime import create_knowledge_fs_runtime
|
||||
|
||||
|
||||
class KnowledgeFSCreateResearchTaskTool(BuiltinTool):
|
||||
@override
|
||||
def _invoke(
|
||||
self,
|
||||
session: Session,
|
||||
user_id: str,
|
||||
tool_parameters: dict[str, Any],
|
||||
conversation_id: str | None = None,
|
||||
app_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
) -> Generator[ToolInvokeMessage, None, None]:
|
||||
_ = (user_id, conversation_id, app_id, message_id)
|
||||
run_context = self.runtime.dify_run_context
|
||||
if run_context is None or self.runtime.tenant_id != run_context.tenant_id:
|
||||
raise ToolInvokeError("KnowledgeFS requires a trusted Dify run context")
|
||||
match self.runtime.tool_invoke_from:
|
||||
case ToolInvokeFrom.AGENT:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
|
||||
case ToolInvokeFrom.WORKFLOW:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
|
||||
case _:
|
||||
raise ToolInvokeError("KnowledgeFS is only available to Agent and Workflow callers")
|
||||
|
||||
try:
|
||||
resource = KnowledgeResourceRef.model_validate(tool_parameters.get("resource"))
|
||||
payload_data: dict[str, object] = {
|
||||
"query": tool_parameters.get("query"),
|
||||
}
|
||||
mode = tool_parameters.get("mode")
|
||||
if mode:
|
||||
payload_data["mode"] = mode
|
||||
payload = KnowledgeFSResearchTaskCreatePayload.model_validate(payload_data)
|
||||
runtime = create_knowledge_fs_runtime(sessionmaker(bind=session.get_bind(), expire_on_commit=False))
|
||||
response = runtime.app_capabilities.create_research_task(
|
||||
run_context=run_context,
|
||||
caller_kind=caller_kind,
|
||||
resource=resource,
|
||||
payload=payload,
|
||||
)
|
||||
except ToolInvokeError:
|
||||
raise
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
raise ToolInvokeError(str(exc)) from exc
|
||||
|
||||
yield self.create_json_message(response.model_dump(mode="json", by_alias=True))
|
||||
@@ -0,0 +1,75 @@
|
||||
identity:
|
||||
name: create_research_task
|
||||
author: Dify
|
||||
label:
|
||||
en_US: Create KnowledgeFS Research Task
|
||||
zh_Hans: 创建 KnowledgeFS Research 任务
|
||||
description:
|
||||
human:
|
||||
en_US: Create a Research task in an explicitly bound KnowledgeFS space.
|
||||
zh_Hans: 在显式绑定的 KnowledgeFS 空间中创建 Research 任务。
|
||||
llm: Create a durable research task using an explicitly configured KnowledgeFS resource.
|
||||
parameters:
|
||||
- name: resource
|
||||
type: object
|
||||
required: true
|
||||
label:
|
||||
en_US: KnowledgeFS resource
|
||||
zh_Hans: KnowledgeFS 资源
|
||||
human_description:
|
||||
en_US: A typed KnowledgeFS control-space reference configured by the app author.
|
||||
zh_Hans: 由应用作者配置的类型化 KnowledgeFS control-space 引用。
|
||||
form: form
|
||||
input_schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
const: knowledge_fs
|
||||
control_space_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
required:
|
||||
- kind
|
||||
- control_space_id
|
||||
- name: query
|
||||
type: string
|
||||
required: true
|
||||
label:
|
||||
en_US: Research query
|
||||
zh_Hans: Research 查询
|
||||
human_description:
|
||||
en_US: The question the Research task should investigate.
|
||||
zh_Hans: Research 任务需要调查的问题。
|
||||
llm_description: The question to investigate with KnowledgeFS.
|
||||
form: llm
|
||||
- name: mode
|
||||
type: select
|
||||
required: false
|
||||
label:
|
||||
en_US: Mode
|
||||
zh_Hans: 模式
|
||||
human_description:
|
||||
en_US: Optional KnowledgeFS retrieval mode.
|
||||
zh_Hans: 可选的 KnowledgeFS 检索模式。
|
||||
llm_description: Optional retrieval mode. Use auto unless the task needs a specific mode.
|
||||
form: llm
|
||||
options:
|
||||
- value: auto
|
||||
label:
|
||||
en_US: Auto
|
||||
zh_Hans: 自动
|
||||
- value: fast
|
||||
label:
|
||||
en_US: Fast
|
||||
zh_Hans: 快速
|
||||
- value: deep
|
||||
label:
|
||||
en_US: Deep
|
||||
zh_Hans: 深度
|
||||
- value: research
|
||||
label:
|
||||
en_US: Research
|
||||
zh_Hans: Research
|
||||
|
||||
@@ -493,6 +493,7 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):
|
||||
self._run_context.invoke_from,
|
||||
variable_pool,
|
||||
)
|
||||
tool_runtime.runtime.dify_run_context = self._run_context
|
||||
except ToolNodeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
||||
@@ -250,6 +250,7 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
agent_config_version_kind="snapshot",
|
||||
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
agent_soul_prompt=soul_prompt or None,
|
||||
workflow_node_job_prompt=workflow_job_prompt,
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
|
||||
|
||||
The primary threshold aggregates statement and branch coverage for every Dify
|
||||
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
|
||||
that only contain narrow integration hooks are checked with changed-line
|
||||
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TypedDict, cast
|
||||
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
|
||||
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
|
||||
{
|
||||
"api/dev/check_knowledge_fs_coverage.py",
|
||||
"api/dev/generate_knowledge_fs_contract.py",
|
||||
"api/dev/knowledge_fs_product_contract.py",
|
||||
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
|
||||
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
|
||||
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
|
||||
}
|
||||
)
|
||||
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoverageSummary(TypedDict):
|
||||
"""Coverage.py counts required by the aggregate gate."""
|
||||
|
||||
covered_lines: int
|
||||
num_statements: int
|
||||
covered_branches: int
|
||||
num_branches: int
|
||||
|
||||
|
||||
class CoverageFile(TypedDict):
|
||||
"""Per-file coverage data emitted by ``coverage json``."""
|
||||
|
||||
executed_lines: list[int]
|
||||
missing_lines: list[int]
|
||||
summary: CoverageSummary
|
||||
|
||||
|
||||
class CoverageReport(TypedDict):
|
||||
"""Relevant top-level shape of a coverage.py JSON report."""
|
||||
|
||||
files: dict[str, CoverageFile]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CoverageTotals:
|
||||
"""Covered and measurable units for one gate surface."""
|
||||
|
||||
covered: int
|
||||
total: int
|
||||
|
||||
@property
|
||||
def percent(self) -> float:
|
||||
return 100.0 if self.total == 0 else self.covered * 100 / self.total
|
||||
|
||||
|
||||
class CoverageGateError(RuntimeError):
|
||||
"""Raised when coverage input is incomplete or below its threshold."""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Validate focused module coverage and changed integration glue."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--coverage-json", type=Path, required=True)
|
||||
parser.add_argument("--glue-manifest", type=Path, required=True)
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--base", default="")
|
||||
parser.add_argument("--minimum", type=float, default=90.0)
|
||||
parser.add_argument("--glue-minimum", type=float, default=90.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
report = load_coverage_report(args.coverage_json)
|
||||
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
|
||||
base = resolve_diff_base(workspace_root, args.base)
|
||||
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
|
||||
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
|
||||
glue_totals = validate_changed_glue_coverage(
|
||||
report,
|
||||
changed_lines=changed_lines,
|
||||
minimum=args.glue_minimum,
|
||||
)
|
||||
logger.info(
|
||||
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
|
||||
core_totals.percent,
|
||||
core_totals.covered,
|
||||
core_totals.total,
|
||||
glue_totals.percent,
|
||||
glue_totals.covered,
|
||||
glue_totals.total,
|
||||
)
|
||||
|
||||
|
||||
def load_coverage_report(path: Path) -> CoverageReport:
|
||||
"""Load the detailed JSON report used by both coverage checks."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"coverage JSON does not exist: {path}")
|
||||
document = json.loads(path.read_text())
|
||||
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
|
||||
raise CoverageGateError(f"coverage JSON has no files object: {path}")
|
||||
return cast(CoverageReport, document)
|
||||
|
||||
|
||||
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
|
||||
try:
|
||||
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
|
||||
except UnicodeDecodeError as error:
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
|
||||
if len(paths) != len(set(paths)):
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
|
||||
invalid_paths = [
|
||||
candidate
|
||||
for candidate in paths
|
||||
if not candidate.startswith("api/")
|
||||
or not candidate.endswith(".py")
|
||||
or not (workspace_root / candidate).is_file()
|
||||
]
|
||||
if invalid_paths:
|
||||
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
|
||||
return paths
|
||||
|
||||
|
||||
def is_core_coverage_path(path: str) -> bool:
|
||||
"""Return whether a repository-relative path belongs to the focused aggregate."""
|
||||
if not path.endswith(".py"):
|
||||
return False
|
||||
if path in {
|
||||
"api/commands/knowledge_fs.py",
|
||||
"api/configs/extra/knowledge_fs_config.py",
|
||||
"api/extensions/ext_knowledge_fs_observability.py",
|
||||
"api/services/knowledge_fs_capability.py",
|
||||
}:
|
||||
return True
|
||||
if path.startswith(
|
||||
(
|
||||
"api/controllers/console/knowledge_fs/",
|
||||
"api/controllers/service_api/knowledge_fs/",
|
||||
"api/core/tools/builtin_tool/providers/knowledge_fs/",
|
||||
"api/services/knowledge_fs/",
|
||||
)
|
||||
):
|
||||
return True
|
||||
filename = path.rsplit("/", maxsplit=1)[-1]
|
||||
return (
|
||||
path.startswith("api/models/")
|
||||
and filename.startswith("knowledge_fs")
|
||||
or path.startswith("api/repositories/")
|
||||
and "knowledge_fs" in filename
|
||||
or path.startswith("api/tasks/")
|
||||
and "knowledge_fs" in filename
|
||||
)
|
||||
|
||||
|
||||
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Classify every KnowledgeFS-named production file or fail closed."""
|
||||
named_paths = discover_knowledge_fs_production_paths(workspace_root)
|
||||
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
|
||||
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
|
||||
if unclassified_paths:
|
||||
raise CoverageGateError(
|
||||
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
|
||||
+ ", ".join(sorted(unclassified_paths))
|
||||
)
|
||||
if not core_paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
|
||||
return tuple(sorted(core_paths))
|
||||
|
||||
|
||||
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
|
||||
api_root = workspace_root / "api"
|
||||
if not api_root.is_dir():
|
||||
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
|
||||
paths: set[str] = set()
|
||||
for directory, child_directories, filenames in os.walk(api_root):
|
||||
current_directory = Path(directory)
|
||||
if current_directory == api_root:
|
||||
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
|
||||
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
|
||||
for filename in filenames:
|
||||
path = (current_directory / filename).relative_to(workspace_root).as_posix()
|
||||
if filename.endswith(".py") and "knowledge_fs" in path:
|
||||
paths.add(path)
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
|
||||
return tuple(sorted(paths))
|
||||
|
||||
|
||||
def validate_core_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
workspace_root: Path,
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require the exact combined line-and-branch percentage for all core files."""
|
||||
paths = discover_core_coverage_paths(workspace_root)
|
||||
missing_paths = [path for path in paths if path not in report["files"]]
|
||||
if missing_paths:
|
||||
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
|
||||
|
||||
covered = 0
|
||||
total = 0
|
||||
for path in paths:
|
||||
summary = report["files"][path]["summary"]
|
||||
covered += summary["covered_lines"] + summary["covered_branches"]
|
||||
total += summary["num_statements"] + summary["num_branches"]
|
||||
if total == 0:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
|
||||
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
|
||||
base = preferred.strip()
|
||||
if not base or set(base) == {"0"}:
|
||||
base = "HEAD^"
|
||||
result = subprocess.run(
|
||||
["git", "cat-file", "-e", f"{base}^{{commit}}"],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "commit is unavailable"
|
||||
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
|
||||
return base
|
||||
|
||||
|
||||
def collect_changed_glue_lines(
|
||||
workspace_root: Path,
|
||||
base: str,
|
||||
*,
|
||||
glue_paths: tuple[str, ...],
|
||||
) -> dict[str, set[int]]:
|
||||
"""Return added line numbers in the narrow Dify modules touched by this integration."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-color",
|
||||
"--unified=0",
|
||||
base,
|
||||
"--",
|
||||
*glue_paths,
|
||||
],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "git diff failed"
|
||||
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
|
||||
return parse_added_lines(result.stdout)
|
||||
|
||||
|
||||
def parse_added_lines(diff: str) -> dict[str, set[int]]:
|
||||
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
|
||||
changed_lines: dict[str, set[int]] = {}
|
||||
current_path: str | None = None
|
||||
current_line: int | None = None
|
||||
for raw_line in diff.splitlines():
|
||||
if raw_line.startswith("diff --git "):
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("+++ "):
|
||||
candidate = raw_line[4:]
|
||||
current_path = candidate[2:] if candidate.startswith("b/") else None
|
||||
if current_path is not None:
|
||||
changed_lines.setdefault(current_path, set())
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("@@ "):
|
||||
match = HUNK_HEADER.match(raw_line)
|
||||
current_line = int(match.group(1)) if match is not None else None
|
||||
continue
|
||||
if current_path is None or current_line is None:
|
||||
continue
|
||||
if raw_line.startswith("+"):
|
||||
changed_lines[current_path].add(current_line)
|
||||
current_line += 1
|
||||
elif raw_line.startswith("-") or raw_line.startswith("\\"):
|
||||
continue
|
||||
else:
|
||||
current_line += 1
|
||||
return changed_lines
|
||||
|
||||
|
||||
def validate_changed_glue_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
changed_lines: dict[str, set[int]],
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require added executable glue lines to be exercised by the focused unit suite."""
|
||||
covered = 0
|
||||
total = 0
|
||||
for path, lines in sorted(changed_lines.items()):
|
||||
if not lines:
|
||||
continue
|
||||
file_coverage = report["files"].get(path)
|
||||
if file_coverage is None:
|
||||
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
|
||||
executed_lines = set(file_coverage["executed_lines"])
|
||||
executable_lines = executed_lines | set(file_coverage["missing_lines"])
|
||||
changed_executable_lines = lines & executable_lines
|
||||
covered += len(changed_executable_lines & executed_lines)
|
||||
total += len(changed_executable_lines)
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
|
||||
if totals.percent + 1e-12 < minimum:
|
||||
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,732 @@
|
||||
"""Pin the in-repository KnowledgeFS contract and validate every Dify product operation.
|
||||
|
||||
The lock is intentionally independent of the enclosing Dify commit: it records the staged ``knowledge-fs/`` tree,
|
||||
the complete generated OpenAPI document, both explicit product-operation manifests, and the active Capability v2
|
||||
profile and deterministic public-key vector. The full OpenAPI hash
|
||||
covers request/response schemas, status codes, security, deprecation, and stream metadata. Field-level validation
|
||||
cross-checks the Dify product registry, Python Capability issuer, TypeScript request guard, and exported OpenAPI;
|
||||
each product operation must be ready or an explicit gap, and KFS-only activation remains explicitly internal.
|
||||
|
||||
Contract export reads the working tree only after proving it matches the staged KnowledgeFS index. This keeps the
|
||||
OpenAPI bytes and auth manifest aligned with the exact subtree tree ID that will be reviewed and committed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict, cast
|
||||
|
||||
import jwt
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
|
||||
from jwt.algorithms import RSAAlgorithm
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from dev.knowledge_fs_product_contract import (
|
||||
capability_operation_runtime_contracts,
|
||||
parse_capability_operation_policy,
|
||||
parse_product_operation_gap_manifest,
|
||||
parse_product_operation_manifest,
|
||||
product_operation_runtime_contracts,
|
||||
validate_product_operation_contracts,
|
||||
)
|
||||
|
||||
WORKSPACE_ROOT = API_ROOT.parent
|
||||
KNOWLEDGE_FS_DIRECTORY = "knowledge-fs"
|
||||
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
|
||||
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
|
||||
UPSTREAM_PROVENANCE_RELATIVE_PATH = Path("upstream-provenance.json")
|
||||
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
|
||||
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
|
||||
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
LOCK_SCHEMA_VERSION = 5
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
"""KnowledgeFS transport contract declared by one Dify Console registry entry."""
|
||||
|
||||
operation_id: str
|
||||
method: str
|
||||
path: str
|
||||
required_scope: str | None
|
||||
response_kind: str
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
class ContractLock(TypedDict):
|
||||
"""Content-addressed contract inputs that must move together."""
|
||||
|
||||
schemaVersion: int
|
||||
subtreeTree: str
|
||||
openapiSha256: str
|
||||
capabilityV2AuthManifestSha256: str
|
||||
capabilityV2AuthTestVectorSha256: str
|
||||
productOperationManifestSha256: str
|
||||
productOperationGapManifestSha256: str
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
]
|
||||
|
||||
DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Update or verify the monorepo pin and validate Dify product declarations."""
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--update-lock", action="store_true")
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
knowledge_fs_root = workspace_root / KNOWLEDGE_FS_DIRECTORY
|
||||
lock_path = workspace_root / LOCK_RELATIVE_PATH
|
||||
capability_v2_auth_manifest_path = knowledge_fs_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH
|
||||
capability_v2_auth_test_vector_path = knowledge_fs_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH
|
||||
product_operations_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
|
||||
product_operation_gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
|
||||
upstream_provenance_path = knowledge_fs_root / UPSTREAM_PROVENANCE_RELATIVE_PATH
|
||||
ensure_clean_knowledge_fs_worktree(workspace_root)
|
||||
ensure_contract_inputs_exist(
|
||||
knowledge_fs_root=knowledge_fs_root,
|
||||
capability_v2_auth_manifest_path=capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path=capability_v2_auth_test_vector_path,
|
||||
product_operations_path=product_operations_path,
|
||||
product_operation_gaps_path=product_operation_gaps_path,
|
||||
upstream_provenance_path=upstream_provenance_path,
|
||||
)
|
||||
|
||||
subtree_tree = staged_subtree_tree(workspace_root)
|
||||
capability_v2_auth_manifest_content = capability_v2_auth_manifest_path.read_bytes()
|
||||
capability_v2_auth_test_vector_content = capability_v2_auth_test_vector_path.read_bytes()
|
||||
product_operation_manifest_content = product_operations_path.read_bytes()
|
||||
product_operation_gap_manifest_content = product_operation_gaps_path.read_bytes()
|
||||
capability_v2_auth_manifest = load_json_object(capability_v2_auth_manifest_path)
|
||||
validate_capability_v2_auth_manifest(capability_v2_auth_manifest)
|
||||
validate_capability_v2_auth_test_vector(
|
||||
load_json_object(capability_v2_auth_test_vector_path),
|
||||
capability_v2_auth_manifest,
|
||||
)
|
||||
validate_upstream_provenance(load_json_object(upstream_provenance_path))
|
||||
product_manifest = parse_product_operation_manifest(load_json_object(product_operations_path))
|
||||
product_gap_manifest = parse_product_operation_gap_manifest(load_json_object(product_operation_gaps_path))
|
||||
product_runtime_operations = product_operation_runtime_contracts()
|
||||
capability_runtime_operations = capability_operation_runtime_contracts()
|
||||
declarations = console_contract_declarations()
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
|
||||
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
|
||||
capability_policy_path = Path(directory) / "dify-capability-v2-operations.json"
|
||||
subprocess.run(
|
||||
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
|
||||
cwd=knowledge_fs_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["pnpm", "capability:export", "--", "--output", str(capability_policy_path)],
|
||||
cwd=knowledge_fs_root,
|
||||
check=True,
|
||||
)
|
||||
openapi_content = openapi_path.read_bytes()
|
||||
capability_policy = parse_capability_operation_policy(load_json_object(capability_policy_path))
|
||||
|
||||
document: dict[str, Any] = json.loads(openapi_content)
|
||||
validate_product_operation_contracts(
|
||||
capability_operations=capability_runtime_operations,
|
||||
capability_policy=capability_policy,
|
||||
document=document,
|
||||
gap_manifest=product_gap_manifest,
|
||||
manifest=product_manifest,
|
||||
product_operations=product_runtime_operations,
|
||||
)
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
expected_lock: ContractLock = {
|
||||
"schemaVersion": LOCK_SCHEMA_VERSION,
|
||||
"subtreeTree": subtree_tree,
|
||||
"openapiSha256": sha256(openapi_content),
|
||||
"capabilityV2AuthManifestSha256": sha256(capability_v2_auth_manifest_content),
|
||||
"capabilityV2AuthTestVectorSha256": sha256(capability_v2_auth_test_vector_content),
|
||||
"productOperationManifestSha256": sha256(product_operation_manifest_content),
|
||||
"productOperationGapManifestSha256": sha256(product_operation_gap_manifest_content),
|
||||
}
|
||||
|
||||
if args.update_lock:
|
||||
lock_path.write_text(json.dumps(expected_lock, indent=2) + "\n")
|
||||
return
|
||||
|
||||
received_lock = parse_contract_lock(load_json_object(lock_path))
|
||||
lock_fields = (
|
||||
(
|
||||
"capabilityV2AuthManifestSha256",
|
||||
received_lock["capabilityV2AuthManifestSha256"],
|
||||
expected_lock["capabilityV2AuthManifestSha256"],
|
||||
),
|
||||
(
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
received_lock["capabilityV2AuthTestVectorSha256"],
|
||||
expected_lock["capabilityV2AuthTestVectorSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationManifestSha256",
|
||||
received_lock["productOperationManifestSha256"],
|
||||
expected_lock["productOperationManifestSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationGapManifestSha256",
|
||||
received_lock["productOperationGapManifestSha256"],
|
||||
expected_lock["productOperationGapManifestSha256"],
|
||||
),
|
||||
("schemaVersion", received_lock["schemaVersion"], expected_lock["schemaVersion"]),
|
||||
("subtreeTree", received_lock["subtreeTree"], expected_lock["subtreeTree"]),
|
||||
("openapiSha256", received_lock["openapiSha256"], expected_lock["openapiSha256"]),
|
||||
)
|
||||
for field, received_value, expected_value in lock_fields:
|
||||
if received_value != expected_value:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS contract lock field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}. "
|
||||
"Run --update-lock intentionally after reviewing the staged subtree and contract changes."
|
||||
)
|
||||
|
||||
|
||||
def ensure_contract_inputs_exist(
|
||||
*,
|
||||
knowledge_fs_root: Path,
|
||||
capability_v2_auth_manifest_path: Path,
|
||||
capability_v2_auth_test_vector_path: Path,
|
||||
product_operations_path: Path,
|
||||
product_operation_gaps_path: Path,
|
||||
upstream_provenance_path: Path,
|
||||
) -> None:
|
||||
"""Fail with a stable error before invoking package tooling when a contract input is absent."""
|
||||
required_paths = (
|
||||
knowledge_fs_root / "package.json",
|
||||
capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path,
|
||||
product_operations_path,
|
||||
product_operation_gaps_path,
|
||||
upstream_provenance_path,
|
||||
)
|
||||
missing_paths = [path for path in required_paths if not path.is_file()]
|
||||
if missing_paths:
|
||||
missing = ", ".join(str(path) for path in missing_paths)
|
||||
raise RuntimeError(f"KnowledgeFS contract input is missing: {missing}")
|
||||
|
||||
|
||||
def ensure_clean_knowledge_fs_worktree(workspace_root: Path) -> None:
|
||||
"""Require exported KnowledgeFS files to exactly match the staged index tree.
|
||||
|
||||
Staged changes are expected during intentional lock updates. Unstaged tracked changes and untracked files are
|
||||
rejected because the export process reads the working tree while the tree ID is calculated from the index.
|
||||
"""
|
||||
subtree_path = f"{KNOWLEDGE_FS_DIRECTORY}/"
|
||||
unstaged = subprocess.run(
|
||||
["git", "diff", "--quiet", "--", subtree_path],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
)
|
||||
if unstaged.returncode > 1:
|
||||
raise RuntimeError("git diff failed while validating the staged KnowledgeFS subtree")
|
||||
untracked = run(
|
||||
"git",
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"--",
|
||||
subtree_path,
|
||||
cwd=workspace_root,
|
||||
).strip()
|
||||
if unstaged.returncode != 0 or untracked:
|
||||
raise RuntimeError(
|
||||
"knowledge-fs/ contains unstaged or untracked changes; stage or remove them before contract export"
|
||||
)
|
||||
|
||||
|
||||
def staged_subtree_tree(workspace_root: Path) -> str:
|
||||
"""Return the Git tree object for the staged ``knowledge-fs/`` subtree."""
|
||||
return run("git", "write-tree", f"--prefix={KNOWLEDGE_FS_DIRECTORY}/", cwd=workspace_root).strip()
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
|
||||
return cast(dict[str, Any], value)
|
||||
|
||||
|
||||
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
|
||||
"""Validate the compact, non-self-referential contract lock schema."""
|
||||
expected_fields = {
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"schemaVersion",
|
||||
"subtreeTree",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
|
||||
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
|
||||
for field in (
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"subtreeTree",
|
||||
):
|
||||
field_value = value.get(field)
|
||||
expected_length = 40 if field == "subtreeTree" else 64
|
||||
if (
|
||||
not isinstance(field_value, str)
|
||||
or len(field_value) != expected_length
|
||||
or any(character not in "0123456789abcdef" for character in field_value)
|
||||
):
|
||||
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
|
||||
return cast(ContractLock, value)
|
||||
|
||||
|
||||
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
|
||||
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
|
||||
available_operation_ids = [
|
||||
operation_id
|
||||
for path_item in document.get("paths", {}).values()
|
||||
for method in OPENAPI_METHODS
|
||||
if isinstance(path_item, dict)
|
||||
for operation in (path_item.get(method),)
|
||||
if isinstance(operation, dict)
|
||||
for operation_id in (operation.get("operationId"),)
|
||||
if isinstance(operation_id, str) and operation_id
|
||||
]
|
||||
for operation_id in required_operation_ids:
|
||||
count = available_operation_ids.count(operation_id)
|
||||
if count != 1:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
|
||||
)
|
||||
|
||||
|
||||
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
|
||||
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
|
||||
expected_fields = {
|
||||
"active",
|
||||
"audience",
|
||||
"callerProfiles",
|
||||
"claimBindings",
|
||||
"issuer",
|
||||
"lifecycle",
|
||||
"maxTtlSeconds",
|
||||
"productionReady",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"requiredClaims",
|
||||
"resourceContract",
|
||||
"runtimeAssembly",
|
||||
"schemaVersion",
|
||||
"signatureAlgorithms",
|
||||
"tokenKind",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
|
||||
fixed_values = {
|
||||
"active": True,
|
||||
"audience": "knowledge-fs",
|
||||
"issuer": "dify-control-plane",
|
||||
"lifecycle": "active",
|
||||
"maxTtlSeconds": 60,
|
||||
"productionReady": True,
|
||||
"profileId": "dify-capability-v2",
|
||||
"schemaVersion": 3,
|
||||
"signatureAlgorithms": ["RS256"],
|
||||
"tokenKind": "jwt",
|
||||
}
|
||||
for field, expected in fixed_values.items():
|
||||
if value.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
|
||||
if value.get("protectedHeader") != {
|
||||
"algorithm": "RS256",
|
||||
"keyIdClaim": "kid",
|
||||
"keyIdRequired": True,
|
||||
"type": "JWT",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
|
||||
required_claims = [
|
||||
"action",
|
||||
"actor",
|
||||
"aud",
|
||||
"authz_revision",
|
||||
"azp",
|
||||
"caller_kind",
|
||||
"cap_ver",
|
||||
"content_policy_revision",
|
||||
"content_scope_ids",
|
||||
"control_space_id",
|
||||
"exp",
|
||||
"grant_id",
|
||||
"iat",
|
||||
"iss",
|
||||
"jti",
|
||||
"namespace_id",
|
||||
"nbf",
|
||||
"resource",
|
||||
"sub",
|
||||
"trace_id",
|
||||
]
|
||||
if value.get("requiredClaims") != required_claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
|
||||
if value.get("claimBindings") != {
|
||||
"action": "action",
|
||||
"callerKind": "caller_kind",
|
||||
"controlSpace": "control_space_id",
|
||||
"namespace": "namespace_id",
|
||||
"resource": "resource",
|
||||
"resourceParent": "resource.parent_id",
|
||||
"subject": "sub",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
|
||||
if value.get("resourceContract") != {
|
||||
"fields": ["id", "parent_id", "type"],
|
||||
"parentForbiddenFor": ["namespace", "knowledge_space"],
|
||||
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
|
||||
if value.get("callerProfiles") != {
|
||||
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
|
||||
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
|
||||
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
|
||||
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
|
||||
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-kfs-credential:"},
|
||||
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
|
||||
if value.get("runtimeAssembly") != {
|
||||
"failClosed": True,
|
||||
"keySelection": "kid",
|
||||
"maximumPublishedKeys": 3,
|
||||
"verificationKeySource": "jwks",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
|
||||
|
||||
|
||||
def validate_capability_v2_auth_test_vector(
|
||||
value: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Verify the deterministic public-key vector and every security-sensitive binding."""
|
||||
expected_fields = {
|
||||
"algorithm",
|
||||
"audience",
|
||||
"expectedClaims",
|
||||
"expectedPrincipal",
|
||||
"issuer",
|
||||
"operation",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"publicJwk",
|
||||
"schemaVersion",
|
||||
"testOnly",
|
||||
"token",
|
||||
"ttlSeconds",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
|
||||
if (
|
||||
value.get("schemaVersion") != 2
|
||||
or value.get("profileId") != manifest.get("profileId")
|
||||
or value.get("testOnly") is not True
|
||||
or value.get("algorithm") != "RS256"
|
||||
or value.get("issuer") != manifest.get("issuer")
|
||||
or value.get("audience") != manifest.get("audience")
|
||||
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
|
||||
protected_header = value.get("protectedHeader")
|
||||
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
kid = protected_header.get("kid")
|
||||
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
public_jwk = value.get("publicJwk")
|
||||
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
if (
|
||||
public_jwk.get("alg") != "RS256"
|
||||
or public_jwk.get("kid") != kid
|
||||
or public_jwk.get("kty") != "RSA"
|
||||
or public_jwk.get("use") != "sig"
|
||||
or not _is_non_blank(public_jwk.get("e"))
|
||||
or not _is_non_blank(public_jwk.get("n"))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
|
||||
claims = value.get("expectedClaims")
|
||||
required_claims = manifest.get("requiredClaims")
|
||||
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
|
||||
operation = value.get("operation")
|
||||
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
|
||||
resource = claims.get("resource")
|
||||
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
|
||||
expected_operation = {
|
||||
"action": "documents.read",
|
||||
"method": "GET",
|
||||
"operationId": "getDocument",
|
||||
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
|
||||
}
|
||||
if operation != expected_operation:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
|
||||
exact_claims = {
|
||||
"action": operation["action"],
|
||||
"aud": value["audience"],
|
||||
"caller_kind": "interactive",
|
||||
"cap_ver": 2,
|
||||
"control_space_id": "control-space-contract-vector",
|
||||
"iss": value["issuer"],
|
||||
"namespace_id": "workspace-contract-vector",
|
||||
"resource": {
|
||||
"id": "document-contract-vector",
|
||||
"parent_id": "space-contract-vector",
|
||||
"type": "document",
|
||||
},
|
||||
"sub": "dify-account:account-contract-vector",
|
||||
}
|
||||
for field, expected in exact_claims.items():
|
||||
if claims.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
|
||||
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
|
||||
issued_at = claims.get("iat")
|
||||
not_before = claims.get("nbf")
|
||||
expires_at = claims.get("exp")
|
||||
if (
|
||||
not isinstance(issued_at, int)
|
||||
or isinstance(issued_at, bool)
|
||||
or not_before != issued_at
|
||||
or not isinstance(expires_at, int)
|
||||
or isinstance(expires_at, bool)
|
||||
or expires_at - issued_at != value["ttlSeconds"]
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
|
||||
expected_principal = {
|
||||
"callerKind": claims["caller_kind"],
|
||||
"subject": {
|
||||
"scopes": ["knowledge-spaces:read"],
|
||||
"subjectId": claims["sub"],
|
||||
"tenantId": claims["namespace_id"],
|
||||
},
|
||||
}
|
||||
if value.get("expectedPrincipal") != expected_principal:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
|
||||
|
||||
token = value.get("token")
|
||||
if not isinstance(token, str) or not _is_non_blank(token):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
|
||||
try:
|
||||
verification_key = RSAAlgorithm.from_jwk(public_jwk)
|
||||
if not isinstance(verification_key, RSAPublicKey):
|
||||
raise ValueError("Capability vector verification key is not RSA public material")
|
||||
header = jwt.get_unverified_header(token)
|
||||
decoded_claims = jwt.decode(
|
||||
token,
|
||||
verification_key,
|
||||
algorithms=["RS256"],
|
||||
audience=cast(str, value["audience"]),
|
||||
issuer=cast(str, value["issuer"]),
|
||||
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
|
||||
)
|
||||
except (jwt.PyJWTError, TypeError, ValueError) as exc:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
|
||||
if header != protected_header or decoded_claims != claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
|
||||
|
||||
|
||||
def _is_non_blank(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
|
||||
|
||||
|
||||
def validate_upstream_provenance(value: dict[str, Any]) -> None:
|
||||
"""Validate the imported-source provenance that is itself covered by the subtree tree ID."""
|
||||
expected_fields = {"commit", "release", "repository", "schemaVersion"}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS upstream provenance fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS upstream provenance must use schemaVersion 1")
|
||||
repository = value.get("repository")
|
||||
commit = value.get("commit")
|
||||
if not isinstance(repository, str) or not repository.startswith("https://"):
|
||||
raise ValueError("KnowledgeFS upstream provenance repository must be an HTTPS URL")
|
||||
if (
|
||||
not isinstance(commit, str)
|
||||
or len(commit) != 40
|
||||
or any(character not in "0123456789abcdef" for character in commit)
|
||||
):
|
||||
raise ValueError("KnowledgeFS upstream provenance commit must be a lowercase full Git SHA")
|
||||
if value.get("release") is not None and not isinstance(value["release"], str):
|
||||
raise ValueError("KnowledgeFS upstream provenance release must be null or a string")
|
||||
|
||||
|
||||
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
|
||||
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
|
||||
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
for method in OPENAPI_METHODS:
|
||||
operation = path_item.get(method)
|
||||
if operation is None:
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if isinstance(operation_id, str) and operation_id:
|
||||
operations_by_id.setdefault(operation_id, []).append((method, path, path_item, operation))
|
||||
|
||||
declared_ids: set[str] = set()
|
||||
for declaration in declarations:
|
||||
operation_id = declaration["operation_id"]
|
||||
if operation_id in declared_ids:
|
||||
raise ValueError(f"Dify Console registry has duplicate operationId: {operation_id}")
|
||||
declared_ids.add(operation_id)
|
||||
|
||||
matches = operations_by_id.get(operation_id, [])
|
||||
if not matches:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has no operationId: {operation_id}")
|
||||
if len(matches) > 1:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has duplicate operationId: {operation_id}")
|
||||
|
||||
method, path, path_item, operation = matches[0]
|
||||
if not path.startswith("/"):
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: ContractDeclaration = {
|
||||
"operation_id": operation_id,
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(document, operation),
|
||||
"response_kind": response_kind(operation),
|
||||
"max_response_bytes": required_max_response_bytes(operation),
|
||||
"request_headers": request_header_names(path_item, operation),
|
||||
"response_headers": response_header_names(operation),
|
||||
"response_media_types": response_media_types(operation),
|
||||
}
|
||||
for field in DECLARATION_FIELDS:
|
||||
expected_value = expected[field]
|
||||
received_value = declaration[field]
|
||||
if received_value != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
|
||||
|
||||
return ()
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
media_types = response_media_types(operation)
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
media_types: set[str] = set()
|
||||
for status, response in operation.get("responses", {}).items():
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit()):
|
||||
media_types.update(response.get("content", {}))
|
||||
return tuple(sorted(media_types))
|
||||
|
||||
|
||||
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
|
||||
scope = operation.get("x-knowledge-fs-required-scope")
|
||||
security = operation["security"] if "security" in operation else document.get("security")
|
||||
if security == []:
|
||||
if scope is not None:
|
||||
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
|
||||
return None
|
||||
if security != [{"bearerAuth": []}]:
|
||||
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
|
||||
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
|
||||
return scope
|
||||
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
|
||||
|
||||
|
||||
def required_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError(f"KnowledgeFS operation has no valid response byte limit: {value}")
|
||||
return value
|
||||
|
||||
|
||||
def request_header_names(path_item: dict[str, Any], operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
names: set[str] = set()
|
||||
for parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]:
|
||||
if "$ref" in parameter:
|
||||
raise ValueError(f"KnowledgeFS request header references are not supported: {parameter['$ref']}")
|
||||
if parameter.get("in") == "header":
|
||||
names.add(parameter["name"].lower())
|
||||
return tuple(sorted(names))
|
||||
|
||||
|
||||
def response_header_names(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
name.lower()
|
||||
for response in operation.get("responses", {}).values()
|
||||
for name in response.get("headers", {})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def sha256(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def run(*command: str, cwd: Path) -> str:
|
||||
return subprocess.run(command, cwd=cwd, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Cross-service KnowledgeFS product-operation contract validation.
|
||||
|
||||
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
|
||||
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
|
||||
policy remain executable sources of truth. A product operation must appear exactly once as ready
|
||||
or as an explicit gap; internal KFS-only operations require a named exclusion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, Literal, NamedTuple, TypedDict, cast
|
||||
|
||||
|
||||
class ProductOperationLimits(TypedDict):
|
||||
kfsMaxResponseBytes: int
|
||||
productMaxRequestBytes: int
|
||||
productMaxResponseBytes: int
|
||||
|
||||
|
||||
class ProductOperationStream(TypedDict):
|
||||
kfsResponseKind: str
|
||||
productKind: str
|
||||
|
||||
|
||||
class ProductOperationManifestEntry(TypedDict):
|
||||
action: str
|
||||
kfsOperationId: str
|
||||
limits: ProductOperationLimits
|
||||
method: str
|
||||
path: str
|
||||
productOperationId: str
|
||||
resource: str
|
||||
stream: ProductOperationStream
|
||||
transport: str
|
||||
|
||||
|
||||
type ProductOperationManifestField = Literal[
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
]
|
||||
|
||||
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
)
|
||||
|
||||
|
||||
class ProductOperationManifest(TypedDict):
|
||||
operations: list[ProductOperationManifestEntry]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationGapEntry(ProductOperationManifestEntry):
|
||||
reason: str
|
||||
reasonCode: str
|
||||
replacementProductOperationIds: list[str]
|
||||
|
||||
|
||||
class InternalKfsOperationExclusion(TypedDict):
|
||||
kfsOperationId: str
|
||||
reason: str
|
||||
reasonCode: str
|
||||
|
||||
|
||||
class ProductOperationGapManifest(TypedDict):
|
||||
gaps: list[ProductOperationGapEntry]
|
||||
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
kfs_operation_id: str
|
||||
max_request_bytes: int
|
||||
max_response_bytes: int
|
||||
method: str
|
||||
path: str
|
||||
product_operation_id: str
|
||||
ready: bool
|
||||
resource: str
|
||||
stream_kind: str
|
||||
transport: str
|
||||
|
||||
|
||||
class CapabilityOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
path: str
|
||||
resource: str
|
||||
|
||||
|
||||
class CapabilityOperationPolicy(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
parent_resource_binding: dict[str, object] | None
|
||||
path: str
|
||||
resource: str
|
||||
resource_binding: dict[str, object]
|
||||
|
||||
|
||||
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
|
||||
"""Read the complete typed Dify product registry without normalizing away gaps."""
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
|
||||
contracts: list[ProductOperationRuntimeContract] = []
|
||||
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
|
||||
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
|
||||
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
|
||||
contracts.append(
|
||||
ProductOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
kfs_operation_id=operation.capability_operation_id,
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
product_operation_id=product_operation_id,
|
||||
ready=is_product_operation_ready(product_operation_id),
|
||||
resource=operation.resource_resolver,
|
||||
stream_kind=operation.stream_kind,
|
||||
transport=operation.transport,
|
||||
)
|
||||
)
|
||||
return tuple(contracts)
|
||||
|
||||
|
||||
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
|
||||
"""Read the Python issuer policy used to mint each operation Capability."""
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
CapabilityOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
allowed_caller_kinds=operation.allowed_caller_kinds,
|
||||
method=operation.method,
|
||||
operation_id=operation_id,
|
||||
path=operation.path,
|
||||
resource=operation.resource_type,
|
||||
)
|
||||
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
|
||||
)
|
||||
|
||||
|
||||
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
|
||||
"""Parse the ready product-operation inventory with a closed schema."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
|
||||
return {
|
||||
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
|
||||
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
|
||||
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
|
||||
if set(value) != expected_fields or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
|
||||
gaps = value.get("gaps")
|
||||
exclusions = value.get("internalKfsOperationExclusions")
|
||||
if not isinstance(gaps, list) or not gaps or not isinstance(exclusions, list):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
|
||||
parsed_exclusions: list[InternalKfsOperationExclusion] = []
|
||||
for raw in exclusions:
|
||||
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
|
||||
for field in ("kfsOperationId", "reason", "reasonCode"):
|
||||
_required_string(raw.get(field), f"internal exclusion {field}")
|
||||
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
|
||||
return {
|
||||
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
|
||||
"internalKfsOperationExclusions": parsed_exclusions,
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
|
||||
"""Parse the runtime-exported TypeScript request-guard registry."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
|
||||
parsed: list[CapabilityOperationPolicy] = []
|
||||
for raw in operations:
|
||||
expected_fields = {
|
||||
"action",
|
||||
"allowedCallerKinds",
|
||||
"method",
|
||||
"operationId",
|
||||
"parentResourceBinding",
|
||||
"path",
|
||||
"resourceBinding",
|
||||
"resourceType",
|
||||
}
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
|
||||
callers = raw.get("allowedCallerKinds")
|
||||
resource_binding = raw.get("resourceBinding")
|
||||
parent_binding = raw.get("parentResourceBinding")
|
||||
if (
|
||||
not isinstance(callers, list)
|
||||
or not callers
|
||||
or not all(isinstance(caller, str) and caller for caller in callers)
|
||||
or not isinstance(resource_binding, dict)
|
||||
or not resource_binding
|
||||
or (parent_binding is not None and not isinstance(parent_binding, dict))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
|
||||
for field in ("action", "method", "operationId", "path", "resourceType"):
|
||||
_required_string(raw.get(field), f"Capability operation {field}")
|
||||
parsed.append(
|
||||
CapabilityOperationPolicy(
|
||||
action=cast(str, raw["action"]),
|
||||
allowed_caller_kinds=tuple(callers),
|
||||
method=cast(str, raw["method"]),
|
||||
operation_id=cast(str, raw["operationId"]),
|
||||
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
|
||||
path=cast(str, raw["path"]),
|
||||
resource=cast(str, raw["resourceType"]),
|
||||
resource_binding=cast(dict[str, object], resource_binding),
|
||||
)
|
||||
)
|
||||
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def validate_product_operation_contracts(
|
||||
*,
|
||||
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
|
||||
capability_policy: tuple[CapabilityOperationPolicy, ...],
|
||||
document: dict[str, Any],
|
||||
gap_manifest: ProductOperationGapManifest,
|
||||
manifest: ProductOperationManifest,
|
||||
product_operations: tuple[ProductOperationRuntimeContract, ...],
|
||||
) -> None:
|
||||
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
|
||||
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
|
||||
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
|
||||
if len(ready_entries) != len(manifest["operations"]):
|
||||
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
|
||||
if len(gap_entries) != len(gap_manifest["gaps"]):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
|
||||
if set(ready_entries) & set(gap_entries):
|
||||
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
|
||||
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
|
||||
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
|
||||
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
|
||||
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
|
||||
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
|
||||
|
||||
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
|
||||
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
|
||||
openapi_operations = _openapi_operations(document)
|
||||
for product_operation_id, runtime in runtime_by_product.items():
|
||||
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
|
||||
if runtime.ready != (product_operation_id in ready_entries):
|
||||
state = "ready" if runtime.ready else "gap"
|
||||
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
|
||||
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
|
||||
if product_operation_id in gap_entries:
|
||||
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
|
||||
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
|
||||
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
|
||||
if not replacements or any(replacement not in ready_entries for replacement in replacements):
|
||||
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
|
||||
|
||||
exclusions = gap_manifest["internalKfsOperationExclusions"]
|
||||
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
|
||||
if len(exclusion_by_id) != len(exclusions):
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
|
||||
lifecycle_operation_ids = {
|
||||
"activateDifyWorkspaceIntegration",
|
||||
"freezeDifyWorkspaceIntegration",
|
||||
}
|
||||
if set(exclusion_by_id) != lifecycle_operation_ids:
|
||||
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
|
||||
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
|
||||
for operation_id in exclusion_by_id:
|
||||
if operation_id in product_capability_ids:
|
||||
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
|
||||
_validate_capability_alignment(
|
||||
operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
python_operation = python_capabilities[operation_id]
|
||||
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
|
||||
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
|
||||
|
||||
|
||||
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
|
||||
base_fields = {
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
}
|
||||
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS product operation entry fields are invalid")
|
||||
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
|
||||
_required_string(raw.get(field), f"product operation {field}")
|
||||
limits = raw.get("limits")
|
||||
stream = raw.get("stream")
|
||||
if not isinstance(limits, dict) or set(limits) != {
|
||||
"kfsMaxResponseBytes",
|
||||
"productMaxRequestBytes",
|
||||
"productMaxResponseBytes",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS product operation limits are invalid")
|
||||
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
|
||||
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
|
||||
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
|
||||
raise ValueError("KnowledgeFS product operation stream contract is invalid")
|
||||
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
|
||||
_required_string(stream.get("productKind"), "product stream kind")
|
||||
if gap:
|
||||
_required_string(raw.get("reason"), "product gap reason")
|
||||
_required_string(raw.get("reasonCode"), "product gap reasonCode")
|
||||
replacements = raw.get("replacementProductOperationIds")
|
||||
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
|
||||
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
|
||||
return cast(ProductOperationManifestEntry, raw)
|
||||
|
||||
|
||||
def _validate_product_entry(
|
||||
entry: ProductOperationManifestEntry,
|
||||
runtime: ProductOperationRuntimeContract,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
_validate_capability_alignment(
|
||||
runtime.kfs_operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
|
||||
expected: ProductOperationManifestEntry = {
|
||||
"action": runtime.action,
|
||||
"kfsOperationId": runtime.kfs_operation_id,
|
||||
"limits": {
|
||||
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
|
||||
"productMaxRequestBytes": runtime.max_request_bytes,
|
||||
"productMaxResponseBytes": runtime.max_response_bytes,
|
||||
},
|
||||
"method": runtime.method,
|
||||
"path": runtime.path,
|
||||
"productOperationId": runtime.product_operation_id,
|
||||
"resource": runtime.resource,
|
||||
"stream": {
|
||||
"kfsResponseKind": _openapi_response_kind(openapi_operation),
|
||||
"productKind": runtime.stream_kind,
|
||||
},
|
||||
"transport": runtime.transport,
|
||||
}
|
||||
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
|
||||
expected_value = expected[field]
|
||||
if entry[field] != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {entry[field]!r}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_capability_alignment(
|
||||
operation_id: str,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
python_operation = python_capabilities.get(operation_id)
|
||||
guard_operation = guard_capabilities.get(operation_id)
|
||||
openapi_operation = openapi_operations.get(operation_id)
|
||||
if python_operation is None or guard_operation is None or openapi_operation is None:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
|
||||
)
|
||||
method, path, _ = openapi_operation
|
||||
expected = (
|
||||
python_operation.action,
|
||||
python_operation.method,
|
||||
python_operation.path,
|
||||
python_operation.resource,
|
||||
)
|
||||
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
|
||||
if received_guard != expected:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
|
||||
if (method, path) != (python_operation.method, python_operation.path):
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
|
||||
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
|
||||
|
||||
|
||||
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
|
||||
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
|
||||
duplicates: set[str] = set()
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
if not isinstance(path, str) or not isinstance(path_item, dict):
|
||||
continue
|
||||
for method in ("delete", "get", "patch", "post", "put"):
|
||||
operation = path_item.get(method)
|
||||
if not isinstance(operation, dict):
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if not isinstance(operation_id, str) or not operation_id:
|
||||
continue
|
||||
if operation_id in operations:
|
||||
duplicates.add(operation_id)
|
||||
operations[operation_id] = (method.upper(), path, operation)
|
||||
if duplicates:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
|
||||
return operations
|
||||
|
||||
|
||||
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
|
||||
media_types = {
|
||||
media_type
|
||||
for status, response in operation.get("responses", {}).items()
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
|
||||
for media_type in response.get("content", {})
|
||||
}
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
||||
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
|
||||
return value
|
||||
|
||||
|
||||
def _unique(values: Iterable[str], label: str) -> None:
|
||||
sequence = tuple(values)
|
||||
if len(sequence) != len(set(sequence)):
|
||||
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CapabilityOperationPolicy",
|
||||
"CapabilityOperationRuntimeContract",
|
||||
"ProductOperationGapManifest",
|
||||
"ProductOperationManifest",
|
||||
"ProductOperationRuntimeContract",
|
||||
"capability_operation_runtime_contracts",
|
||||
"parse_capability_operation_policy",
|
||||
"parse_product_operation_gap_manifest",
|
||||
"parse_product_operation_manifest",
|
||||
"product_operation_runtime_contracts",
|
||||
"validate_product_operation_contracts",
|
||||
]
|
||||
@@ -164,6 +164,14 @@ def init_app(app: DifyApp) -> Celery:
|
||||
|
||||
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
|
||||
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
|
||||
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
|
||||
|
||||
if get_configured_knowledge_fs_lifecycle_worker_readiness().ready:
|
||||
imports.append("tasks.knowledge_fs_lifecycle_tasks")
|
||||
beat_schedule["knowledge_fs_lifecycle_worker"] = {
|
||||
"task": "tasks.knowledge_fs_lifecycle_tasks.run_knowledge_fs_lifecycle_worker",
|
||||
"schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
|
||||
}
|
||||
if dify_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK:
|
||||
imports.append("schedule.clean_embedding_cache_task")
|
||||
beat_schedule["clean_embedding_cache_task"] = {
|
||||
|
||||
@@ -27,6 +27,7 @@ def init_app(app: DifyApp):
|
||||
import_migration_data,
|
||||
install_plugins,
|
||||
install_rag_pipeline_plugins,
|
||||
knowledge_fs_control_space,
|
||||
migrate_data_for_plugin,
|
||||
migrate_dataset_permissions_to_rbac,
|
||||
migrate_member_roles_to_rbac,
|
||||
@@ -88,6 +89,7 @@ def init_app(app: DifyApp):
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
knowledge_fs_control_space,
|
||||
]
|
||||
for cmd in cmds_to_register:
|
||||
app.cli.add_command(cmd)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Production assembly for KnowledgeFS database-backed metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
|
||||
)
|
||||
from services.knowledge_fs.observability import get_knowledge_fs_operational_metrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_app(app: Flask) -> None:
|
||||
del app
|
||||
try:
|
||||
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_factory.get_session_maker())
|
||||
get_knowledge_fs_operational_metrics().register_control_space_state_gauge(reader)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeFS operational metric registration failed; continuing application startup")
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "04f9a6189c215f4fcccdccbb3ae7fc810b927463",
|
||||
"openapiSha256": "df702b15769f79c561e34d72472a2ec8dddcdefe4bd4c82858f06a98cd6eae8c",
|
||||
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
"productOperationManifestSha256": "5ceef85419dfba9c803732f78f31c803a0836e883972bb67f26ae43b0c78d341",
|
||||
"productOperationGapManifestSha256": "ccbae37fe658177a77529822211a2cf02e72b5815cbc9b7e7e65e6817ba5e0a9"
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"gaps": [
|
||||
{
|
||||
"productOperationId": "createDocument",
|
||||
"kfsOperationId": "uploadDocument",
|
||||
"method": "POST",
|
||||
"path": "/knowledge-spaces/{id}/documents",
|
||||
"action": "documents.create",
|
||||
"resource": "knowledge_space",
|
||||
"transport": "multipart",
|
||||
"stream": {
|
||||
"productKind": "buffered-multipart",
|
||||
"kfsResponseKind": "buffered"
|
||||
},
|
||||
"limits": {
|
||||
"productMaxRequestBytes": 0,
|
||||
"productMaxResponseBytes": 0,
|
||||
"kfsMaxResponseBytes": 1048576
|
||||
},
|
||||
"reasonCode": "NON_HOMOMORPHIC_MULTIPART_TRANSPORT",
|
||||
"reason": "Legacy buffered multipart document creation has no homomorphic Dify product transport; use Capability v2 direct-upload operations.",
|
||||
"replacementProductOperationIds": [
|
||||
"createUploadSession",
|
||||
"presignUploadSessionPart",
|
||||
"uploadSmallFile",
|
||||
"completeUploadSession",
|
||||
"abortUploadSession"
|
||||
]
|
||||
}
|
||||
],
|
||||
"internalKfsOperationExclusions": [
|
||||
{
|
||||
"kfsOperationId": "activateDifyWorkspaceIntegration",
|
||||
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
|
||||
"reason": "Workspace activation is an internal cutover control-plane operation and is never exposed as a product operation."
|
||||
},
|
||||
{
|
||||
"kfsOperationId": "freezeDifyWorkspaceIntegration",
|
||||
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
|
||||
"reason": "Workspace maintenance freeze is an internal cutover control-plane operation and is never exposed as a product operation."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"operations": [
|
||||
{"productOperationId":"batchSpaceSummaries","kfsOperationId":"batchKnowledgeSpaceProductSummaries","method":"POST","path":"/internal/knowledge-spaces/product-summaries/batch","action":"knowledge_spaces.status.batch","resource":"namespace","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSpace","kfsOperationId":"getKnowledgeSpace","method":"GET","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSpace","kfsOperationId":"updateKnowledgeSpace","method":"PATCH","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSettings","kfsOperationId":"getKnowledgeSpaceProductSettings","method":"GET","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocument","kfsOperationId":"getDocument","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocumentOutline","kfsOperationId":"getDocumentOutline","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/outline","action":"documents.outline.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocumentRevisions","kfsOperationId":"listDocumentRevisions","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions","action":"documents.revisions.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateDocumentMetadata","kfsOperationId":"patchDocumentMetadata","method":"PATCH","path":"/knowledge-spaces/{id}/documents/{documentId}/metadata","action":"documents.metadata.update","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocumentChunks","kfsOperationId":"listDocumentChunks","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks","action":"documents.chunks.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocumentChunk","kfsOperationId":"getDocumentChunk","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}","action":"documents.chunks.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"deleteDocument","kfsOperationId":"requestDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.delete","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"bulkDeleteDocuments","kfsOperationId":"requestBulkDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/bulk","action":"documents.bulk.delete","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"reindexDocuments","kfsOperationId":"bulkReindexDocuments","method":"POST","path":"/knowledge-spaces/{id}/documents/bulk/reindex","action":"documents.bulk.reindex","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getCompilationJob","kfsOperationId":"getDocumentCompilationJob","method":"GET","path":"/jobs/{id}","action":"document_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"cancelCompilationJob","kfsOperationId":"cancelDocumentCompilationJob","method":"DELETE","path":"/jobs/{id}","action":"document_jobs.cancel","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"retryCompilationJob","kfsOperationId":"retryDocumentCompilationJob","method":"POST","path":"/jobs/{id}/retry","action":"document_jobs.retry","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getBulkJob","kfsOperationId":"getBulkOperation","method":"GET","path":"/bulk-jobs/{id}","action":"bulk_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSources","kfsOperationId":"listKnowledgeSpaceSources","method":"GET","path":"/knowledge-spaces/{id}/sources","action":"sources.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createSource","kfsOperationId":"createKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources","action":"sources.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSource","kfsOperationId":"getKnowledgeSpaceSource","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.read","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSource","kfsOperationId":"updateKnowledgeSpaceSource","method":"PATCH","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"deleteSource","kfsOperationId":"requestSourceDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.delete","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"testSource","kfsOperationId":"testKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/test","action":"sources.test","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"crawlSource","kfsOperationId":"crawlKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl","action":"sources.crawl","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSourcePages","kfsOperationId":"listKnowledgeSpaceSourcePages","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/pages","action":"sources.pages.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"importSourcePages","kfsOperationId":"importKnowledgeSpaceSourcePages","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import","action":"sources.pages.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSourceFiles","kfsOperationId":"listKnowledgeSpaceSourceFiles","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/files","action":"sources.files.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"importSourceFiles","kfsOperationId":"importKnowledgeSpaceSourceFiles","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import-files","action":"sources.files.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createQuery","kfsOperationId":"createQuery","method":"POST","path":"/queries","action":"queries.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}},
|
||||
{"productOperationId":"listResearchTasks","kfsOperationId":"listKnowledgeSpaceResearchTasks","method":"GET","path":"/knowledge-spaces/{id}/research-tasks","action":"research_tasks.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createResearchTask","kfsOperationId":"createResearchTask","method":"POST","path":"/research-tasks","action":"research_tasks.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"planResearchTask","kfsOperationId":"planResearchTask","method":"POST","path":"/research-tasks/plan","action":"research_tasks.plan","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getResearchTask","kfsOperationId":"getResearchTask","method":"GET","path":"/research-tasks/{id}","action":"research_tasks.read","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listResearchTaskPartials","kfsOperationId":"listResearchTaskPartials","method":"GET","path":"/research-tasks/{id}/partials","action":"research_tasks.partials.list","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"cancelResearchTask","kfsOperationId":"cancelResearchTask","method":"DELETE","path":"/research-tasks/{id}","action":"research_tasks.cancel","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraces","kfsOperationId":"listKnowledgeSpaceQualityTraces","method":"GET","path":"/knowledge-spaces/{id}/quality/traces","action":"quality.traces.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getTrace","kfsOperationId":"getAnswerTrace","method":"GET","path":"/queries/{traceId}","action":"queries.read","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceEvidence","kfsOperationId":"listQueryEvidence","method":"GET","path":"/queries/{traceId}/evidence","action":"queries.evidence.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceConflicts","kfsOperationId":"listQueryConflicts","method":"GET","path":"/queries/{traceId}/conflicts","action":"queries.conflicts.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceMissing","kfsOperationId":"listQueryMissing","method":"GET","path":"/queries/{traceId}/missing","action":"queries.missing.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createUploadSession","kfsOperationId":"createUploadSession","method":"POST","path":"/knowledge-spaces/{id}/upload-sessions","action":"upload_sessions.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"presignUploadSessionPart","kfsOperationId":"presignUploadSessionPart","method":"POST","path":"/upload-sessions/{id}/parts/{partNumber}/presign","action":"upload_sessions.write","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"uploadSmallFile","kfsOperationId":"uploadSmallFile","method":"POST","path":"/upload-sessions/{id}/small-file","action":"upload_sessions.write","resource":"upload_session","transport":"binary","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":8388608,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"completeUploadSession","kfsOperationId":"completeUploadSession","method":"POST","path":"/upload-sessions/{id}/complete","action":"upload_sessions.complete","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"abortUploadSession","kfsOperationId":"abortUploadSession","method":"POST","path":"/upload-sessions/{id}/abort","action":"upload_sessions.abort","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"streamResearchTask","kfsOperationId":"streamResearchTaskProgress","method":"GET","path":"/research-tasks/{id}/events","action":"research_tasks.stream","resource":"research_task","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}}
|
||||
]
|
||||
}
|
||||
+33
-12
@@ -16,12 +16,19 @@ from urllib.parse import quote
|
||||
import boto3
|
||||
import orjson
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from configs import dify_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OBJECT_NOT_FOUND_ERROR_CODES = frozenset({"404", "NoSuchKey", "NotFound"})
|
||||
|
||||
|
||||
def _is_object_not_found_error(error: ClientError) -> bool:
|
||||
error_code = str(error.response.get("Error", {}).get("Code", ""))
|
||||
return error_code in _OBJECT_NOT_FOUND_ERROR_CODES
|
||||
|
||||
|
||||
class ArchiveStorageError(Exception):
|
||||
"""Base exception for archive storage operations."""
|
||||
@@ -138,10 +145,11 @@ class ArchiveStorage:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
return response["Body"].read()
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code")
|
||||
if error_code == "NoSuchKey":
|
||||
raise FileNotFoundError(f"Archive object not found: {key}")
|
||||
raise ArchiveStorageError(f"Failed to download object '{key}': {e}")
|
||||
if _is_object_not_found_error(e):
|
||||
raise FileNotFoundError(f"Archive object not found: {key}") from e
|
||||
raise ArchiveStorageError(f"Failed to download object '{key}': {e}") from e
|
||||
except BotoCoreError as e:
|
||||
raise ArchiveStorageError(f"Failed to download object '{key}': {e}") from e
|
||||
|
||||
def get_object_stream(self, key: str) -> Generator[bytes, None, None]:
|
||||
"""
|
||||
@@ -161,10 +169,11 @@ class ArchiveStorage:
|
||||
response = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
yield from response["Body"].iter_chunks()
|
||||
except ClientError as e:
|
||||
error_code = e.response.get("Error", {}).get("Code")
|
||||
if error_code == "NoSuchKey":
|
||||
raise FileNotFoundError(f"Archive object not found: {key}")
|
||||
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}")
|
||||
if _is_object_not_found_error(e):
|
||||
raise FileNotFoundError(f"Archive object not found: {key}") from e
|
||||
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}") from e
|
||||
except BotoCoreError as e:
|
||||
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}") from e
|
||||
|
||||
def object_exists(self, key: str) -> bool:
|
||||
"""
|
||||
@@ -175,12 +184,19 @@ class ArchiveStorage:
|
||||
|
||||
Returns:
|
||||
True if object exists, False otherwise
|
||||
|
||||
Raises:
|
||||
ArchiveStorageError: If storage cannot authoritatively determine object existence
|
||||
"""
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError:
|
||||
return False
|
||||
except ClientError as e:
|
||||
if _is_object_not_found_error(e):
|
||||
return False
|
||||
raise ArchiveStorageError(f"Failed to check archive object '{key}': {e}") from e
|
||||
except BotoCoreError as e:
|
||||
raise ArchiveStorageError(f"Failed to check archive object '{key}': {e}") from e
|
||||
|
||||
def delete_object(self, key: str) -> None:
|
||||
"""
|
||||
@@ -196,7 +212,12 @@ class ArchiveStorage:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
logger.debug("Deleted object: %s", key)
|
||||
except ClientError as e:
|
||||
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}")
|
||||
if _is_object_not_found_error(e):
|
||||
logger.debug("Archive object was already absent: %s", key)
|
||||
return
|
||||
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}") from e
|
||||
except BotoCoreError as e:
|
||||
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}") from e
|
||||
|
||||
def generate_presigned_url(
|
||||
self,
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""add workflow-run archive bundle cursor indexes
|
||||
|
||||
Revision ID: 3c9f8e2a1d7b
|
||||
Revises: 7a1c2d9e4b60
|
||||
Create Date: 2026-07-15 16:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "3c9f8e2a1d7b"
|
||||
down_revision = "7a1c2d9e4b60"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE_NAME = "workflow_run_archive_bundles"
|
||||
_INDEXES = (
|
||||
("workflow_run_archive_bundle_month_id_idx", ("year", "month", "id")),
|
||||
("workflow_run_archive_bundle_month_shard_id_idx", ("year", "month", "shard", "id")),
|
||||
)
|
||||
|
||||
|
||||
def _is_postgresql() -> bool:
|
||||
return op.get_bind().dialect.name == "postgresql"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _is_postgresql():
|
||||
with op.get_context().autocommit_block():
|
||||
for index_name, columns in _INDEXES:
|
||||
op.create_index(index_name, _TABLE_NAME, columns, postgresql_concurrently=True)
|
||||
return
|
||||
for index_name, columns in _INDEXES:
|
||||
op.create_index(index_name, _TABLE_NAME, columns)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _is_postgresql():
|
||||
with op.get_context().autocommit_block():
|
||||
for index_name, _ in reversed(_INDEXES):
|
||||
op.drop_index(index_name, table_name=_TABLE_NAME, postgresql_concurrently=True)
|
||||
return
|
||||
for index_name, _ in reversed(_INDEXES):
|
||||
op.drop_index(index_name, table_name=_TABLE_NAME)
|
||||
+387
@@ -0,0 +1,387 @@
|
||||
"""add independent KnowledgeFS control-plane tables
|
||||
|
||||
Revision ID: a4e7c2f91b30
|
||||
Revises: 3c9f8e2a1d7b
|
||||
Create Date: 2026-07-21 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "a4e7c2f91b30"
|
||||
down_revision = "3c9f8e2a1d7b"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def _timestamps() -> tuple[sa.Column, sa.Column]:
|
||||
return (
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_control_spaces",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("knowledge_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("knowledge_space_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("owner_account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("visibility", sa.String(length=32), server_default=sa.text("'only_me'"), nullable=False),
|
||||
sa.Column("provisioning_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("lifecycle_operation_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("state", sa.String(length=32), server_default=sa.text("'provisioning'"), nullable=False),
|
||||
sa.Column("resource_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(length=255), nullable=True),
|
||||
sa.Column("last_error_message", models.types.LongText(), nullable=True),
|
||||
sa.Column("last_synced_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("deletion_irreversible_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
|
||||
sa.UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id"],
|
||||
["tenants.id"],
|
||||
name="kfs_control_space_workspace_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("resource_version >= 0", name="kfs_control_space_resource_version_ck"),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="kfs_control_space_attempt_count_ck"),
|
||||
sa.CheckConstraint(
|
||||
"state != 'active' OR knowledge_space_id IS NOT NULL",
|
||||
name="kfs_control_space_active_registration_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
|
||||
name="kfs_control_space_irreversible_state_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"knowledge_space_revision >= 0",
|
||||
name="kfs_control_space_remote_revision_ck",
|
||||
),
|
||||
)
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "knowledge_space_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
|
||||
)
|
||||
else:
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "knowledge_space_id"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_state_updated_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["state", "updated_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_state_updated_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "state", "updated_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_owner_state_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "owner_account_id", "state"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_control_space_permissions",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("role", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("granted_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"account_id",
|
||||
name="kfs_control_space_permission_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_control_space_permission_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_control_space_permission_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_permission_account_idx",
|
||||
"knowledge_fs_control_space_permissions",
|
||||
["tenant_id", "account_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_external_access_policies",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("service_api_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("agent_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("workflow_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("mcp_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("updated_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_external_access_policy_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_external_access_policy_revision_ck"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_api_credentials",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("credential_hash", sa.String(length=255), nullable=False),
|
||||
sa.Column("credential_prefix", sa.String(length=32), nullable=False),
|
||||
sa.Column("credential_last4", sa.String(length=4), nullable=False),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("allowed_actions", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_used_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoke_reason", sa.String(length=255), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
|
||||
sa.UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_api_credential_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_api_credential_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_api_credential_tenant_space_status_idx",
|
||||
"knowledge_fs_api_credentials",
|
||||
["tenant_id", "control_space_id", "status"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_api_credential_tenant_prefix_idx",
|
||||
"knowledge_fs_api_credentials",
|
||||
["tenant_id", "credential_prefix"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"app_knowledge_fs_space_joins",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("app_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("join_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"app_id",
|
||||
"control_space_id",
|
||||
"join_type",
|
||||
name="app_kfs_space_join_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="app_kfs_space_join_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="app_kfs_space_join_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"app_kfs_space_join_app_status_idx",
|
||||
"app_knowledge_fs_space_joins",
|
||||
["tenant_id", "app_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_authorization_revisions",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("membership_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("space_acl_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("external_access_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("content_policy_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("revoke_sequence", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_authorization_revision_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("membership_epoch >= 0", name="kfs_authorization_membership_epoch_ck"),
|
||||
sa.CheckConstraint("space_acl_epoch >= 0", name="kfs_authorization_space_acl_epoch_ck"),
|
||||
sa.CheckConstraint("external_access_epoch >= 0", name="kfs_authorization_external_access_epoch_ck"),
|
||||
sa.CheckConstraint("content_policy_revision >= 0", name="kfs_authorization_content_policy_revision_ck"),
|
||||
sa.CheckConstraint("revoke_sequence >= 0", name="kfs_authorization_revoke_sequence_ck"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("trace_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("jti_hash", sa.String(length=80), nullable=False),
|
||||
sa.Column("claims_summary", sa.JSON(), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
|
||||
sa.UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_audit_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
["tenant_id", "control_space_id", "created_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_audit_trace_idx",
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
["tenant_id", "trace_id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("operation_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("operation", sa.String(length=32), nullable=False),
|
||||
sa.Column("command_payload", sa.JSON(), nullable=False),
|
||||
sa.Column("expected_control_space_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("expected_knowledge_space_revision", sa.BigInteger(), nullable=True),
|
||||
sa.Column("command_schema_version", sa.Integer(), server_default=sa.text("1"), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'pending'"), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(length=255), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(length=255), nullable=True),
|
||||
sa.Column("last_error_message", models.types.LongText(), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("retain_until", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
|
||||
sa.UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_lifecycle_outbox_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("command_schema_version >= 1", name="kfs_lifecycle_outbox_schema_version_ck"),
|
||||
sa.CheckConstraint("expected_control_space_version >= 0", name="kfs_lifecycle_outbox_expected_version_ck"),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="kfs_lifecycle_outbox_attempt_count_ck"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
|
||||
name="kfs_lifecycle_outbox_status_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
|
||||
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
|
||||
name="kfs_lifecycle_outbox_lease_state_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
|
||||
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
|
||||
name="kfs_lifecycle_outbox_terminal_state_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_lifecycle_outbox_dispatch_idx",
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
["status", "next_attempt_at", "id"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_lifecycle_outbox_space_created_idx",
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
["tenant_id", "control_space_id", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("kfs_lifecycle_outbox_space_created_idx", table_name="knowledge_fs_lifecycle_outbox")
|
||||
op.drop_index("kfs_lifecycle_outbox_dispatch_idx", table_name="knowledge_fs_lifecycle_outbox")
|
||||
op.drop_table("knowledge_fs_lifecycle_outbox")
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_audit_trace_idx",
|
||||
table_name="knowledge_fs_capability_issuance_audits",
|
||||
)
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
table_name="knowledge_fs_capability_issuance_audits",
|
||||
)
|
||||
op.drop_table("knowledge_fs_capability_issuance_audits")
|
||||
op.drop_table("knowledge_fs_authorization_revisions")
|
||||
op.drop_index("app_kfs_space_join_app_status_idx", table_name="app_knowledge_fs_space_joins")
|
||||
op.drop_table("app_knowledge_fs_space_joins")
|
||||
op.drop_index("kfs_api_credential_tenant_prefix_idx", table_name="knowledge_fs_api_credentials")
|
||||
op.drop_index("kfs_api_credential_tenant_space_status_idx", table_name="knowledge_fs_api_credentials")
|
||||
op.drop_table("knowledge_fs_api_credentials")
|
||||
op.drop_table("knowledge_fs_external_access_policies")
|
||||
op.drop_index(
|
||||
"kfs_control_space_permission_account_idx",
|
||||
table_name="knowledge_fs_control_space_permissions",
|
||||
)
|
||||
op.drop_table("knowledge_fs_control_space_permissions")
|
||||
op.drop_index("kfs_control_space_tenant_owner_state_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_tenant_state_updated_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_state_updated_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_tenant_space_uq", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_table("knowledge_fs_control_spaces")
|
||||
@@ -0,0 +1,205 @@
|
||||
"""add KnowledgeFS workspace cutover ledger and migration evidence
|
||||
|
||||
Revision ID: b7f2a9d41c60
|
||||
Revises: a4e7c2f91b30
|
||||
Create Date: 2026-07-21 13:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "b7f2a9d41c60"
|
||||
down_revision = "a4e7c2f91b30"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def _timestamps() -> tuple[sa.Column, sa.Column]:
|
||||
return (
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_workspace_cutover_ledgers",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("phase", sa.String(length=32), server_default=sa.text("'inventory'"), nullable=False),
|
||||
sa.Column("source_revision_watermark", sa.JSON(), nullable=False),
|
||||
sa.Column("final_revision_watermark", sa.JSON(), nullable=True),
|
||||
sa.Column("applied_revision_watermark", sa.JSON(), nullable=False),
|
||||
sa.Column("source_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("final_task_watermark", sa.BigInteger(), nullable=True),
|
||||
sa.Column("applied_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("freeze_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("cutover_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("rolled_back_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("rollback_cutoff_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_started_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_window_ends_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_completed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("maximum_task_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("irreversible_cleanup_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("product_routes_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("capability_v2_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("integrated_mode_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("legacy_acl_read_only", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("smoke_results", sa.JSON(), nullable=True),
|
||||
sa.Column("legacy_dependency_report", sa.JSON(), nullable=True),
|
||||
sa.Column("legacy_dependency_checked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("legacy_dependency_ready", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("cas_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
|
||||
sa.UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
|
||||
sa.CheckConstraint("cas_version >= 0", name="kfs_workspace_cutover_cas_version_ck"),
|
||||
sa.CheckConstraint(
|
||||
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
|
||||
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
|
||||
name="kfs_workspace_cutover_task_watermark_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"cutover_at IS NULL OR freeze_at IS NOT NULL",
|
||||
name="kfs_workspace_cutover_freeze_before_cutover_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
|
||||
name="kfs_workspace_cutover_rollback_cutoff_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_workspace_cutover_ledger_phase_updated_idx",
|
||||
"knowledge_fs_workspace_cutover_ledgers",
|
||||
["phase", "updated_at"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_migration_issues",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("issue_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("kind", sa.String(length=48), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'open'"), nullable=False),
|
||||
sa.Column("resource_type", sa.String(length=64), nullable=True),
|
||||
sa.Column("resource_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_issue_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_migration_issue_gate_idx",
|
||||
"knowledge_fs_migration_issues",
|
||||
["tenant_id", "ledger_id", "status", "kind"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_migration_quarantine",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("source_kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("reason_code", sa.String(length=64), nullable=False),
|
||||
sa.Column("disposition", sa.String(length=32), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"source_kind",
|
||||
"source_id",
|
||||
name="kfs_migration_quarantine_source_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_quarantine_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_migration_quarantine_disposition_idx",
|
||||
"knowledge_fs_migration_quarantine",
|
||||
["tenant_id", "ledger_id", "disposition"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_shadow_authorization_diffs",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("diff_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
|
||||
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
|
||||
sa.Column("decision", sa.String(length=32), nullable=False),
|
||||
sa.Column("reason", models.types.LongText(), nullable=False),
|
||||
sa.Column("observed_revision", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_diff_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_shadow_authorization_diff_gate_idx",
|
||||
"knowledge_fs_shadow_authorization_diffs",
|
||||
["tenant_id", "ledger_id", "status", "decision"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"kfs_shadow_authorization_diff_gate_idx",
|
||||
table_name="knowledge_fs_shadow_authorization_diffs",
|
||||
)
|
||||
op.drop_table("knowledge_fs_shadow_authorization_diffs")
|
||||
op.drop_index(
|
||||
"kfs_migration_quarantine_disposition_idx",
|
||||
table_name="knowledge_fs_migration_quarantine",
|
||||
)
|
||||
op.drop_table("knowledge_fs_migration_quarantine")
|
||||
op.drop_index("kfs_migration_issue_gate_idx", table_name="knowledge_fs_migration_issues")
|
||||
op.drop_table("knowledge_fs_migration_issues")
|
||||
op.drop_index(
|
||||
"kfs_workspace_cutover_ledger_phase_updated_idx",
|
||||
table_name="knowledge_fs_workspace_cutover_ledgers",
|
||||
)
|
||||
op.drop_table("knowledge_fs_workspace_cutover_ledgers")
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
"""add guarded cleanup, cutover evidence, and capability issuance reservations
|
||||
|
||||
Revision ID: c8e31b7d52a4
|
||||
Revises: b7f2a9d41c60
|
||||
Create Date: 2026-07-21 14:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "c8e31b7d52a4"
|
||||
down_revision = "b7f2a9d41c60"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_cleanup_authorizations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("request_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("plan_schema_version", sa.String(length=32), nullable=False),
|
||||
sa.Column("plan_digest", sa.String(length=71), nullable=False),
|
||||
sa.Column("targets", sa.JSON(), nullable=False),
|
||||
sa.Column("readiness_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("requested_by_account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("requested_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("readiness_ledger_cas_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), server_default=sa.text("'requested'"), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("approval_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("approved_ledger_cas_version", sa.BigInteger(), nullable=True),
|
||||
sa.Column("started_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("started_ledger_cas_version", sa.BigInteger(), nullable=True),
|
||||
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"request_id",
|
||||
name="kfs_cleanup_authorization_request_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_cleanup_authorization_ledger_fk",
|
||||
),
|
||||
sa.CheckConstraint("row_version >= 0", name="kfs_cleanup_authorization_version_ck"),
|
||||
sa.CheckConstraint(
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
|
||||
name="kfs_cleanup_authorization_ledger_versions_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL)",
|
||||
name="kfs_cleanup_authorization_status_fields_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
|
||||
name="kfs_cleanup_authorization_approval_window_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_cleanup_authorization_status_idx",
|
||||
"knowledge_fs_cleanup_authorizations",
|
||||
["tenant_id", "status", "updated_at"],
|
||||
)
|
||||
op.create_table(
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("grant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("trace_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("subject", sa.String(length=255), nullable=False),
|
||||
sa.Column("caller_kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("request_summary", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), server_default=sa.text("'reserved'"), nullable=False),
|
||||
sa.Column("issued_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("token_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("failed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("failure_code", sa.String(length=128), nullable=True),
|
||||
sa.Column("cleanup_after", sa.DateTime(), nullable=True),
|
||||
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"grant_id",
|
||||
name="kfs_capability_issuance_reservation_grant_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_reservation_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"row_version >= 0",
|
||||
name="kfs_capability_issuance_reservation_version_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
|
||||
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
|
||||
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
|
||||
name="kfs_capability_issuance_reservation_status_fields_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
|
||||
name="kfs_capability_issuance_reservation_cleanup_window_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
["tenant_id", "control_space_id", "subject", "caller_kind"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
["tenant_id", "trace_id"],
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("resolved_by_operator", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_migration_quarantine_version_ck",
|
||||
"row_version >= 0",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_migration_quarantine_resolution_fields_ck",
|
||||
"(disposition = 'resolved' "
|
||||
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
|
||||
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
|
||||
"(disposition <> 'resolved' "
|
||||
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
|
||||
"AND evidence IS NULL AND resolved_at IS NULL)",
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("shadow_started_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_evidence_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(
|
||||
sa.Column("shadow_observation_count", sa.BigInteger(), server_default=sa.text("0"), nullable=False)
|
||||
)
|
||||
batch_op.add_column(sa.Column("shadow_window_started_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_window_ended_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(
|
||||
sa.Column("shadow_traffic_zero", sa.Boolean(), server_default=sa.text("false"), nullable=False)
|
||||
)
|
||||
batch_op.add_column(sa.Column("shadow_traffic_zero_evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_latest_observed_revision", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_producer", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_by_operator", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_id", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_revision", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_control_space_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_activated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_updated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_acknowledged_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_applied", sa.Boolean(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_replayed", sa.Boolean(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_count_ck",
|
||||
"shadow_observation_count >= 0",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_window_ck",
|
||||
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_completion_fields_ck",
|
||||
"(shadow_completed_at IS NULL "
|
||||
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
|
||||
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
|
||||
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
|
||||
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
|
||||
"AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
|
||||
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
|
||||
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
|
||||
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
|
||||
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
|
||||
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_activation_fields_ck",
|
||||
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
|
||||
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
|
||||
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
|
||||
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
|
||||
"AND remote_activation_replayed IS NULL) OR "
|
||||
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
|
||||
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
|
||||
"AND remote_activation_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
|
||||
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_activation_time_ck",
|
||||
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("current_evidence_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("last_observed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_shadow_authorization_diff_version_ck",
|
||||
"row_version >= 0",
|
||||
)
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE knowledge_fs_shadow_authorization_diffs "
|
||||
"SET current_evidence_digest = 'sha256:legacy-unversioned', last_observed_at = created_at "
|
||||
"WHERE current_evidence_digest IS NULL OR last_observed_at IS NULL"
|
||||
)
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.alter_column("current_evidence_digest", existing_type=sa.String(length=71), nullable=False)
|
||||
batch_op.alter_column("last_observed_at", existing_type=sa.DateTime(), nullable=False)
|
||||
op.create_table(
|
||||
"knowledge_fs_shadow_authorization_observations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("diff_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("producer", sa.String(length=255), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
|
||||
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
|
||||
sa.Column("decision", sa.String(length=32), nullable=False),
|
||||
sa.Column("reason", models.types.LongText(), nullable=False),
|
||||
sa.Column("observed_revision", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("evidence_digest", sa.String(length=71), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"diff_key",
|
||||
"evidence_digest",
|
||||
name="kfs_shadow_authorization_observation_evidence_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_observation_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
"knowledge_fs_shadow_authorization_observations",
|
||||
["tenant_id", "ledger_id", "observed_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
table_name="knowledge_fs_shadow_authorization_observations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_shadow_authorization_observations")
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_shadow_authorization_diff_version_ck", type_="check")
|
||||
batch_op.drop_column("row_version")
|
||||
batch_op.drop_column("last_observed_at")
|
||||
batch_op.drop_column("current_evidence_digest")
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_completion_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_window_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_count_ck", type_="check")
|
||||
batch_op.drop_column("remote_activation_replayed")
|
||||
batch_op.drop_column("remote_activation_applied")
|
||||
batch_op.drop_column("remote_activation_acknowledged_at")
|
||||
batch_op.drop_column("remote_activation_updated_at")
|
||||
batch_op.drop_column("remote_activation_activated_at")
|
||||
batch_op.drop_column("remote_activation_control_space_id")
|
||||
batch_op.drop_column("remote_activation_digest")
|
||||
batch_op.drop_column("remote_activation_revision")
|
||||
batch_op.drop_column("remote_activation_id")
|
||||
batch_op.drop_column("shadow_completed_by_account_id")
|
||||
batch_op.drop_column("shadow_completed_by_operator")
|
||||
batch_op.drop_column("shadow_producer")
|
||||
batch_op.drop_column("shadow_latest_observed_revision")
|
||||
batch_op.drop_column("shadow_traffic_zero_evidence")
|
||||
batch_op.drop_column("shadow_traffic_zero")
|
||||
batch_op.drop_column("shadow_window_ended_at")
|
||||
batch_op.drop_column("shadow_window_started_at")
|
||||
batch_op.drop_column("shadow_observation_count")
|
||||
batch_op.drop_column("shadow_evidence_digest")
|
||||
batch_op.drop_column("shadow_completed_at")
|
||||
batch_op.drop_column("shadow_started_at")
|
||||
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
|
||||
batch_op.drop_constraint(
|
||||
"kfs_migration_quarantine_resolution_fields_ck",
|
||||
type_="check",
|
||||
)
|
||||
batch_op.drop_constraint("kfs_migration_quarantine_version_ck", type_="check")
|
||||
batch_op.drop_column("row_version")
|
||||
batch_op.drop_column("evidence")
|
||||
batch_op.drop_column("resolved_by_account_id")
|
||||
batch_op.drop_column("resolved_by_operator")
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
table_name="knowledge_fs_capability_issuance_reservations",
|
||||
)
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
table_name="knowledge_fs_capability_issuance_reservations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_capability_issuance_reservations")
|
||||
op.drop_index(
|
||||
"kfs_cleanup_authorization_status_idx",
|
||||
table_name="knowledge_fs_cleanup_authorizations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_cleanup_authorizations")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""add durable KnowledgeFS remote freeze evidence
|
||||
|
||||
Revision ID: d4f6e8a1c305
|
||||
Revises: c8e31b7d52a4
|
||||
Create Date: 2026-07-21 15:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "d4f6e8a1c305"
|
||||
down_revision = "c8e31b7d52a4"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("remote_freeze_id", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_revision", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_task_watermark", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_control_space_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_frozen_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_updated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_acknowledged_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_applied", sa.Boolean(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_replayed", sa.Boolean(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_freeze_fields_ck",
|
||||
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
|
||||
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
|
||||
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
|
||||
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
|
||||
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
|
||||
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
|
||||
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
|
||||
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
|
||||
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_freeze_time_ck",
|
||||
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_fields_ck", type_="check")
|
||||
batch_op.drop_column("remote_freeze_replayed")
|
||||
batch_op.drop_column("remote_freeze_applied")
|
||||
batch_op.drop_column("remote_freeze_acknowledged_at")
|
||||
batch_op.drop_column("remote_freeze_updated_at")
|
||||
batch_op.drop_column("remote_freeze_frozen_at")
|
||||
batch_op.drop_column("remote_freeze_control_space_id")
|
||||
batch_op.drop_column("remote_freeze_task_watermark")
|
||||
batch_op.drop_column("remote_freeze_digest")
|
||||
batch_op.drop_column("remote_freeze_revision")
|
||||
batch_op.drop_column("remote_freeze_id")
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
"""add global KnowledgeFS cleanup completion evidence
|
||||
|
||||
Revision ID: e5a7c9b2d416
|
||||
Revises: d4f6e8a1c305
|
||||
Create Date: 2026-07-21 16:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "e5a7c9b2d416"
|
||||
down_revision = "d4f6e8a1c305"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_LEDGER_VERSIONS_CHECK = (
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
|
||||
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)"
|
||||
)
|
||||
_STATUS_FIELDS_CHECK = (
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'completed' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
|
||||
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)"
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
|
||||
batch_op.add_column(sa.Column("completed_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("completed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("completion_evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("completed_ledger_cas_version", sa.BigInteger(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_ledger_versions_ck",
|
||||
_LEDGER_VERSIONS_CHECK,
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_status_fields_ck",
|
||||
_STATUS_FIELDS_CHECK,
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_completion_time_ck",
|
||||
"completed_at IS NULL OR completed_at >= started_at",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_completion_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_ledger_versions_ck",
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_status_fields_ck",
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL)",
|
||||
)
|
||||
batch_op.drop_column("completed_ledger_cas_version")
|
||||
batch_op.drop_column("completion_evidence")
|
||||
batch_op.drop_column("completed_at")
|
||||
batch_op.drop_column("completed_by_account_id")
|
||||
@@ -66,6 +66,45 @@ from .enums import (
|
||||
)
|
||||
from .execution_extra_content import ExecutionExtraContent, HumanInputContent
|
||||
from .human_input import HumanInputForm, HumanInputFormUploadFile, HumanInputFormUploadToken
|
||||
from .knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSApiCredentialStatus,
|
||||
KnowledgeFSAppSpaceJoinStatus,
|
||||
KnowledgeFSAppSpaceJoinType,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSCapabilityIssuanceAudit,
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSCapabilityIssuanceReservationStatus,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpacePermissionStatus,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSControlSpaceVisibility,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSLifecycleOperation,
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSLifecycleOutboxStatus,
|
||||
)
|
||||
from .knowledge_fs_cleanup import (
|
||||
KnowledgeFSCleanupAuthorization,
|
||||
KnowledgeFSCleanupAuthorizationStatus,
|
||||
KnowledgeFSCleanupTarget,
|
||||
)
|
||||
from .knowledge_fs_cutover import (
|
||||
KnowledgeFSMigrationIssue,
|
||||
KnowledgeFSMigrationIssueKind,
|
||||
KnowledgeFSMigrationIssueStatus,
|
||||
KnowledgeFSMigrationQuarantine,
|
||||
KnowledgeFSMigrationQuarantineDisposition,
|
||||
KnowledgeFSMigrationQuarantineKind,
|
||||
KnowledgeFSShadowAuthorizationDecision,
|
||||
KnowledgeFSShadowAuthorizationDiff,
|
||||
KnowledgeFSShadowAuthorizationObservation,
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
)
|
||||
from .model import (
|
||||
AccountTrialAppRecord,
|
||||
ApiRequest,
|
||||
@@ -180,6 +219,7 @@ __all__ = [
|
||||
"AppAnnotationHitHistory",
|
||||
"AppAnnotationSetting",
|
||||
"AppDatasetJoin",
|
||||
"AppKnowledgeFSSpaceJoin",
|
||||
"AppMCPServer",
|
||||
"AppMode",
|
||||
"AppModelConfig",
|
||||
@@ -224,6 +264,38 @@ __all__ = [
|
||||
"IconType",
|
||||
"InstalledApp",
|
||||
"InvitationCode",
|
||||
"KnowledgeFSApiCredential",
|
||||
"KnowledgeFSApiCredentialStatus",
|
||||
"KnowledgeFSAppSpaceJoinStatus",
|
||||
"KnowledgeFSAppSpaceJoinType",
|
||||
"KnowledgeFSAuthorizationRevision",
|
||||
"KnowledgeFSCapabilityIssuanceAudit",
|
||||
"KnowledgeFSCapabilityIssuanceReservation",
|
||||
"KnowledgeFSCapabilityIssuanceReservationStatus",
|
||||
"KnowledgeFSCleanupAuthorization",
|
||||
"KnowledgeFSCleanupAuthorizationStatus",
|
||||
"KnowledgeFSCleanupTarget",
|
||||
"KnowledgeFSControlSpace",
|
||||
"KnowledgeFSControlSpacePermission",
|
||||
"KnowledgeFSControlSpacePermissionRole",
|
||||
"KnowledgeFSControlSpacePermissionStatus",
|
||||
"KnowledgeFSControlSpaceState",
|
||||
"KnowledgeFSControlSpaceVisibility",
|
||||
"KnowledgeFSExternalAccessPolicy",
|
||||
"KnowledgeFSLifecycleOperation",
|
||||
"KnowledgeFSLifecycleOutbox",
|
||||
"KnowledgeFSLifecycleOutboxStatus",
|
||||
"KnowledgeFSMigrationIssue",
|
||||
"KnowledgeFSMigrationIssueKind",
|
||||
"KnowledgeFSMigrationIssueStatus",
|
||||
"KnowledgeFSMigrationQuarantine",
|
||||
"KnowledgeFSMigrationQuarantineDisposition",
|
||||
"KnowledgeFSMigrationQuarantineKind",
|
||||
"KnowledgeFSShadowAuthorizationDecision",
|
||||
"KnowledgeFSShadowAuthorizationDiff",
|
||||
"KnowledgeFSShadowAuthorizationObservation",
|
||||
"KnowledgeFSWorkspaceCutoverLedger",
|
||||
"KnowledgeFSWorkspaceCutoverPhase",
|
||||
"LoadBalancingModelConfig",
|
||||
"Message",
|
||||
"MessageAgentThought",
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
"""Dify-owned control-plane persistence for the KnowledgeFS product.
|
||||
|
||||
These models intentionally have no dependency on Dify Dataset or Document
|
||||
models. KnowledgeFS data-plane objects remain owned by KnowledgeFS; these rows
|
||||
only describe Dify authorization, registration, and durable lifecycle intent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import NotRequired, TypedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, LongText, StringUUID
|
||||
|
||||
KnowledgeFSAllowedActions = list[str]
|
||||
|
||||
|
||||
class KnowledgeFSCommandEnvelope(TypedDict):
|
||||
schema_version: int
|
||||
idempotency_key: str
|
||||
expected_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSModelSelectionIntentPayload(TypedDict):
|
||||
pluginId: str
|
||||
provider: str
|
||||
model: str
|
||||
|
||||
|
||||
class KnowledgeFSRerankIntentPayload(TypedDict):
|
||||
enabled: bool
|
||||
model: NotRequired[KnowledgeFSModelSelectionIntentPayload]
|
||||
|
||||
|
||||
class KnowledgeFSScoreThresholdIntentPayload(TypedDict):
|
||||
enabled: bool
|
||||
stage: str
|
||||
value: NotRequired[float]
|
||||
|
||||
|
||||
class KnowledgeFSRetrievalProfileIntentPayload(TypedDict):
|
||||
defaultMode: str
|
||||
reasoningModel: KnowledgeFSModelSelectionIntentPayload
|
||||
rerank: KnowledgeFSRerankIntentPayload
|
||||
scoreThreshold: KnowledgeFSScoreThresholdIntentPayload
|
||||
topK: int
|
||||
|
||||
|
||||
class KnowledgeFSProvisionCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
provisioning_key: str
|
||||
name: str
|
||||
icon: str | None
|
||||
description: str | None
|
||||
slug: str
|
||||
model_intent: KnowledgeFSModelSelectionIntentPayload
|
||||
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
|
||||
|
||||
|
||||
class KnowledgeFSMetadataUpdateCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
metadata: dict[str, object]
|
||||
|
||||
|
||||
class KnowledgeFSDeleteCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
knowledge_space_id: str | None
|
||||
provisioning_key: str
|
||||
|
||||
|
||||
class KnowledgeFSRevokeCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
event_id: str
|
||||
grant_id: str
|
||||
knowledge_space_id: str
|
||||
principal: str
|
||||
reason_code: str
|
||||
revoke_sequence: int
|
||||
|
||||
|
||||
class KnowledgeFSRepairCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
repair_reason: str
|
||||
|
||||
|
||||
KnowledgeFSCommandPayload = (
|
||||
KnowledgeFSProvisionCommandPayload
|
||||
| KnowledgeFSMetadataUpdateCommandPayload
|
||||
| KnowledgeFSDeleteCommandPayload
|
||||
| KnowledgeFSRevokeCommandPayload
|
||||
| KnowledgeFSRepairCommandPayload
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityAuthzRevisionSummary(TypedDict):
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
credential_revision: int | None
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityClaimsSummary(TypedDict):
|
||||
action: str
|
||||
actor: str
|
||||
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
|
||||
caller_kind: str
|
||||
content_policy_revision: int
|
||||
content_scope_ids: list[str]
|
||||
control_space_id: str
|
||||
expires_at: str
|
||||
grant_id: str
|
||||
issued_at: str
|
||||
namespace_id: str
|
||||
operation_id: str
|
||||
resource_id: str
|
||||
resource_parent_id: str | None
|
||||
resource_type: str
|
||||
subject: str
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityReservationSummary(TypedDict):
|
||||
"""Sanitized, exact input bound before a capability can be signed."""
|
||||
|
||||
action: str
|
||||
actor: str
|
||||
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
|
||||
caller_kind: str
|
||||
content_policy_revision: int
|
||||
content_scope_ids: list[str]
|
||||
control_space_id: str
|
||||
grant_id: str
|
||||
namespace_id: str
|
||||
operation_id: str
|
||||
resource_id: str
|
||||
resource_parent_id: str | None
|
||||
resource_type: str
|
||||
subject: str
|
||||
trace_id: str
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceVisibility(StrEnum):
|
||||
ONLY_ME = "only_me"
|
||||
ALL_TEAM_MEMBERS = "all_team_members"
|
||||
PARTIAL_MEMBERS = "partial_members"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceState(StrEnum):
|
||||
PROVISIONING = "provisioning"
|
||||
ACTIVE = "active"
|
||||
DELETING = "deleting"
|
||||
DELETED = "deleted"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermissionRole(StrEnum):
|
||||
OWNER = "owner"
|
||||
EDITOR = "editor"
|
||||
VIEWER = "viewer"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermissionStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class KnowledgeFSApiCredentialStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class KnowledgeFSAppSpaceJoinType(StrEnum):
|
||||
AGENT = "agent"
|
||||
WORKFLOW = "workflow"
|
||||
|
||||
|
||||
class KnowledgeFSAppSpaceJoinStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservationStatus(StrEnum):
|
||||
RESERVED = "reserved"
|
||||
ISSUED = "issued"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOperation(StrEnum):
|
||||
PROVISION = "provision"
|
||||
METADATA_UPDATE = "metadata_update"
|
||||
DELETE = "delete"
|
||||
REVOKE = "revoke"
|
||||
REPAIR = "repair"
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutboxStatus(StrEnum):
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
SUCCEEDED = "succeeded"
|
||||
RETRY = "retry"
|
||||
DEAD_LETTER = "dead_letter"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpace(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Dify product resource registered to at most one KnowledgeFS Space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_control_spaces"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
|
||||
UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
|
||||
UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id"],
|
||||
["tenants.id"],
|
||||
name="kfs_control_space_workspace_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"tenant_id",
|
||||
"knowledge_space_id",
|
||||
unique=True,
|
||||
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
|
||||
),
|
||||
Index("kfs_control_space_state_updated_idx", "state", "updated_at"),
|
||||
Index("kfs_control_space_tenant_state_updated_idx", "tenant_id", "state", "updated_at"),
|
||||
Index("kfs_control_space_tenant_owner_state_idx", "tenant_id", "owner_account_id", "state"),
|
||||
sa.CheckConstraint(
|
||||
"resource_version >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_resource_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint("attempt_count >= 0", name=sa.schema.conv("kfs_control_space_attempt_count_ck")),
|
||||
sa.CheckConstraint(
|
||||
"state != 'active' OR knowledge_space_id IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_control_space_active_registration_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
|
||||
name=sa.schema.conv("kfs_control_space_irreversible_state_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"knowledge_space_revision >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_remote_revision_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
owner_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
provisioning_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
knowledge_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
knowledge_space_revision: Mapped[int] = mapped_column(
|
||||
sa.BigInteger,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
visibility: Mapped[KnowledgeFSControlSpaceVisibility] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpaceVisibility, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'only_me'"),
|
||||
default=KnowledgeFSControlSpaceVisibility.ONLY_ME,
|
||||
)
|
||||
lifecycle_operation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
state: Mapped[KnowledgeFSControlSpaceState] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpaceState, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'provisioning'"),
|
||||
default=KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
)
|
||||
resource_version: Mapped[int] = mapped_column(
|
||||
sa.BigInteger,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(
|
||||
sa.Integer,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
||||
last_synced_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
deletion_irreversible_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermission(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Account-level product authorization for one control-space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_control_space_permissions"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"account_id",
|
||||
name="kfs_control_space_permission_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_control_space_permission_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_control_space_permission_account_idx", "tenant_id", "account_id", "status"),
|
||||
sa.CheckConstraint(
|
||||
"revision >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_permission_revision_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
role: Mapped[KnowledgeFSControlSpacePermissionRole] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpacePermissionRole, length=32), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSControlSpacePermissionStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpacePermissionStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
granted_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSExternalAccessPolicy(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Fail-closed caller-channel policy for a control-space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_external_access_policies"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
|
||||
UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_external_access_policy_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_external_access_policy_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
service_api_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
agent_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
workflow_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
mcp_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
updated_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSApiCredential(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Hashed, resource-bound KnowledgeFS API credential metadata."""
|
||||
|
||||
__tablename__ = "knowledge_fs_api_credentials"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
|
||||
UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_api_credential_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_api_credential_tenant_space_status_idx", "tenant_id", "control_space_id", "status"),
|
||||
Index("kfs_api_credential_tenant_prefix_idx", "tenant_id", "credential_prefix"),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_api_credential_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
credential_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
credential_prefix: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
credential_last4: Mapped[str] = mapped_column(String(4), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
allowed_actions: Mapped[KnowledgeFSAllowedActions] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSApiCredentialStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSApiCredentialStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSApiCredentialStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoke_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
|
||||
|
||||
class AppKnowledgeFSSpaceJoin(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Explicit Agent or Workflow app authorization for a control-space."""
|
||||
|
||||
__tablename__ = "app_knowledge_fs_space_joins"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"app_id",
|
||||
"control_space_id",
|
||||
"join_type",
|
||||
name="app_kfs_space_join_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="app_kfs_space_join_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("app_kfs_space_join_app_status_idx", "tenant_id", "app_id", "status"),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("app_kfs_space_join_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
join_type: Mapped[KnowledgeFSAppSpaceJoinType] = mapped_column(
|
||||
EnumText(KnowledgeFSAppSpaceJoinType, length=32), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSAppSpaceJoinStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSAppSpaceJoinStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSAppSpaceJoinStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSAuthorizationRevision(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Monotonic authorization epochs included in future Capability claims."""
|
||||
|
||||
__tablename__ = "knowledge_fs_authorization_revisions"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
|
||||
UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_authorization_revision_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"membership_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_membership_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"space_acl_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_space_acl_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"external_access_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_external_access_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"content_policy_revision >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_content_policy_revision_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"revoke_sequence >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_revoke_sequence_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
membership_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
space_acl_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
external_access_epoch: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
content_policy_revision: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
revoke_sequence: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceAudit(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Sanitized durable evidence for a Capability v2 issuance."""
|
||||
|
||||
__tablename__ = "knowledge_fs_capability_issuance_audits"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
|
||||
UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_audit_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"created_at",
|
||||
),
|
||||
Index("kfs_capability_issuance_audit_trace_idx", "tenant_id", "trace_id"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
jti_hash: Mapped[str] = mapped_column(String(80), nullable=False)
|
||||
claims_summary: Mapped[KnowledgeFSCapabilityClaimsSummary] = mapped_column(sa.JSON, nullable=False)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservation(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Durable authorization fence written before signing starts.
|
||||
|
||||
The summary intentionally excludes bearer tokens, signing material, and raw JTI values.
|
||||
Revocation producers scan these rows as well as completed issuance audits, closing the
|
||||
authorization-to-signing race without holding a database transaction across signing.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_fs_capability_issuance_reservations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"grant_id",
|
||||
name="kfs_capability_issuance_reservation_grant_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_reservation_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"subject",
|
||||
"caller_kind",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
"tenant_id",
|
||||
"trace_id",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"row_version >= 0",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
|
||||
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
|
||||
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_status_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_cleanup_window_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
grant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
subject: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
caller_kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
request_summary: Mapped[KnowledgeFSCapabilityReservationSummary] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSCapabilityIssuanceReservationStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSCapabilityIssuanceReservationStatus, length=16),
|
||||
nullable=False,
|
||||
server_default=sa.text("'reserved'"),
|
||||
default=KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
|
||||
)
|
||||
issued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
token_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
failed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
failure_code: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
|
||||
cleanup_after: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutbox(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Durable command snapshot; it is never a product read-model source.
|
||||
|
||||
Provision payload writers must snapshot name, icon, description, slug,
|
||||
model/profile intent, schema version, idempotency key, and expected
|
||||
revision before enabling the later P1B dispatcher.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_fs_lifecycle_outbox"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
|
||||
UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
|
||||
UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_lifecycle_outbox_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_lifecycle_outbox_dispatch_idx", "status", "next_attempt_at", "id"),
|
||||
Index("kfs_lifecycle_outbox_space_created_idx", "tenant_id", "control_space_id", "created_at"),
|
||||
sa.CheckConstraint(
|
||||
"command_schema_version >= 1",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_schema_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"expected_control_space_version >= 0",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_expected_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"attempt_count >= 0",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_attempt_count_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_status_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
|
||||
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_lease_state_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
|
||||
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_terminal_state_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
operation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
operation: Mapped[KnowledgeFSLifecycleOperation] = mapped_column(
|
||||
EnumText(KnowledgeFSLifecycleOperation, length=32), nullable=False
|
||||
)
|
||||
command_payload: Mapped[KnowledgeFSCommandPayload] = mapped_column(sa.JSON, nullable=False)
|
||||
expected_control_space_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
|
||||
expected_knowledge_space_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
command_schema_version: Mapped[int] = mapped_column(
|
||||
sa.Integer, nullable=False, server_default=sa.text("1"), default=1
|
||||
)
|
||||
status: Mapped[KnowledgeFSLifecycleOutboxStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSLifecycleOutboxStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'pending'"),
|
||||
default=KnowledgeFSLifecycleOutboxStatus.PENDING,
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"), default=0)
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
retain_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppKnowledgeFSSpaceJoin",
|
||||
"KnowledgeFSAllowedActions",
|
||||
"KnowledgeFSApiCredential",
|
||||
"KnowledgeFSApiCredentialStatus",
|
||||
"KnowledgeFSAppSpaceJoinStatus",
|
||||
"KnowledgeFSAppSpaceJoinType",
|
||||
"KnowledgeFSAuthorizationRevision",
|
||||
"KnowledgeFSCapabilityAuthzRevisionSummary",
|
||||
"KnowledgeFSCapabilityClaimsSummary",
|
||||
"KnowledgeFSCapabilityIssuanceAudit",
|
||||
"KnowledgeFSCapabilityIssuanceReservation",
|
||||
"KnowledgeFSCapabilityIssuanceReservationStatus",
|
||||
"KnowledgeFSCapabilityReservationSummary",
|
||||
"KnowledgeFSCommandEnvelope",
|
||||
"KnowledgeFSCommandPayload",
|
||||
"KnowledgeFSControlSpace",
|
||||
"KnowledgeFSControlSpacePermission",
|
||||
"KnowledgeFSControlSpacePermissionRole",
|
||||
"KnowledgeFSControlSpacePermissionStatus",
|
||||
"KnowledgeFSControlSpaceState",
|
||||
"KnowledgeFSControlSpaceVisibility",
|
||||
"KnowledgeFSDeleteCommandPayload",
|
||||
"KnowledgeFSExternalAccessPolicy",
|
||||
"KnowledgeFSLifecycleOperation",
|
||||
"KnowledgeFSLifecycleOutbox",
|
||||
"KnowledgeFSLifecycleOutboxStatus",
|
||||
"KnowledgeFSMetadataUpdateCommandPayload",
|
||||
"KnowledgeFSModelSelectionIntentPayload",
|
||||
"KnowledgeFSProvisionCommandPayload",
|
||||
"KnowledgeFSRepairCommandPayload",
|
||||
"KnowledgeFSRerankIntentPayload",
|
||||
"KnowledgeFSRetrievalProfileIntentPayload",
|
||||
"KnowledgeFSRevokeCommandPayload",
|
||||
"KnowledgeFSScoreThresholdIntentPayload",
|
||||
]
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Persistent authorization for the irreversible P9 KnowledgeFS cleanup window."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, StringUUID
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorizationStatus(StrEnum):
|
||||
REQUESTED = "requested"
|
||||
APPROVED = "approved"
|
||||
STARTED = "started"
|
||||
COMPLETED = "completed"
|
||||
|
||||
|
||||
class KnowledgeFSCleanupTarget(StrEnum):
|
||||
LEGACY_SNAPSHOT_FOREIGN_KEYS = "legacy_snapshot_foreign_keys"
|
||||
LEGACY_ACL_ROUTES = "legacy_acl_routes"
|
||||
LEGACY_ACL_SCHEMA = "legacy_acl_schema"
|
||||
LEGACY_API_KEY_SCHEMA = "legacy_api_key_schema"
|
||||
LEGACY_V1_AUTH = "legacy_v1_auth"
|
||||
RAW_LIST_CREATE_PROXY = "raw_list_create_proxy"
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorization(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Four-eyes approval and immutable evidence for one cleanup start fence."""
|
||||
|
||||
__tablename__ = "knowledge_fs_cleanup_authorizations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "request_id", name="kfs_cleanup_authorization_request_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_cleanup_authorization_ledger_fk",
|
||||
),
|
||||
Index("kfs_cleanup_authorization_status_idx", "tenant_id", "status", "updated_at"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_cleanup_authorization_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
|
||||
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_ledger_versions_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'completed' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
|
||||
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_status_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_approval_window_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"completed_at IS NULL OR completed_at >= started_at",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_completion_time_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
request_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
plan_schema_version: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
plan_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
targets: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False)
|
||||
readiness_evidence: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False)
|
||||
requested_by_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
requested_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
readiness_ledger_cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
|
||||
status: Mapped[KnowledgeFSCleanupAuthorizationStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSCleanupAuthorizationStatus, length=16),
|
||||
nullable=False,
|
||||
server_default=sa.text("'requested'"),
|
||||
default=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
|
||||
)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
approval_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
approved_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
started_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
started_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
completion_evidence: Mapped[dict[str, object] | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
completed_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCleanupAuthorization",
|
||||
"KnowledgeFSCleanupAuthorizationStatus",
|
||||
"KnowledgeFSCleanupTarget",
|
||||
]
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Workspace-scoped KnowledgeFS migration and cutover persistence.
|
||||
|
||||
The ledger is Dify control-plane state only. It deliberately stores opaque
|
||||
KnowledgeFS identifiers and authorization watermarks without foreign keys or
|
||||
queries to legacy product tables. Cutover feature switches live on one
|
||||
row so the repository can change them with a single CAS statement. Quarantine
|
||||
remediation is separately CAS-versioned and retains immutable operator evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import TypedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, LongText, StringUUID
|
||||
|
||||
|
||||
class KnowledgeFSCutoverRevisionWatermark(TypedDict):
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
content_policy_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeChecks(TypedDict):
|
||||
authorization: bool
|
||||
list_spaces: bool
|
||||
create_space: bool
|
||||
query: bool
|
||||
upload: bool
|
||||
stream: bool
|
||||
deletion: bool
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeEvidenceReferences(TypedDict):
|
||||
authorization: str
|
||||
list_spaces: str
|
||||
create_space: str
|
||||
query: str
|
||||
upload: str
|
||||
stream: str
|
||||
deletion: str
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeResults(TypedDict):
|
||||
schema_version: str
|
||||
tenant_id: str
|
||||
environment: str
|
||||
operator: str
|
||||
operator_account_id: str
|
||||
observed_at: str
|
||||
checks: KnowledgeFSCutoverSmokeChecks
|
||||
evidence_references: KnowledgeFSCutoverSmokeEvidenceReferences
|
||||
|
||||
|
||||
def knowledge_fs_cutover_smoke_results_passed(value: KnowledgeFSCutoverSmokeResults | None) -> bool:
|
||||
"""Reject incomplete or pre-production JSON evidence instead of treating metadata as booleans."""
|
||||
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
checks = value.get("checks")
|
||||
references = value.get("evidence_references")
|
||||
expected = {
|
||||
"authorization",
|
||||
"list_spaces",
|
||||
"create_space",
|
||||
"query",
|
||||
"upload",
|
||||
"stream",
|
||||
"deletion",
|
||||
}
|
||||
return bool(
|
||||
value.get("schema_version") == "knowledge-fs-p8-cutover-smoke/v1"
|
||||
and value.get("environment") == "production"
|
||||
and isinstance(checks, dict)
|
||||
and set(checks) == expected
|
||||
and all(checks.get(name) is True for name in expected)
|
||||
and isinstance(references, dict)
|
||||
and set(references) == expected
|
||||
and all(isinstance(reference := references.get(name), str) and bool(reference) for name in expected)
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceCutoverPhase(StrEnum):
|
||||
INVENTORY = "inventory"
|
||||
BACKFILL = "backfill"
|
||||
SHADOW = "shadow"
|
||||
FROZEN = "frozen"
|
||||
CUTOVER = "cutover"
|
||||
OBSERVING = "observing"
|
||||
READY_FOR_CLEANUP = "ready_for_cleanup"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssueStatus(StrEnum):
|
||||
OPEN = "open"
|
||||
APPROVED_FAIL_CLOSED = "approved_fail_closed"
|
||||
RESOLVED = "resolved"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssueKind(StrEnum):
|
||||
REGISTRATION_CONFLICT = "registration_conflict"
|
||||
UNRESOLVED_SUBJECT = "unresolved_subject"
|
||||
UNKNOWN_EXTERNAL_ACCESS = "unknown_external_access"
|
||||
REVISION_DRIFT = "revision_drift"
|
||||
TASK_QUARANTINE = "task_quarantine"
|
||||
LEGACY_SNAPSHOT_DEPENDENCY = "legacy_snapshot_dependency"
|
||||
LEGACY_FOREIGN_KEY_DEPENDENCY = "legacy_foreign_key_dependency"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantineKind(StrEnum):
|
||||
CONTROL_SPACE = "control_space"
|
||||
SUBJECT = "subject"
|
||||
TASK = "task"
|
||||
LEGACY_API_KEY = "legacy_api_key"
|
||||
ORPHAN_RESOURCE = "orphan_resource"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantineDisposition(StrEnum):
|
||||
PENDING = "pending"
|
||||
MIGRATABLE = "migratable"
|
||||
WAIT_FOR_COMPLETION = "wait_for_completion"
|
||||
CANCEL = "cancel"
|
||||
ISOLATE = "isolate"
|
||||
ROTATE_CREDENTIAL = "rotate_credential"
|
||||
RESOLVED = "resolved"
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationDecision(StrEnum):
|
||||
MATCH = "match"
|
||||
TIGHTENED = "tightened"
|
||||
EXPANDED = "expanded"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceCutoverLedger(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Single-Workspace migration state, watermarks, switches, and time fences."""
|
||||
|
||||
__tablename__ = "knowledge_fs_workspace_cutover_ledgers"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
|
||||
UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
|
||||
UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
|
||||
Index("kfs_workspace_cutover_ledger_phase_updated_idx", "phase", "updated_at"),
|
||||
sa.CheckConstraint("cas_version >= 0", name=sa.schema.conv("kfs_workspace_cutover_cas_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
|
||||
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_task_watermark_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"cutover_at IS NULL OR freeze_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_freeze_before_cutover_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_rollback_cutoff_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"shadow_observation_count >= 0",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_count_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_window_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(shadow_completed_at IS NULL "
|
||||
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
|
||||
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
|
||||
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
|
||||
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
|
||||
"AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
|
||||
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
|
||||
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
|
||||
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
|
||||
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
|
||||
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_completion_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
|
||||
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
|
||||
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
|
||||
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
|
||||
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
|
||||
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
|
||||
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
|
||||
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
|
||||
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_time_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
|
||||
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
|
||||
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
|
||||
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
|
||||
"AND remote_activation_replayed IS NULL) OR "
|
||||
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
|
||||
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
|
||||
"AND remote_activation_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
|
||||
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_time_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
source_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
applied_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
phase: Mapped[KnowledgeFSWorkspaceCutoverPhase] = mapped_column(
|
||||
EnumText(KnowledgeFSWorkspaceCutoverPhase, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'inventory'"),
|
||||
default=KnowledgeFSWorkspaceCutoverPhase.INVENTORY,
|
||||
)
|
||||
final_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
|
||||
sa.JSON, nullable=True, default=None
|
||||
)
|
||||
source_task_watermark: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
final_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
applied_task_watermark: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
shadow_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_evidence_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
shadow_observation_count: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
shadow_window_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_window_ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_traffic_zero: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
shadow_traffic_zero_evidence: Mapped[dict[str, object] | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
shadow_latest_observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
shadow_producer: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
shadow_completed_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
shadow_completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_freeze_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
remote_freeze_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_freeze_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
remote_freeze_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_freeze_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_freeze_frozen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_freeze_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_activation_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
remote_activation_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_activation_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
remote_activation_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_activation_activated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_activation_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
freeze_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
cutover_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
rollback_cutoff_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_window_ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
maximum_task_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
irreversible_cleanup_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
product_routes_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
capability_v2_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
integrated_mode_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
legacy_acl_read_only: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
smoke_results: Mapped[KnowledgeFSCutoverSmokeResults | None] = mapped_column(sa.JSON, nullable=True, default=None)
|
||||
legacy_dependency_report: Mapped[list[dict[str, object]] | None] = mapped_column(
|
||||
sa.JSON, nullable=True, default=None
|
||||
)
|
||||
legacy_dependency_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
legacy_dependency_ready: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssue(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Auditable blocker; unknown access can only be approved as fail-closed."""
|
||||
|
||||
__tablename__ = "knowledge_fs_migration_issues"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_issue_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_migration_issue_gate_idx", "tenant_id", "ledger_id", "status", "kind"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
issue_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
kind: Mapped[KnowledgeFSMigrationIssueKind] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueKind, length=48), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'open'"),
|
||||
default=KnowledgeFSMigrationIssueStatus.OPEN,
|
||||
)
|
||||
resource_type: Mapped[str | None] = mapped_column(String(64), nullable=True, default=None)
|
||||
resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantine(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Fail-closed inventory item whose operator resolution is immutable and CAS-versioned."""
|
||||
|
||||
__tablename__ = "knowledge_fs_migration_quarantine"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"source_kind",
|
||||
"source_id",
|
||||
name="kfs_migration_quarantine_source_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_quarantine_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_migration_quarantine_disposition_idx", "tenant_id", "ledger_id", "disposition"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_migration_quarantine_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"(disposition = 'resolved' "
|
||||
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
|
||||
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
|
||||
"(disposition <> 'resolved' "
|
||||
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
|
||||
"AND evidence IS NULL AND resolved_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_migration_quarantine_resolution_fields_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
source_kind: Mapped[KnowledgeFSMigrationQuarantineKind] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationQuarantineKind, length=32), nullable=False
|
||||
)
|
||||
source_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
reason_code: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
disposition: Mapped[KnowledgeFSMigrationQuarantineDisposition] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationQuarantineDisposition, length=32), nullable=False
|
||||
)
|
||||
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
|
||||
resolved_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
evidence: Mapped[dict[str, object] | None] = mapped_column(sa.JSON(none_as_null=True), nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationDiff(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Current disposition for a diff key; append-only observations retain its history."""
|
||||
|
||||
__tablename__ = "knowledge_fs_shadow_authorization_diffs"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_diff_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_shadow_authorization_diff_gate_idx", "tenant_id", "ledger_id", "status", "decision"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_shadow_authorization_diff_version_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
|
||||
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
|
||||
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(LongText, nullable=False)
|
||||
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueStatus, length=32), nullable=False
|
||||
)
|
||||
current_evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
last_observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationObservation(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Append-only producer evidence used to derive a shadow diff disposition."""
|
||||
|
||||
__tablename__ = "knowledge_fs_shadow_authorization_observations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"diff_key",
|
||||
"evidence_digest",
|
||||
name="kfs_shadow_authorization_observation_evidence_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_observation_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"observed_at",
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
producer: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
|
||||
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
|
||||
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(LongText, nullable=False)
|
||||
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCutoverRevisionWatermark",
|
||||
"KnowledgeFSCutoverSmokeChecks",
|
||||
"KnowledgeFSCutoverSmokeEvidenceReferences",
|
||||
"KnowledgeFSCutoverSmokeResults",
|
||||
"KnowledgeFSMigrationIssue",
|
||||
"KnowledgeFSMigrationIssueKind",
|
||||
"KnowledgeFSMigrationIssueStatus",
|
||||
"KnowledgeFSMigrationQuarantine",
|
||||
"KnowledgeFSMigrationQuarantineDisposition",
|
||||
"KnowledgeFSMigrationQuarantineKind",
|
||||
"KnowledgeFSShadowAuthorizationDecision",
|
||||
"KnowledgeFSShadowAuthorizationDiff",
|
||||
"KnowledgeFSShadowAuthorizationObservation",
|
||||
"KnowledgeFSWorkspaceCutoverLedger",
|
||||
"KnowledgeFSWorkspaceCutoverPhase",
|
||||
"knowledge_fs_cutover_smoke_results_passed",
|
||||
]
|
||||
@@ -1476,6 +1476,8 @@ class WorkflowRunArchiveBundle(DefaultFieldsDCMixin, TypeBase):
|
||||
name="workflow_run_archive_bundle_identity_uq",
|
||||
),
|
||||
sa.Index("workflow_run_archive_bundle_tenant_month_idx", "tenant_id", "year", "month"),
|
||||
sa.Index("workflow_run_archive_bundle_month_id_idx", "year", "month", "id"),
|
||||
sa.Index("workflow_run_archive_bundle_month_shard_id_idx", "year", "month", "shard", "id"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
|
||||
@@ -9639,6 +9639,27 @@ Bedrock retrieval test (internal use only)
|
||||
| ---- | ----------- | ------ |
|
||||
| 200 | Success | **application/json**: [TrialDatasetListResponse](#trialdatasetlistresponse)<br> |
|
||||
|
||||
### [POST] /trial-apps/{app_id}/files/upload
|
||||
**Upload a file into the tenant that owns the trial app**
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Located in | Description | Required | Schema |
|
||||
| ---- | ---------- | ----------- | -------- | ------ |
|
||||
| app_id | path | | Yes | string (uuid) |
|
||||
|
||||
#### Request Body
|
||||
|
||||
| Required | Schema |
|
||||
| -------- | ------ |
|
||||
| Yes | **multipart/form-data**: { **"file"**: binary, **"source"**: string, <br>**Available values:** "datasets" }<br> |
|
||||
|
||||
#### Responses
|
||||
|
||||
| Code | Description | Schema |
|
||||
| ---- | ----------- | ------ |
|
||||
| 201 | File uploaded successfully | **application/json**: [FileResponse](#fileresponse)<br> |
|
||||
|
||||
### [GET] /trial-apps/{app_id}/messages/{message_id}/suggested-questions
|
||||
#### Parameters
|
||||
|
||||
@@ -9668,6 +9689,27 @@ Bedrock retrieval test (internal use only)
|
||||
| ---- | ----------- | ------ |
|
||||
| 200 | Success | **application/json**: [Parameters](#parameters)<br> |
|
||||
|
||||
### [POST] /trial-apps/{app_id}/remote-files/upload
|
||||
**Upload a remote file into the tenant that owns the trial app**
|
||||
|
||||
#### Parameters
|
||||
|
||||
| Name | Located in | Description | Required | Schema |
|
||||
| ---- | ---------- | ----------- | -------- | ------ |
|
||||
| app_id | path | | Yes | string (uuid) |
|
||||
|
||||
#### Request Body
|
||||
|
||||
| Required | Schema |
|
||||
| -------- | ------ |
|
||||
| Yes | **application/json**: [RemoteFileUploadPayload](#remotefileuploadpayload)<br> |
|
||||
|
||||
#### Responses
|
||||
|
||||
| Code | Description | Schema |
|
||||
| ---- | ----------- | ------ |
|
||||
| 201 | File uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |
|
||||
|
||||
### [GET] /trial-apps/{app_id}/site
|
||||
**Retrieve app site info**
|
||||
|
||||
@@ -21859,6 +21901,7 @@ The subscription constructor of the trigger provider
|
||||
|
||||
| Name | Type | Description | Required |
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| _is_collaborative | boolean | | No |
|
||||
| conversation_variables | [ object ] | | No |
|
||||
| environment_variables | [ object ] | | No |
|
||||
| features | object | | Yes |
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Persistence contract for irreversible KnowledgeFS cleanup authorization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs_cleanup import (
|
||||
KnowledgeFSCleanupAuthorization,
|
||||
KnowledgeFSCleanupAuthorizationStatus,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSCleanupAuthorizationCASUpdate:
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
request_id: str
|
||||
expected_status: KnowledgeFSCleanupAuthorizationStatus
|
||||
expected_row_version: int
|
||||
new_status: KnowledgeFSCleanupAuthorizationStatus
|
||||
approved_by_account_id: str | None = None
|
||||
approved_at: datetime | None = None
|
||||
approval_expires_at: datetime | None = None
|
||||
approved_ledger_cas_version: int | None = None
|
||||
started_by_account_id: str | None = None
|
||||
started_at: datetime | None = None
|
||||
started_ledger_cas_version: int | None = None
|
||||
completed_by_account_id: str | None = None
|
||||
completed_at: datetime | None = None
|
||||
completion_evidence: dict[str, object] | None = None
|
||||
completed_ledger_cas_version: int | None = None
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorizationRepository(Protocol):
|
||||
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization: ...
|
||||
|
||||
def get(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
request_id: str,
|
||||
) -> KnowledgeFSCleanupAuthorization | None: ...
|
||||
|
||||
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool: ...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCleanupAuthorizationCASUpdate",
|
||||
"KnowledgeFSCleanupAuthorizationRepository",
|
||||
]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Persistence boundary for tenant-scoped KnowledgeFS lifecycle operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSControlSpaceCASUpdate:
|
||||
"""Values applied only when the stored state and resource version match."""
|
||||
|
||||
tenant_id: str
|
||||
control_space_id: str
|
||||
expected_resource_version: int
|
||||
expected_state: KnowledgeFSControlSpaceState
|
||||
new_state: KnowledgeFSControlSpaceState
|
||||
lifecycle_operation_id: str
|
||||
knowledge_space_id: str | None = None
|
||||
knowledge_space_revision: int | None = None
|
||||
attempted_at: datetime | None = None
|
||||
last_error_code: str | None = None
|
||||
last_error_message: str | None = None
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceRepository(Protocol):
|
||||
"""Repository operations that always include the owning Dify tenant."""
|
||||
|
||||
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace: ...
|
||||
|
||||
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def find_by_knowledge_space_id(
|
||||
self, *, tenant_id: str, knowledge_space_id: str
|
||||
) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]: ...
|
||||
|
||||
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]: ...
|
||||
|
||||
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool: ...
|
||||
|
||||
def mark_deletion_irreversible(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
lifecycle_operation_id: str,
|
||||
irreversible_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]: ...
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSControlSpaceCASUpdate", "KnowledgeFSControlSpaceRepository"]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Persistence contract for KnowledgeFS workspace migration and cutover state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs_cutover import (
|
||||
KnowledgeFSCutoverRevisionWatermark,
|
||||
KnowledgeFSCutoverSmokeResults,
|
||||
KnowledgeFSMigrationIssue,
|
||||
KnowledgeFSMigrationIssueStatus,
|
||||
KnowledgeFSMigrationQuarantine,
|
||||
KnowledgeFSMigrationQuarantineDisposition,
|
||||
KnowledgeFSMigrationQuarantineKind,
|
||||
KnowledgeFSShadowAuthorizationDecision,
|
||||
KnowledgeFSShadowAuthorizationDiff,
|
||||
KnowledgeFSShadowAuthorizationObservation,
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSCutoverCASUpdate:
|
||||
"""Optional values changed with the phase and CAS version in one SQL statement."""
|
||||
|
||||
tenant_id: str
|
||||
expected_phase: KnowledgeFSWorkspaceCutoverPhase
|
||||
expected_cas_version: int
|
||||
new_phase: KnowledgeFSWorkspaceCutoverPhase
|
||||
source_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
final_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
applied_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
source_task_watermark: int | None = None
|
||||
final_task_watermark: int | None = None
|
||||
applied_task_watermark: int | None = None
|
||||
shadow_started_at: datetime | None = None
|
||||
shadow_completed_at: datetime | None = None
|
||||
shadow_evidence_digest: str | None = None
|
||||
shadow_observation_count: int | None = None
|
||||
shadow_window_started_at: datetime | None = None
|
||||
shadow_window_ended_at: datetime | None = None
|
||||
shadow_traffic_zero: bool | None = None
|
||||
shadow_traffic_zero_evidence: dict[str, object] | None = None
|
||||
shadow_latest_observed_revision: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
shadow_producer: str | None = None
|
||||
shadow_completed_by_operator: str | None = None
|
||||
shadow_completed_by_account_id: str | None = None
|
||||
remote_freeze_id: str | None = None
|
||||
remote_freeze_revision: int | None = None
|
||||
remote_freeze_digest: str | None = None
|
||||
remote_freeze_task_watermark: int | None = None
|
||||
remote_freeze_control_space_id: str | None = None
|
||||
remote_freeze_frozen_at: datetime | None = None
|
||||
remote_freeze_updated_at: datetime | None = None
|
||||
remote_freeze_acknowledged_at: datetime | None = None
|
||||
remote_freeze_applied: bool | None = None
|
||||
remote_freeze_replayed: bool | None = None
|
||||
remote_activation_id: str | None = None
|
||||
remote_activation_revision: int | None = None
|
||||
remote_activation_digest: str | None = None
|
||||
remote_activation_control_space_id: str | None = None
|
||||
remote_activation_activated_at: datetime | None = None
|
||||
remote_activation_updated_at: datetime | None = None
|
||||
remote_activation_acknowledged_at: datetime | None = None
|
||||
remote_activation_applied: bool | None = None
|
||||
remote_activation_replayed: bool | None = None
|
||||
clear_smoke_results: bool = False
|
||||
freeze_at: datetime | None = None
|
||||
cutover_at: datetime | None = None
|
||||
rolled_back_at: datetime | None = None
|
||||
rollback_cutoff_at: datetime | None = None
|
||||
observation_started_at: datetime | None = None
|
||||
observation_window_ends_at: datetime | None = None
|
||||
observation_completed_at: datetime | None = None
|
||||
maximum_task_expires_at: datetime | None = None
|
||||
irreversible_cleanup_at: datetime | None = None
|
||||
smoke_results: KnowledgeFSCutoverSmokeResults | None = None
|
||||
legacy_dependency_report: list[dict[str, object]] | None = None
|
||||
legacy_dependency_checked_at: datetime | None = None
|
||||
legacy_dependency_ready: bool | None = None
|
||||
product_routes_enabled: bool | None = None
|
||||
capability_v2_enabled: bool | None = None
|
||||
integrated_mode_enabled: bool | None = None
|
||||
legacy_acl_read_only: bool | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSQuarantineCASUpdate:
|
||||
"""Tenant/item-scoped transition from an operator disposition to resolved."""
|
||||
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
source_kind: KnowledgeFSMigrationQuarantineKind
|
||||
source_id: str
|
||||
expected_disposition: KnowledgeFSMigrationQuarantineDisposition
|
||||
expected_row_version: int
|
||||
resolved_by_operator: str
|
||||
resolved_by_account_id: str
|
||||
evidence: dict[str, object]
|
||||
resolved_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSShadowDiffCASUpdate:
|
||||
"""Replace one current diff disposition after append-only safe re-evaluation."""
|
||||
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
diff_key: str
|
||||
expected_row_version: int
|
||||
control_space_id: str | None
|
||||
principal: str
|
||||
legacy_allowed: bool | None
|
||||
dify_allowed: bool
|
||||
decision: KnowledgeFSShadowAuthorizationDecision
|
||||
reason: str
|
||||
observed_revision: KnowledgeFSCutoverRevisionWatermark
|
||||
status: KnowledgeFSMigrationIssueStatus
|
||||
current_evidence_digest: str
|
||||
last_observed_at: datetime
|
||||
|
||||
|
||||
class KnowledgeFSCutoverRepository(Protocol):
|
||||
"""Tenant-scoped storage; callers own the surrounding transaction."""
|
||||
|
||||
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger: ...
|
||||
|
||||
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None: ...
|
||||
|
||||
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]: ...
|
||||
|
||||
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool: ...
|
||||
|
||||
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue: ...
|
||||
|
||||
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None: ...
|
||||
|
||||
def set_issue_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
issue_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine: ...
|
||||
|
||||
def get_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kind: KnowledgeFSMigrationQuarantineKind,
|
||||
source_id: str,
|
||||
) -> KnowledgeFSMigrationQuarantine | None: ...
|
||||
|
||||
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool: ...
|
||||
|
||||
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff: ...
|
||||
|
||||
def get_shadow_diff(
|
||||
self, *, tenant_id: str, ledger_id: str, diff_key: str
|
||||
) -> KnowledgeFSShadowAuthorizationDiff | None: ...
|
||||
|
||||
def set_shadow_diff_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool: ...
|
||||
|
||||
def add_shadow_observation(
|
||||
self, observation: KnowledgeFSShadowAuthorizationObservation
|
||||
) -> KnowledgeFSShadowAuthorizationObservation: ...
|
||||
|
||||
def get_shadow_observation(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
evidence_digest: str,
|
||||
) -> KnowledgeFSShadowAuthorizationObservation | None: ...
|
||||
|
||||
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int: ...
|
||||
|
||||
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int: ...
|
||||
|
||||
def count_unresolved_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
|
||||
) -> int: ...
|
||||
|
||||
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]: ...
|
||||
|
||||
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]: ...
|
||||
|
||||
def list_shadow_diffs(
|
||||
self, *, tenant_id: str, ledger_id: str
|
||||
) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]: ...
|
||||
|
||||
def list_shadow_observations(
|
||||
self, *, tenant_id: str, ledger_id: str
|
||||
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]: ...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCutoverCASUpdate",
|
||||
"KnowledgeFSCutoverRepository",
|
||||
"KnowledgeFSQuarantineCASUpdate",
|
||||
"KnowledgeFSShadowDiffCASUpdate",
|
||||
]
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Persistence boundary for durable KnowledgeFS lifecycle commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSLifecycleOperation, KnowledgeFSLifecycleOutbox
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutboxRepository(Protocol):
|
||||
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox: ...
|
||||
|
||||
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def find_open_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def find_latest_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def reactivate_dead_letter(self, *, outbox_id: str) -> bool: ...
|
||||
|
||||
def supersede_unattempted(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def supersede_after_remote_absence(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
observed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
lease_owner: str,
|
||||
now: datetime,
|
||||
lease_duration: timedelta,
|
||||
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def acknowledge(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def schedule_retry(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
next_attempt_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def mark_dead_letter(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSLifecycleOutboxRepository"]
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Durable SQLAlchemy sink for sanitized KnowledgeFS capability issuance audits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSCapabilityClaimsSummary, KnowledgeFSCapabilityIssuanceAudit
|
||||
from services.knowledge_fs_capability import CapabilityIssuanceAuditEvent, CapabilityIssuanceAuditor
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor(CapabilityIssuanceAuditor):
|
||||
"""Commit the audit independently so token issuance fails closed on audit failure."""
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session]):
|
||||
self._session_maker = session_maker
|
||||
|
||||
@override
|
||||
def record(self, event: CapabilityIssuanceAuditEvent) -> None:
|
||||
serialized = event.model_dump(mode="json")
|
||||
trace_id = cast(str, serialized.pop("trace_id"))
|
||||
jti_hash = cast(str, serialized.pop("jti_hash"))
|
||||
audit = KnowledgeFSCapabilityIssuanceAudit(
|
||||
tenant_id=event.namespace_id,
|
||||
control_space_id=event.control_space_id,
|
||||
trace_id=trace_id,
|
||||
jti_hash=jti_hash,
|
||||
claims_summary=cast(KnowledgeFSCapabilityClaimsSummary, serialized),
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
session.add(audit)
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor"]
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
"""Transactional persistence for pre-signing KnowledgeFS capability reservations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import cast
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSCapabilityIssuanceReservationStatus,
|
||||
KnowledgeFSCapabilityReservationSummary,
|
||||
)
|
||||
from services.knowledge_fs_capability import (
|
||||
CAPABILITY_ISSUANCE_PROFILES,
|
||||
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
|
||||
CapabilityIssueRequest,
|
||||
)
|
||||
|
||||
_TERMINAL_RETENTION = timedelta(days=1)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservationError(RuntimeError):
|
||||
"""A retry attempted to change or lose a durable issuance binding."""
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self._session = session
|
||||
|
||||
def reserve(self, request: CapabilityIssueRequest) -> KnowledgeFSCapabilityIssuanceReservation:
|
||||
summary = _request_summary(request)
|
||||
reservation = self._session.scalar(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.tenant_id == request.namespace_id,
|
||||
KnowledgeFSCapabilityIssuanceReservation.grant_id == request.grant_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if reservation is None:
|
||||
reservation = KnowledgeFSCapabilityIssuanceReservation(
|
||||
tenant_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
grant_id=request.grant_id,
|
||||
trace_id=request.trace_id,
|
||||
subject=summary["subject"],
|
||||
caller_kind=request.caller_kind,
|
||||
request_summary=summary,
|
||||
)
|
||||
self._session.add(reservation)
|
||||
self._session.flush()
|
||||
return reservation
|
||||
if (
|
||||
reservation.control_space_id != request.control_space_id
|
||||
or reservation.trace_id != request.trace_id
|
||||
or reservation.subject != summary["subject"]
|
||||
or reservation.caller_kind != request.caller_kind
|
||||
or reservation.request_summary != summary
|
||||
):
|
||||
raise KnowledgeFSCapabilityIssuanceReservationError(
|
||||
"Capability grant retry does not match its durable reservation"
|
||||
)
|
||||
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.FAILED:
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED
|
||||
reservation.failed_at = None
|
||||
reservation.failure_code = None
|
||||
reservation.cleanup_after = None
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
return reservation
|
||||
|
||||
def mark_issued(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
grant_id: str,
|
||||
issued_at: datetime,
|
||||
token_expires_at: datetime,
|
||||
) -> None:
|
||||
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED
|
||||
reservation.issued_at = issued_at
|
||||
reservation.token_expires_at = token_expires_at
|
||||
reservation.failed_at = None
|
||||
reservation.failure_code = None
|
||||
reservation.cleanup_after = token_expires_at + _TERMINAL_RETENTION
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
|
||||
def mark_failed(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
grant_id: str,
|
||||
failed_at: datetime,
|
||||
failure_code: str,
|
||||
) -> None:
|
||||
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
|
||||
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED:
|
||||
return
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.FAILED
|
||||
reservation.issued_at = None
|
||||
reservation.token_expires_at = None
|
||||
reservation.failed_at = failed_at
|
||||
reservation.failure_code = failure_code[:128]
|
||||
reservation.cleanup_after = failed_at + _TERMINAL_RETENTION
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
|
||||
def cleanup_terminal(self, *, before: datetime, limit: int = 1_000) -> int:
|
||||
if limit <= 0:
|
||||
raise ValueError("Capability reservation cleanup limit must be positive")
|
||||
reservation_ids = tuple(
|
||||
self._session.scalars(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation.id)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.status
|
||||
!= KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
|
||||
KnowledgeFSCapabilityIssuanceReservation.cleanup_after <= before,
|
||||
)
|
||||
.order_by(
|
||||
KnowledgeFSCapabilityIssuanceReservation.cleanup_after,
|
||||
KnowledgeFSCapabilityIssuanceReservation.id,
|
||||
)
|
||||
.limit(limit)
|
||||
.with_for_update()
|
||||
)
|
||||
)
|
||||
if not reservation_ids:
|
||||
return 0
|
||||
result = self._session.execute(
|
||||
sa.delete(KnowledgeFSCapabilityIssuanceReservation).where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.id.in_(reservation_ids)
|
||||
)
|
||||
)
|
||||
return cast(CursorResult[tuple[object, ...]], result).rowcount or 0
|
||||
|
||||
def _locked(self, *, tenant_id: str, grant_id: str) -> KnowledgeFSCapabilityIssuanceReservation:
|
||||
reservation = self._session.scalar(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.tenant_id == tenant_id,
|
||||
KnowledgeFSCapabilityIssuanceReservation.grant_id == grant_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if reservation is None:
|
||||
raise KnowledgeFSCapabilityIssuanceReservationError("Capability issuance reservation is missing")
|
||||
return reservation
|
||||
|
||||
|
||||
def _request_summary(request: CapabilityIssueRequest) -> KnowledgeFSCapabilityReservationSummary:
|
||||
operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[request.operation_id]
|
||||
profile = CAPABILITY_ISSUANCE_PROFILES[request.caller_kind]
|
||||
subject = f"{profile.subject_prefix}:{request.principal_id}"
|
||||
return cast(
|
||||
KnowledgeFSCapabilityReservationSummary,
|
||||
{
|
||||
"action": operation.action,
|
||||
"actor": request.actor,
|
||||
"authz_revision": request.authz_revision.model_dump(mode="json"),
|
||||
"caller_kind": request.caller_kind,
|
||||
"content_policy_revision": request.content_policy_revision,
|
||||
"content_scope_ids": list(request.content_scope_ids),
|
||||
"control_space_id": request.control_space_id,
|
||||
"grant_id": request.grant_id,
|
||||
"namespace_id": request.namespace_id,
|
||||
"operation_id": request.operation_id,
|
||||
"resource_id": request.resource.id,
|
||||
"resource_parent_id": request.resource.parent_id,
|
||||
"resource_type": request.resource.type,
|
||||
"subject": subject,
|
||||
"trace_id": request.trace_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCapabilityIssuanceReservationError",
|
||||
"SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository",
|
||||
]
|
||||
@@ -0,0 +1,87 @@
|
||||
"""SQLAlchemy cleanup-authorization repository with status and version CAS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs_cleanup import KnowledgeFSCleanupAuthorization
|
||||
from repositories.knowledge_fs_cleanup_repository import (
|
||||
KnowledgeFSCleanupAuthorizationCASUpdate,
|
||||
KnowledgeFSCleanupAuthorizationRepository,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(KnowledgeFSCleanupAuthorizationRepository):
|
||||
_session: Session
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization:
|
||||
self._session.add(authorization)
|
||||
self._session.flush()
|
||||
return authorization
|
||||
|
||||
@override
|
||||
def get(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
request_id: str,
|
||||
) -> KnowledgeFSCleanupAuthorization | None:
|
||||
statement = (
|
||||
select(KnowledgeFSCleanupAuthorization)
|
||||
.where(
|
||||
KnowledgeFSCleanupAuthorization.tenant_id == tenant_id,
|
||||
KnowledgeFSCleanupAuthorization.ledger_id == ledger_id,
|
||||
KnowledgeFSCleanupAuthorization.request_id == request_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"status": update_values.new_status,
|
||||
"row_version": KnowledgeFSCleanupAuthorization.row_version + 1,
|
||||
}
|
||||
for field_name in (
|
||||
"approved_by_account_id",
|
||||
"approved_at",
|
||||
"approval_expires_at",
|
||||
"approved_ledger_cas_version",
|
||||
"started_by_account_id",
|
||||
"started_at",
|
||||
"started_ledger_cas_version",
|
||||
"completed_by_account_id",
|
||||
"completed_at",
|
||||
"completion_evidence",
|
||||
"completed_ledger_cas_version",
|
||||
):
|
||||
value = getattr(update_values, field_name)
|
||||
if value is not None:
|
||||
values[field_name] = value
|
||||
statement = (
|
||||
update(KnowledgeFSCleanupAuthorization)
|
||||
.where(
|
||||
KnowledgeFSCleanupAuthorization.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSCleanupAuthorization.ledger_id == update_values.ledger_id,
|
||||
KnowledgeFSCleanupAuthorization.request_id == update_values.request_id,
|
||||
KnowledgeFSCleanupAuthorization.status == update_values.expected_status,
|
||||
KnowledgeFSCleanupAuthorization.row_version == update_values.expected_row_version,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSCleanupAuthorizationRepository"]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""SQLAlchemy implementation of the KnowledgeFS control-space repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy import case, select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
from repositories.knowledge_fs_control_space_repository import (
|
||||
KnowledgeFSControlSpaceCASUpdate,
|
||||
KnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSControlSpaceRepository(KnowledgeFSControlSpaceRepository):
|
||||
"""Caller-transaction-owned repository with tenant and CAS guards."""
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace:
|
||||
self._session.add(control_space)
|
||||
self._session.flush()
|
||||
return control_space
|
||||
|
||||
@override
|
||||
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.provisioning_key == provisioning_key)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_by_knowledge_space_id(self, *, tenant_id: str, knowledge_space_id: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.knowledge_space_id == knowledge_space_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
|
||||
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]:
|
||||
if limit <= 0:
|
||||
return ()
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED)
|
||||
.order_by(
|
||||
case(
|
||||
(
|
||||
KnowledgeFSControlSpace.state.in_(
|
||||
(
|
||||
KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
)
|
||||
),
|
||||
0,
|
||||
),
|
||||
else_=1,
|
||||
),
|
||||
KnowledgeFSControlSpace.updated_at,
|
||||
KnowledgeFSControlSpace.id,
|
||||
)
|
||||
.limit(limit)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"state": update_values.new_state,
|
||||
"resource_version": KnowledgeFSControlSpace.resource_version + 1,
|
||||
"lifecycle_operation_id": update_values.lifecycle_operation_id,
|
||||
"last_error_code": update_values.last_error_code,
|
||||
"last_error_message": update_values.last_error_message,
|
||||
}
|
||||
if update_values.knowledge_space_id is not None:
|
||||
values["knowledge_space_id"] = update_values.knowledge_space_id
|
||||
if update_values.knowledge_space_revision is not None:
|
||||
values["knowledge_space_revision"] = update_values.knowledge_space_revision
|
||||
if update_values.attempted_at is not None:
|
||||
values["attempt_count"] = KnowledgeFSControlSpace.attempt_count + 1
|
||||
values["last_attempt_at"] = update_values.attempted_at
|
||||
|
||||
statement = (
|
||||
update(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSControlSpace.id == update_values.control_space_id,
|
||||
KnowledgeFSControlSpace.resource_version == update_values.expected_resource_version,
|
||||
KnowledgeFSControlSpace.state == update_values.expected_state,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if update_values.knowledge_space_id is not None:
|
||||
statement = statement.where(
|
||||
(KnowledgeFSControlSpace.knowledge_space_id.is_(None))
|
||||
| (KnowledgeFSControlSpace.knowledge_space_id == update_values.knowledge_space_id)
|
||||
)
|
||||
if update_values.knowledge_space_revision is not None:
|
||||
statement = statement.where(
|
||||
KnowledgeFSControlSpace.knowledge_space_revision <= update_values.knowledge_space_revision
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def mark_deletion_irreversible(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
lifecycle_operation_id: str,
|
||||
irreversible_at: datetime,
|
||||
) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
KnowledgeFSControlSpace.lifecycle_operation_id == lifecycle_operation_id,
|
||||
KnowledgeFSControlSpace.state.in_(
|
||||
(
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
KnowledgeFSControlSpaceState.DELETED,
|
||||
)
|
||||
),
|
||||
(KnowledgeFSControlSpace.deletion_irreversible_at.is_(None))
|
||||
| (KnowledgeFSControlSpace.deletion_irreversible_at == irreversible_at),
|
||||
)
|
||||
.values(deletion_irreversible_at=irreversible_at)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace.id)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
|
||||
)
|
||||
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceRepository"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user