Compare commits

...
Author SHA1 Message Date
Jyong d74499c09d update github action 2026-07-22 06:07:12 -04:00
Jyong 29858d0db0 Expand application capabilities and refactor core workflows 2026-07-22 05:09:09 -04:00
Jyong 77732c7bb1 docs: decouple KnowledgeFS from Dify datasets 2026-07-20 06:02:06 -04:00
Jyong dde79fc9bf docs: add Dify KnowledgeFS integration plan 2026-07-20 05:33:56 -04:00
Jyong 4ee43b8afc chore: migrate knowledge-fs source tree
Import the committed KnowledgeFS snapshot dc4072ee302317145612087ce7440851dc329fd0 under knowledge-fs/ without its Git history, local IDE settings, or build artifacts.
2026-07-20 04:54:20 -04:00
yyhandGitHub 873ef2b592 docs: refine frontend testing guidance (#39290) 2026-07-20 08:18:26 +00:00
Stephen ZhouGitHubautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
43d725a1ce feat(dataset): proxy KnowledgeFS Console requests (#39158)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2026-07-20 08:10:06 +00:00
JoelandGitHub c8ccfba960 fix: the input of agent chat would be replaced if conversation history updated (#39289) 2026-07-20 08:07:12 +00:00
eee8d6cf7b feat(web): add webapp access control to agent access point (#39285)
Co-authored-by: yyh <yuanyouhuilyz@gmail.com>
2026-07-20 08:05:29 +00:00
zyssyz123andGitHub 11016563b1 fix(agent): preserve complete model usage pricing (#39201) 2026-07-20 08:03:13 +00:00
Ingram ZandGitHub b7edc5ef0c refactor: use catalog for workflow archive maintenance (#39001) 2026-07-20 07:59:29 +00:00
林玮 (Jade Lin)GitHubautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
f93a5b95c6 fix(api, web): scope trial app file uploads to the app tenant (#39149)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2026-07-20 07:00:45 +00:00
Kevin9703GitHubhjlarryautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>yyhyyh
6022939bf0 fix: prevent hidden-tab collaboration leader from saving stale drafts (#38997)
Co-authored-by: hjlarry <hjlarry@163.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: yyh <yuanyouhuilyz@gmail.com>
Co-authored-by: yyh <92089059+lyzno1@users.noreply.github.com>
2026-07-20 06:58:01 +00:00
非法操作andGitHub 30df1433cc fix: can't create chat app when get tenant default model schema error (#39266) 2026-07-20 06:47:16 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
aa7b65c602 chore: bump the github-actions-dependencies group with 5 updates (#39247)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-07-20 06:47:01 +00:00
yyhandGitHub ba24745c07 fix(web): restore permission selector semantics (#39269)
Signed-off-by: yyh <yuanyouhuilyz@gmail.com>
2026-07-20 06:30:09 +00:00
38ea0de907 refactor(web): avoid direct setState in effect in jina reader crawler (#25194) (#39216)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
2026-07-20 06:28:08 +00:00
JoelandGitHub bbc31a3403 fix: text generation web app keeps loading after timeout (#39268) 2026-07-20 06:27:32 +00:00
605c0fe02b refactor(web): avoid setState in website crawler effects (#39237)
Co-authored-by: Wu Tianwei <30284043+WTW0313@users.noreply.github.com>
2026-07-20 06:26:50 +00:00
yyhandGitHub 15674a9db6 test(web): align test state boundaries (#39239) 2026-07-20 06:06:17 +00:00
非法操作GitHubautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
9a2d039e93 chore: the node should not draggable when preview a workflow (#39131)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2026-07-20 05:59:14 +00:00
yyhandGitHub 58d1dd8873 ci: route default Depot jobs to 4-core runners (#39277) 2026-07-20 05:42:56 +00:00
ChecoandGitHub 5ea884f799 fix(dataset_service): use flush() instead of commit() to preserve caller transaction (#39223) 2026-07-19 07:00:50 +00:00
yyhGitHubautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2ec34b2cfb feat(workflow): improve block selector navigation and previews (#39212)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2026-07-18 09:56:36 +00:00
fa65f45bff refactor(web): align integrations sidebar components (#39202)
Co-authored-by: Jingyi <jingyi.qi@dify.ai>
2026-07-17 22:52:47 +00:00
yyhGitHubautofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
63ca2b94b5 refactor(web): use generated OAuth query options (#39204)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
2026-07-17 14:50:56 +00:00
yyhandGitHub a7ef41a8f5 refactor(web): use generated dataset card queries (#39203) 2026-07-17 14:50:40 +00:00
2369 changed files with 571772 additions and 13008 deletions
+2 -3
View File
@@ -9,12 +9,11 @@ Use this skill for Vitest work under `web/` and `packages/dify-ui/`. Do not use
## Required Source
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill defines the execution workflow and must not add requirements that conflict with or duplicate that guide.
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill provides an execution checklist and must not redefine or extend that policy.
## Workflow
1. Read the source, its behavior owner, nearby specs, and relevant public dependencies.
1. Identify whether the contract belongs in `web/`, Dify UI Browser Mode, or a styled Storybook test.
1. Apply the canonical guide to decide whether a test is needed and choose its boundary.
1. For a behavior change or bug fix, write or identify the failing scenario first when practical.
1. Implement one coherent scenario at a time and run the focused spec before expanding scope.
@@ -33,4 +32,4 @@ vp test run path/to/spec-or-directory
vp test run --project unit src/path/to/spec
```
For styled Dify UI behavior, run `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
Run Dify UI Storybook tests with `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
+15
View File
@@ -0,0 +1,15 @@
{
"hooks": {
"PreToolUse": [
{
"matcher": "Bash",
"hooks": [
{
"type": "command",
"command": "npx -y block-no-verify@1.1.1"
}
]
}
]
}
}
+1
View File
@@ -8,6 +8,7 @@
**/*.pyc
**/.mypy_cache
**/.ruff_cache
knowledge-fs/
.git
.github
*.md
+9
View File
@@ -1,6 +1,15 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/knowledge-fs"
open-pull-requests-limit: 10
schedule:
interval: "weekly"
groups:
knowledge-fs-dependencies:
patterns:
- "*"
- package-ecosystem: "uv"
directory: "/api"
open-pull-requests-limit: 10
+3 -3
View File
@@ -16,7 +16,7 @@ concurrency:
jobs:
api-unit:
name: API Unit Tests
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
env:
COVERAGE_FILE: coverage-unit
defaults:
@@ -75,7 +75,7 @@ jobs:
api-integration:
name: API Integration Tests
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
env:
COVERAGE_FILE: coverage-integration
STORAGE_TYPE: opendal
@@ -129,7 +129,7 @@ jobs:
api-coverage:
name: API Coverage
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
needs:
- api-unit
- api-integration
+1 -1
View File
@@ -173,7 +173,7 @@ jobs:
create-manifest:
needs: build
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: github.repository == 'langgenius/dify'
strategy:
matrix:
+2 -2
View File
@@ -23,7 +23,7 @@ concurrency:
jobs:
validate:
name: validate manifest + resolve target Dify release
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: github.repository == 'langgenius/dify'
permissions:
contents: read
@@ -87,7 +87,7 @@ jobs:
release:
name: build + attach standalone binaries (all targets)
needs: validate
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
contents: write
defaults:
+2 -2
View File
@@ -9,7 +9,7 @@ concurrency:
jobs:
db-migration-test-postgres:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
@@ -59,7 +59,7 @@ jobs:
run: uv run --directory api flask upgrade-db
db-migration-test-mysql:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
+1 -1
View File
@@ -13,7 +13,7 @@ on:
jobs:
deploy:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'deploy/agent'
+1 -1
View File
@@ -10,7 +10,7 @@ on:
jobs:
deploy:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'deploy/dev'
+1 -1
View File
@@ -13,7 +13,7 @@ on:
jobs:
deploy:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'deploy/enterprise'
+1 -1
View File
@@ -13,7 +13,7 @@ on:
jobs:
deploy:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
if: |
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'deploy/saas'
+1 -1
View File
@@ -22,7 +22,7 @@ concurrency:
jobs:
check-cherry-pick-provenance:
name: Require cherry-pick provenance
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
+504
View File
@@ -0,0 +1,504 @@
name: KnowledgeFS CI
on:
pull_request:
branches: ["main"]
merge_group:
branches: ["main"]
types: [checks_requested]
push:
branches: ["main"]
workflow_dispatch:
permissions:
contents: read
pull-requests: read
concurrency:
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
CI: true
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
jobs:
check-changes:
name: Check KnowledgeFS changes
runs-on: depot-ubuntu-24.04-4
outputs:
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Detect KnowledgeFS changes
id: changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
with:
filters: |
knowledge-fs:
- 'knowledge-fs/**'
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
- 'knowledge-fs/scripts/export-openapi.mjs'
- 'api/dev/generate_knowledge_fs_contract.py'
- 'api/dev/knowledge_fs_product_contract.py'
- 'api/knowledge-fs-contract.lock.json'
- 'api/knowledge-fs-product-operation-gaps.json'
- 'api/knowledge-fs-product-operations.json'
- 'api/**/knowledge_fs/**'
- 'api/**/*knowledge_fs*'
- 'api/**/*knowledge-fs*'
- 'api/.env.example'
- 'api/app_factory.py'
- 'api/commands/__init__.py'
- 'api/controllers/console/__init__.py'
- 'api/controllers/console/workspace/rbac.py'
- 'api/controllers/service_api/__init__.py'
- 'api/core/agent/base_agent_runner.py'
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
- 'api/core/rbac/entities.py'
- 'api/core/tools/__base/tool_runtime.py'
- 'api/core/tools/builtin_tool/_position.yaml'
- 'api/core/workflow/node_runtime.py'
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
- 'api/extensions/ext_celery.py'
- 'api/extensions/ext_commands.py'
- 'api/models/__init__.py'
- 'api/services/account_service.py'
- 'api/services/agent_tool_inner_service.py'
- 'api/services/enterprise/rbac_service.py'
- 'api/services/entities/agent_tool_inner.py'
- 'api/services/knowledge_fs/**'
- 'api/services/knowledge_fs_capability.py'
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
- 'api/tests/unit_tests/services/test_account_service.py'
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
- 'api/pyproject.toml'
- 'api/uv.lock'
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
- 'packages/contracts/generated/api/console/**'
- 'packages/contracts/generated/api/service/**'
- 'docker/.env.example'
- 'docker/README.md'
- 'docker/dify-env-sync.py'
- 'docker/dify-env-sync.sh'
- 'docker/docker-compose-template.yaml'
- 'docker/docker-compose.yaml'
- 'docker/envs/core-services/api.env.example'
- 'docker/envs/core-services/knowledge-fs.env.example'
- 'docker/generate_docker_compose'
- 'docs/design/knowledge-fs*'
- '.github/dependabot.yml'
- '.github/workflows/knowledge-fs-ci.yml'
build:
name: Build KnowledgeFS API production image
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to Docker Hub
if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract KnowledgeFS image metadata
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=branch
type=sha,format=long
- name: Build KnowledgeFS API image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./knowledge-fs
file: ./knowledge-fs/apps/api/Dockerfile
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64
push: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
tags: ${{ steps.meta.outputs.tags }}
quality:
name: Run KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
working-directory: ./knowledge-fs
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
package_json_file: knowledge-fs/package.json
run_install: false
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: pnpm
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
- name: Install KnowledgeFS dependencies
run: pnpm install --frozen-lockfile
- name: Scan KnowledgeFS secrets
run: pnpm security:secrets
- name: Audit KnowledgeFS production dependencies
run: pnpm security:dependencies
- name: Run KnowledgeFS checks
run: pnpm check
- name: Build KnowledgeFS
run: pnpm build
- name: Lint KnowledgeFS
run: pnpm lint
- name: Setup UV and Python
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: "3.12"
cache-dependency-glob: |
api/uv.lock
dify-agent/uv.lock
- name: Verify Dify dependency lock
working-directory: .
run: uv lock --project api --check
- name: Install Dify contract dependencies
working-directory: .
run: uv sync --project api --locked --dev
- name: Collect Dify KnowledgeFS gate targets
working-directory: .
run: |
set -euo pipefail
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
mkdir -p "$target_dir"
production_targets=()
add_production_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS production target is missing: $path" >&2
exit 1
fi
production_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_production_target "$path"
fi
done < <(
find api \
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
-o -type f -name '*.py' -print0
)
production_touchpoints=(
api/app_factory.py
api/commands/__init__.py
api/controllers/console/__init__.py
api/controllers/console/workspace/rbac.py
api/controllers/service_api/__init__.py
api/core/agent/base_agent_runner.py
api/core/app/apps/agent_app/runtime_request_builder.py
api/core/rbac/entities.py
api/core/tools/__base/tool_runtime.py
api/core/workflow/node_runtime.py
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
api/extensions/ext_celery.py
api/extensions/ext_commands.py
api/models/__init__.py
api/services/account_service.py
api/services/agent_tool_inner_service.py
api/services/enterprise/rbac_service.py
api/services/entities/agent_tool_inner.py
)
for path in "${production_touchpoints[@]}"; do
add_production_target "$path"
done
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
if ((${#production_targets[@]} == 0)); then
echo "Dify KnowledgeFS production target set is empty" >&2
exit 1
fi
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
test_targets=()
add_test_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
exit 1
fi
test_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_test_target "$path"
fi
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
test_touchpoints=(
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
api/tests/unit_tests/core/agent/test_base_agent_runner.py
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
api/tests/unit_tests/core/workflow/test_node_runtime.py
api/tests/unit_tests/services/enterprise/test_rbac_service.py
api/tests/unit_tests/services/test_account_service.py
api/tests/unit_tests/services/test_agent_tool_inner_service.py
)
for path in "${test_touchpoints[@]}"; do
add_test_target "$path"
done
required_test_scopes=(
/commands/
/configs/
/controllers/
/core/agent/
/core/app/apps/agent_app/
/core/tools/builtin_tool/providers/knowledge_fs/
/core/workflow/
/dev/
/extensions/
/migrations/
/models/
/repositories/
/services/
/tasks/
)
for required_scope in "${required_test_scopes[@]}"; do
scope_found=false
for path in "${test_targets[@]}"; do
if [[ "$path" == *"$required_scope"* ]]; then
scope_found=true
break
fi
done
if [[ "$scope_found" != true ]]; then
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
exit 1
fi
done
if ((${#test_targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target set is empty" >&2
exit 1
fi
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
- name: Lint Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
uv run --project api --dev ruff format --check "${targets[@]}"
uv run --project api --dev ruff check "${targets[@]}"
- name: Type-check Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
mypy_targets=()
for path in "${targets[@]}"; do
if [[ "$path" != api/migrations/* ]]; then
mypy_targets+=("${path#api/}")
fi
done
if ((${#mypy_targets[@]} == 0)); then
echo "Dify KnowledgeFS Mypy target set is empty" >&2
exit 1
fi
uv run --directory api --dev mypy \
--explicit-package-bases \
--exclude-gitignore \
--exclude '(^|/)conftest\.py$' \
--exclude 'tests/' \
--exclude 'migrations/' \
--check-untyped-defs \
--disable-error-code=import-untyped \
"${mypy_targets[@]}"
- name: Test Dify KnowledgeFS unit surface
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
exit 1
fi
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
- name: Enforce Dify KnowledgeFS focused coverage
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
KNOWLEDGE_FS_COVERAGE_BASE: >-
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
run: |
set -euo pipefail
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
uv run --project api --dev coverage json --show-contexts -o "$report"
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
--coverage-json "$report" \
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
--base "$KNOWLEDGE_FS_COVERAGE_BASE" \
--minimum 90 \
--glue-minimum 90
- name: Verify Dify KnowledgeFS contract
working-directory: .
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
- name: Verify Dify Agent dependency lock
working-directory: .
run: uv lock --project dify-agent --check
- name: Install Dify Agent gate dependencies
working-directory: .
run: uv sync --project dify-agent --locked --dev
- name: Lint Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: |
uv run --project . --dev ruff format --check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
uv run --project . --dev ruff check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Type-check Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev basedpyright --level error
src/dify_agent/layers/dify_core_tools/client.py
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Test Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev python -m pytest
tests/local/dify_agent/layers/dify_core_tools/test_client.py
-q
skip:
name: Skip KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped KnowledgeFS checks
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
final:
name: KnowledgeFS CI
if: ${{ always() }}
needs:
- check-changes
- build
- quality
- skip
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize KnowledgeFS CI status
env:
EVENT_NAME: ${{ github.event_name }}
BUILD_RESULT: ${{ needs.build.result }}
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
QUALITY_RESULT: ${{ needs.quality.result }}
SKIP_RESULT: ${{ needs.skip.result }}
run: |
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
echo "KnowledgeFS build and checks ran successfully."
exit 0
fi
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
exit 1
fi
if [[ "$SKIP_RESULT" == 'success' ]]; then
echo "KnowledgeFS checks were skipped because no related files changed."
exit 0
fi
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
exit 1
+1 -1
View File
@@ -7,7 +7,7 @@ jobs:
permissions:
contents: read
pull-requests: write
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- uses: actions/labeler@b8dd2d9be0f68b860e7dae5dae7d772984eacd6d # v6.2.0
with:
+17 -17
View File
@@ -21,13 +21,13 @@ concurrency:
jobs:
pre_job:
name: Skip Duplicate Checks
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip || 'false' }}
steps:
- id: skip_check
continue-on-error: true
uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
uses: fkirc/skip-duplicate-actions@b974a9395958c231af965b70070979a577efa578 # v5.3.2
with:
cancel_others: 'true'
concurrent_skipping: same_content_newer
@@ -37,7 +37,7 @@ jobs:
name: Check Changed Files
needs: pre_job
if: needs.pre_job.outputs.should_skip != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
outputs:
api-changed: ${{ steps.changes.outputs.api }}
cli-changed: ${{ steps.changes.outputs.cli }}
@@ -164,7 +164,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.api-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped API tests
run: echo "No API-related changes detected; skipping API tests."
@@ -177,7 +177,7 @@ jobs:
- check-changes
- api-tests-run
- api-tests-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize API Tests status
env:
@@ -224,7 +224,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.cli-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped CLI tests
run: echo "No CLI-related changes detected; skipping CLI tests."
@@ -237,7 +237,7 @@ jobs:
- check-changes
- cli-tests-run
- cli-tests-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize CLI Tests status
env:
@@ -284,7 +284,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.web-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped web tests
run: echo "No web-related changes detected; skipping web tests."
@@ -297,7 +297,7 @@ jobs:
- check-changes
- web-tests-run
- web-tests-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize Web Tests status
env:
@@ -344,7 +344,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped web full-stack e2e
run: echo "No E2E-related changes detected; skipping web full-stack E2E."
@@ -357,7 +357,7 @@ jobs:
- check-changes
- web-e2e-run
- web-e2e-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize Web Full-Stack E2E status
env:
@@ -411,7 +411,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.vdb-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped VDB tests
run: echo "No VDB-related changes detected; skipping VDB tests."
@@ -424,7 +424,7 @@ jobs:
- check-changes
- vdb-tests-run
- vdb-tests-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize VDB Tests status
env:
@@ -470,7 +470,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.migration-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped DB migration tests
run: echo "No migration-related changes detected; skipping DB migration tests."
@@ -483,7 +483,7 @@ jobs:
- check-changes
- db-migration-test-run
- db-migration-test-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize DB Migration Test status
env:
@@ -530,7 +530,7 @@ jobs:
- pre_job
- check-changes
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.sandbox-runtime-changed != 'true'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped sandbox runtime tests
run: echo "No sandbox-runtime-related changes detected; skipping sandbox runtime tests."
@@ -543,7 +543,7 @@ jobs:
- check-changes
- sandbox-runtime-tests-run
- sandbox-runtime-tests-skip
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize Sandbox Runtime Tests status
env:
+1 -1
View File
@@ -14,7 +14,7 @@ concurrency:
jobs:
check-changes:
name: Check Changed Files
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
outputs:
external-e2e-changed: ${{ steps.changes.outputs.external_e2e }}
steps:
+1 -1
View File
@@ -12,7 +12,7 @@ permissions: {}
jobs:
comment:
name: Comment PR with pyrefly diff
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
actions: read
contents: read
+1 -1
View File
@@ -10,7 +10,7 @@ permissions:
jobs:
pyrefly-diff:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
contents: read
issues: write
@@ -12,7 +12,7 @@ permissions: {}
jobs:
comment:
name: Comment PR with type coverage
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
actions: read
contents: read
+1 -1
View File
@@ -10,7 +10,7 @@ permissions:
jobs:
pyrefly-type-coverage:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
contents: read
issues: write
+7 -7
View File
@@ -13,7 +13,7 @@ concurrency:
jobs:
sandbox-runtime-unit:
name: Sandbox Runtime Unit Tests
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
@@ -27,7 +27,7 @@ jobs:
persist-credentials: false
- name: Setup Go
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: dify-agent-runtime/go.mod
cache-dependency-path: dify-agent-runtime/go.sum
@@ -37,7 +37,7 @@ jobs:
sandbox-runtime-lint:
name: Sandbox Runtime Lint
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
@@ -51,20 +51,20 @@ jobs:
persist-credentials: false
- name: Setup Go
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: dify-agent-runtime/go.mod
cache-dependency-path: dify-agent-runtime/go.sum
- name: Run golangci-lint
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v6.5.0
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6.5.0
with:
working-directory: dify-agent-runtime
version: latest
sandbox-runtime-integration:
name: Sandbox Runtime Integration Tests
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
@@ -78,7 +78,7 @@ jobs:
persist-credentials: false
- name: Setup Go
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: dify-agent-runtime/go.mod
cache-dependency-path: dify-agent-runtime/go.sum
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
name: Validate PR title
permissions:
pull-requests: read
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Complete merge group check
if: github.event_name == 'merge_group'
+1 -1
View File
@@ -12,7 +12,7 @@ on:
jobs:
stale:
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
permissions:
issues: write
pull-requests: write
+3 -3
View File
@@ -19,7 +19,7 @@ permissions:
jobs:
python-style:
name: Python Style
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
@@ -83,7 +83,7 @@ jobs:
web-style:
name: Web Style
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
defaults:
run:
working-directory: ./web
@@ -182,7 +182,7 @@ jobs:
superlinter:
name: SuperLinter
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
+2 -2
View File
@@ -17,7 +17,7 @@ concurrency:
jobs:
build:
name: unit test for Node.js SDK
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
defaults:
run:
@@ -29,7 +29,7 @@ jobs:
persist-credentials: false
- name: Use Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: ''
+2 -2
View File
@@ -35,7 +35,7 @@ concurrency:
jobs:
translate:
if: github.repository == 'langgenius/dify'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 120
steps:
@@ -158,7 +158,7 @@ jobs:
- name: Run Claude Code for Translation Sync
if: steps.context.outputs.CHANGED_FILES != ''
uses: anthropics/claude-code-action@e90deca47693f9457b72f2b53c17d7c445a87342 # v1.0.171
uses: anthropics/claude-code-action@af0559ee4f514d1ef21826982bed13f7edc3c35e # v1.0.178
with:
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
+1 -1
View File
@@ -16,7 +16,7 @@ concurrency:
jobs:
trigger:
if: github.repository == 'langgenius/dify'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
timeout-minutes: 5
steps:
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
test:
name: Full VDB Tests
if: github.repository == 'langgenius/dify'
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
strategy:
matrix:
python-version:
+1 -1
View File
@@ -13,7 +13,7 @@ concurrency:
jobs:
test:
name: VDB Smoke Tests
runs-on: depot-ubuntu-24.04
runs-on: depot-ubuntu-24.04-4
strategy:
matrix:
python-version:
+5
View File
@@ -30,6 +30,11 @@ share/python-wheels/
*.egg
MANIFEST
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
# directory contains source files rather than Python build output.
!/knowledge-fs/apps/admin/lib/
!/knowledge-fs/apps/admin/lib/**
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
+25
View File
@@ -681,6 +681,31 @@ AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# KnowledgeFS (Dataset 2.0)
KNOWLEDGE_FS_ENABLED=false
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
KNOWLEDGE_FS_BASE_URL=
KNOWLEDGE_FS_DIRECT_ORIGIN=
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
# Legacy rollback-only HMAC; Capability v2 deployments leave this blank.
KNOWLEDGE_FS_JWT_SECRET=
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
# Marketplace configuration
MARKETPLACE_ENABLED=true
MARKETPLACE_API_URL=https://marketplace.dify.ai
+2
View File
@@ -154,6 +154,7 @@ def initialize_extensions(app: DifyApp):
ext_forward_refs,
ext_hosting_provider,
ext_import_modules,
ext_knowledge_fs_observability,
ext_logging,
ext_login,
ext_logstore,
@@ -204,6 +205,7 @@ def initialize_extensions(app: DifyApp):
ext_enterprise_telemetry,
ext_request_logging,
ext_session_factory,
ext_knowledge_fs_observability,
ext_oauth_bearer,
]
for ext in extensions:
+2
View File
@@ -10,6 +10,7 @@ from .data_migration import (
import_migration_data,
migration_data_wizard,
)
from .knowledge_fs import knowledge_fs_control_space
from .plugin import (
backfill_plugin_auto_upgrade,
extract_plugins,
@@ -75,6 +76,7 @@ __all__ = [
"import_migration_data",
"install_plugins",
"install_rag_pipeline_plugins",
"knowledge_fs_control_space",
"legacy_model_types",
"migrate_annotation_vector_database",
"migrate_data_for_plugin",
+604
View File
@@ -0,0 +1,604 @@
"""Operator commands for the independent KnowledgeFS control-plane."""
from __future__ import annotations
import json
from collections.abc import Callable
from datetime import datetime
from functools import partial
from pathlib import Path
import click
from pydantic import BaseModel, ValidationError
from core.db.session_factory import session_factory
from services.knowledge_fs.cleanup import (
CleanupApprovalInput,
CleanupCompletionEvidenceInput,
CleanupReadinessEvidenceInput,
CleanupStartInput,
KnowledgeFSCleanupError,
KnowledgeFSCleanupService,
)
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
from services.knowledge_fs.control_space_management import (
KnowledgeFSControlSpaceManagementService,
KnowledgeFSControlSpaceRegistration,
)
from services.knowledge_fs.cutover import (
CutoverSmokeResultsInput,
FinalDeltaInput,
KnowledgeFSCutoverError,
KnowledgeFSWorkspaceCutoverService,
LegacyDependencyInput,
QuarantineResolutionInput,
ShadowAuthorizationObservationInput,
ShadowCompletionInput,
WorkspaceInventoryInput,
)
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
@click.group("knowledge-fs-control-space")
def knowledge_fs_control_space() -> None:
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
@knowledge_fs_control_space.command("dry-run")
@click.option("--tenant-id", default=None)
def dry_run(tenant_id: str | None) -> None:
report = _management_service().dry_run(tenant_id=tenant_id)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("inventory")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
def inventory(input_path: Path, apply: bool) -> None:
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.inventory, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("register")
@click.option("--tenant-id", required=True)
@click.option("--owner-account-id", required=True)
@click.option("--provisioning-key", required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def register(
tenant_id: str,
owner_account_id: str,
provisioning_key: str,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space, replayed = _management_service().register(
KnowledgeFSControlSpaceRegistration(
tenant_id,
owner_account_id,
provisioning_key,
knowledge_space_id,
knowledge_space_revision,
)
)
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
@knowledge_fs_control_space.command("backfill")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
def backfill(input_path: Path, apply: bool) -> None:
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.backfill, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("quarantine-resolve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
def quarantine_resolve(input_path: Path, apply: bool) -> None:
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
service = _cutover_service()
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-start")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
service = _cutover_service()
_operator_call(
lambda: service.begin_shadow(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at) if started_at is not None else None,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-report")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
def shadow_report(input_path: Path, apply: bool) -> None:
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
def shadow_complete(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("issue-approve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_issue_fail_closed(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("issue-resolve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_issue(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-approve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-resolve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("legacy-dashboard")
@click.option("--tenant-id", required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--checked-at", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
def legacy_dashboard(
tenant_id: str,
input_path: Path,
checked_at: str,
expected_cas_version: int | None,
apply: bool,
) -> None:
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
report = _operator_call(
lambda: _cutover_service().legacy_dependency_dashboard(
tenant_id=tenant_id,
dependencies=dependencies,
expected_cas_version=expected_cas_version,
checked_at=_parse_timestamp(checked_at),
apply=apply,
)
)
_echo_json(report._asdict())
@knowledge_fs_control_space.command("legacy-check")
@click.option("--tenant-id", required=True)
def legacy_check(tenant_id: str) -> None:
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
passed = (
bool(status_report["legacy_dependency_ready"])
and status_report["open_issues"] == 0
and status_report["unresolved_cutover_quarantine"] == 0
)
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
if not passed:
raise click.exceptions.Exit(1)
@knowledge_fs_control_space.command("freeze")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "freeze_at", required=True)
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.freeze(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
freeze_at=_parse_timestamp(freeze_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("final-delta")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def final_delta(input_path: Path) -> None:
payload = _read_one_jsonl(input_path, FinalDeltaInput)
service = _cutover_service()
_operator_call(lambda: service.apply_final_delta(payload))
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
@knowledge_fs_control_space.command("cutover")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "cutover_at", required=True)
@click.option("--rollback-cutoff-at", required=True)
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.cutover(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
cutover_at=_parse_timestamp(cutover_at),
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("smoke")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
service = _cutover_service()
_operator_call(
lambda: service.record_smoke_results(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
results=results,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("observe")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--started-at", default=None)
@click.option("--window-ends-at", default=None)
@click.option("--maximum-task-expires-at", default=None)
@click.option("--observed-at", default=None)
def observe(
tenant_id: str,
expected_cas_version: int,
started_at: str | None,
window_ends_at: str | None,
maximum_task_expires_at: str | None,
observed_at: str | None,
) -> None:
service = _cutover_service()
if observed_at is not None:
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
raise click.UsageError("--observed-at cannot be combined with observation start options")
_operator_call(
lambda: service.complete_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
observed_at=_parse_timestamp(observed_at),
)
)
else:
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
raise click.UsageError(
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
)
_operator_call(
lambda: service.begin_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at),
window_ends_at=_parse_timestamp(window_ends_at),
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("rollback")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "rolled_back_at", required=True)
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.rollback(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
rolled_back_at=_parse_timestamp(rolled_back_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("status")
@click.option("--tenant-id", required=True)
def status(tenant_id: str) -> None:
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
@knowledge_fs_control_space.command("cleanup-request")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
def cleanup_request(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-approve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
def cleanup_approve(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-start")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
@click.option(
"--acknowledge-irreversible",
is_flag=True,
default=False,
help="Required with --apply; confirms rollback will be permanently closed.",
)
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupStartInput)
if apply and not acknowledge_irreversible:
raise click.UsageError("--apply requires --acknowledge-irreversible")
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
@click.option(
"--acknowledge-executed",
is_flag=True,
default=False,
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
)
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
if apply and not acknowledge_executed:
raise click.UsageError("--apply requires --acknowledge-executed")
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-status")
@click.option("--tenant-id", required=True)
@click.option("--request-id", required=True)
def cleanup_status(tenant_id: str, request_id: str) -> None:
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
@knowledge_fs_control_space.command("repair")
@click.option("--tenant-id", required=True)
@click.option("--control-space-id", required=True)
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def repair(
tenant_id: str,
control_space_id: str,
expected_resource_version: int,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space = _management_service().repair_registration(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=expected_resource_version,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
@knowledge_fs_control_space.command("orphan-report")
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
def orphan_report(limit: int) -> None:
report = KnowledgeFSOrphanReconciler(
session_factory.get_session_maker(),
get_knowledge_fs_lifecycle_remote(),
).reconcile(limit=limit, apply_repairs=False)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("workspace-delete-request")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
if not apply:
report = _management_service().dry_run(tenant_id=tenant_id)
_echo_json(
{
"apply": False,
"by_state": report.by_state,
"tenant_id": tenant_id,
"total": report.total,
}
)
return
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
_echo_json(
{
"apply": True,
"control_space_ids": [result.control_space.id for result in results],
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
"tenant_id": tenant_id,
}
)
@knowledge_fs_control_space.command("workspace-delete-finalize")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
@click.option(
"--acknowledge-control-plane-purge",
is_flag=True,
default=False,
help="Required with --apply after every remote Space has reached deleted.",
)
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
"""Release the Workspace FK only after all remote deletions are terminal."""
service = _lifecycle_service()
if not apply:
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
return
if not acknowledge_control_plane_purge:
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
def _read_jsonl[InputT: BaseModel](
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
) -> tuple[InputT, ...]:
records: list[InputT] = []
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
records.append(input_type.model_validate_json(line))
except ValidationError as exc:
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
if not records and not allow_empty:
raise click.ClickException("strict JSONL input must contain at least one record")
return tuple(records)
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
records = _read_jsonl(input_path, input_type)
if len(records) != 1:
raise click.ClickException("this command requires exactly one JSONL record")
return records[0]
def _parse_timestamp(value: str) -> datetime:
try:
parsed = datetime.fromisoformat(value)
except ValueError as exc:
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
if parsed.tzinfo is None:
raise click.ClickException("operator timestamps must include an explicit timezone")
return parsed
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCutoverError as exc:
raise click.ClickException(str(exc)) from exc
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCleanupError as exc:
raise click.ClickException(str(exc)) from exc
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSControlSpaceLifecycleError as exc:
raise click.ClickException(str(exc)) from exc
def _echo_json(payload: object) -> None:
click.echo(json.dumps(payload, default=str, sort_keys=True))
def _management_service() -> KnowledgeFSControlSpaceManagementService:
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
return KnowledgeFSWorkspaceCutoverService(
session_factory.get_session_maker(),
remote_factory=get_knowledge_fs_lifecycle_remote,
)
def _cleanup_service() -> KnowledgeFSCleanupService:
return KnowledgeFSCleanupService(session_factory.get_session_maker())
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
__all__ = ["knowledge_fs_control_space"]
+209 -63
View File
@@ -1,6 +1,8 @@
import datetime
import logging
import re
import time
import uuid
from collections.abc import Callable
from typing import TypedDict
@@ -21,6 +23,7 @@ from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
logger = logging.getLogger(__name__)
_HEX_PREFIXES = tuple("0123456789abcdef")
_TARGET_MONTH_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
class WorkflowRunArchivePlanRow(TypedDict):
@@ -66,6 +69,7 @@ def _parse_tenant_prefixes(prefixes: str | None) -> list[str]:
def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[str] | None:
"""Keep an omitted scope unset while rejecting an explicitly empty scope."""
if raw_ids is None:
return None
parsed = sorted({raw_id.strip() for raw_id in raw_ids.split(",") if raw_id.strip()})
@@ -74,6 +78,27 @@ def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[
return parsed
def _parse_archive_target_month(target_month: str) -> tuple[int, int]:
"""Validate the V2 catalog month selector and return its numeric components."""
if not _TARGET_MONTH_PATTERN.fullmatch(target_month):
raise click.BadParameter("target-month must use YYYY-MM format", param_hint="--target-month")
year_text, month_text = target_month.split("-", maxsplit=1)
return int(year_text), int(month_text)
def _parse_archive_catalog_cursor(after_catalog_id: str | None) -> str | None:
"""Normalize the exclusive V2 catalog keyset cursor when one is provided."""
if after_catalog_id is None:
return None
try:
return str(uuid.UUID(after_catalog_id))
except ValueError as exc:
raise click.BadParameter(
"after-catalog-id must be a UUID returned by the same V2 operation and scope",
param_hint="--after-catalog-id",
) from exc
def _get_archive_candidate_tenant_ids_by_prefix(
session: Session,
prefix: str,
@@ -810,9 +835,11 @@ def backfill_workflow_run_archive_bundles(
click.echo(click.style(f" ... and {len(summary.errors) - 10} more failures", fg="red"))
def _echo_bundle_archive_operation_summary(summary) -> None:
def _echo_bundle_archive_operation_summary(summary, *, dry_run: bool) -> None:
status = "completed successfully" if summary.bundles_failed == 0 else "completed with failures"
fg = "green" if summary.bundles_failed == 0 else "red"
cursor_label = "preview_next_catalog_id" if dry_run else "next_catalog_id"
cursor_value = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
click.echo(
click.style(
f"{summary.operation} {status}. "
@@ -821,10 +848,12 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
f"archive_bytes={summary.archive_bytes} duration={summary.elapsed_time:.2f}s "
f"validation_time={summary.validation_time:.2f}s "
f"runs_per_second={summary.runs_per_second:.2f} rows_per_second={summary.rows_per_second:.2f} "
f"bytes_per_second={summary.bytes_per_second:.2f}",
f"bytes_per_second={summary.bytes_per_second:.2f} {cursor_label}={cursor_value or 'none'}",
fg=fg,
)
)
if dry_run:
click.echo(click.style("Dry-run cursor is preview-only; do not persist it for a destructive run.", fg="yellow"))
click.echo(click.style("table,row_count", fg="white"))
for table_name in [
"workflow_runs",
@@ -842,7 +871,8 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
click.style(
f" bundle={result.bundle_id} tenant={result.tenant_id} runs={result.run_count} "
f"rows={result.row_count} archive_bytes={result.archive_bytes} "
f"time={result.elapsed_time:.2f}s validation={result.validation_time:.2f}s",
f"catalog_id={result.catalog_id} time={result.elapsed_time:.2f}s "
f"validation={result.validation_time:.2f}s",
fg="white",
)
)
@@ -850,7 +880,7 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
click.echo(
click.style(
f" failed bundle={result.bundle_id} tenant={result.tenant_id} "
f"object_prefix={result.object_prefix} error={result.error}",
f"catalog_id={result.catalog_id} object_prefix={result.object_prefix} error={result.error}",
fg="red",
)
)
@@ -867,25 +897,24 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
)
@click.option("--run-id", required=False, help="Workflow run ID to restore.")
@click.option(
"--start-from",
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
"--target-month",
metavar="YYYY-MM",
default=None,
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
help="V2 catalog month to restore; required unless --run-id is used.",
)
@click.option(
"--end-before",
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
"--after-catalog-id",
default=None,
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
help="Exclusive V2 cursor from the same restore month and tenant scope.",
)
@click.option("--workers", default=1, show_default=True, type=int, help="V1 --run-id compatibility only.")
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to restore.")
@click.option("--limit", type=click.IntRange(min=1), default=100, show_default=True, help="Maximum V2 catalog rows.")
@click.option("--dry-run", is_flag=True, help="Preview without restoring.")
def restore_workflow_runs(
tenant_ids: str | None,
run_id: str | None,
start_from: datetime.datetime | None,
end_before: datetime.datetime | None,
target_month: str | None,
after_catalog_id: str | None,
workers: int,
limit: int,
dry_run: bool,
@@ -905,23 +934,20 @@ def restore_workflow_runs(
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
from services.retention.workflow_run.restore_archived_workflow_run import WorkflowRunRestore
parsed_tenant_ids = None
if tenant_ids:
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
if not parsed_tenant_ids:
raise click.BadParameter("tenant-ids must not be empty")
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
if (start_from is None) ^ (end_before is None):
raise click.UsageError("--start-from and --end-before must be provided together.")
if run_id is None and (start_from is None or end_before is None):
raise click.UsageError("--start-from and --end-before are required for batch restore.")
if workers < 1:
raise click.BadParameter("workers must be at least 1")
if run_id is not None and (target_month is not None or after_catalog_id is not None):
raise click.UsageError("--target-month and --after-catalog-id are only valid for V2 batch restore.")
if run_id is None and target_month is None:
raise click.UsageError("--target-month is required for V2 batch restore.")
start_time = datetime.datetime.now(datetime.UTC)
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
click.echo(
click.style(
f"Starting restore of workflow run {run_id} at {start_time.isoformat()}.",
f"Starting restore of {target_desc} at {start_time.isoformat()}.",
fg="white",
)
)
@@ -955,17 +981,20 @@ def restore_workflow_runs(
click.echo(
click.style("--workers is ignored for V2 bundle restore; bundles are processed serially.", fg="yellow")
)
assert start_from is not None
assert end_before is not None
assert target_month is not None
target_year, target_month_number = _parse_archive_target_month(target_month)
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
bundle_restorer = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
summary = bundle_restorer.restore_batch(
tenant_ids=parsed_tenant_ids,
start_date=start_from,
end_date=end_before,
target_year=target_year,
target_month=target_month_number,
after_catalog_id=catalog_cursor,
limit=limit,
)
_echo_bundle_archive_operation_summary(summary)
return
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
if summary.bundles_failed:
raise click.exceptions.Exit(1)
@click.command(
@@ -979,23 +1008,41 @@ def restore_workflow_runs(
)
@click.option("--run-id", required=False, help="Workflow run ID to delete.")
@click.option(
"--start-from",
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
"--target-month",
metavar="YYYY-MM",
default=None,
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
help="V2 catalog month to delete; required unless --run-id is used.",
)
@click.option(
"--end-before",
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
"--after-catalog-id",
default=None,
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
help="Exclusive V2 cursor from the same delete month and tenant scope.",
)
@click.option(
"--run-shard-index",
default=None,
type=click.IntRange(min=0),
help="Zero-based archive shard index. Must be paired with --run-shard-total.",
)
@click.option(
"--run-shard-total",
default=None,
type=click.IntRange(min=1, max=16),
help="Total archive shard count. Must be paired with --run-shard-index.",
)
@click.option("--all-pages", is_flag=True, help="Process catalog pages until an empty page is reached.")
@click.option(
"--limit",
type=click.IntRange(min=1),
default=100,
show_default=True,
help="Maximum V2 catalog rows per page.",
)
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to delete.")
@click.option("--dry-run", is_flag=True, help="Preview without deleting.")
@click.option(
"--skip-bad-archives",
is_flag=True,
help="Continue batch deletion when one archive object fails validation.",
help="V1 --run-id only: continue when one archive object fails validation.",
)
@click.option(
"--restore-sample-interval",
@@ -1007,8 +1054,11 @@ def restore_workflow_runs(
def delete_archived_workflow_runs(
tenant_ids: str | None,
run_id: str | None,
start_from: datetime.datetime | None,
end_before: datetime.datetime | None,
target_month: str | None,
after_catalog_id: str | None,
run_shard_index: int | None,
run_shard_total: int | None,
all_pages: bool,
limit: int,
dry_run: bool,
skip_bad_archives: bool,
@@ -1018,26 +1068,38 @@ def delete_archived_workflow_runs(
Delete archived workflow runs from the database.
Batch delete uses V2 bundle metadata and validates object existence, manifest schema, object size, checksum, row
counts, and source/archive content checksums before deleting source rows. `--run-id` keeps the V1 per-run path.
counts, and source/archive content checksums before deleting source rows. Parallel workers may select one exact
archive shard; all-pages mode keeps only the current bounded page in memory. `--run-id` keeps the V1 per-run path.
"""
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
from services.retention.workflow_run.delete_archived_workflow_run import ArchivedWorkflowRunDeletion
parsed_tenant_ids = None
if tenant_ids:
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
if not parsed_tenant_ids:
raise click.BadParameter("tenant-ids must not be empty")
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
if (start_from is None) ^ (end_before is None):
raise click.UsageError("--start-from and --end-before must be provided together.")
if run_id is None and (start_from is None or end_before is None):
raise click.UsageError("--start-from and --end-before are required for batch delete.")
if restore_sample_interval < 0:
raise click.BadParameter("restore-sample-interval must be >= 0")
if run_id is not None and (
target_month is not None
or after_catalog_id is not None
or run_shard_index is not None
or run_shard_total is not None
or all_pages
):
raise click.UsageError(
"--target-month, --after-catalog-id, --run-shard-index, --run-shard-total, and --all-pages "
"are only valid for V2 batch delete."
)
if run_id is None and target_month is None:
raise click.UsageError("--target-month is required for V2 batch delete.")
if run_id is None and skip_bad_archives:
raise click.UsageError("--skip-bad-archives is not supported for V2 catalog batches; they fail fast.")
if (run_shard_index is None) ^ (run_shard_total is None):
raise click.UsageError("--run-shard-index and --run-shard-total must be provided together.")
if run_shard_index is not None and run_shard_total is not None and run_shard_index >= run_shard_total:
raise click.UsageError("--run-shard-index must be less than --run-shard-total.")
start_time = datetime.datetime.now(datetime.UTC)
target_desc = f"workflow run {run_id}" if run_id else "workflow runs"
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
click.echo(
click.style(
f"Starting delete of {target_desc} at {start_time.isoformat()}.",
@@ -1110,20 +1172,104 @@ def delete_archived_workflow_runs(
if restore_sample_interval:
click.echo(click.style("--restore-sample-interval is ignored for V2 bundle delete.", fg="yellow"))
assert start_from is not None
assert end_before is not None
bundle_deleter = WorkflowRunBundleArchiveMaintenance(
dry_run=dry_run,
strict_content_validation=True,
stop_on_error=not skip_bad_archives,
assert target_month is not None
target_year, target_month_number = _parse_archive_target_month(target_month)
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
shard = (
f"{run_shard_index:02d}-of-{run_shard_total:02d}"
if run_shard_index is not None and run_shard_total is not None
else None
)
summary = bundle_deleter.delete_batch(
tenant_ids=parsed_tenant_ids,
start_date=start_from,
end_date=end_before,
limit=limit,
)
_echo_bundle_archive_operation_summary(summary)
bundle_deleter = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
if run_shard_total is not None:
try:
bundle_deleter.validate_catalog_shards(
target_year=target_year,
target_month=target_month_number,
shard_total=run_shard_total,
tenant_ids=parsed_tenant_ids,
)
except ValueError as exc:
logger.exception(
"Archive catalog shard preflight failed: target_month=%s shard=%s",
target_month,
shard,
)
raise click.ClickException(
f"Archive catalog shard preflight failed for target_month={target_month} shard={shard}: {exc}"
) from exc
initial_catalog_cursor = catalog_cursor
pages_processed = 0
bundles_succeeded = 0
runs_processed = 0
rows_processed = 0
archive_bytes = 0
while True:
summary = bundle_deleter.delete_batch(
tenant_ids=parsed_tenant_ids,
target_year=target_year,
target_month=target_month_number,
after_catalog_id=catalog_cursor,
limit=limit,
shard=shard,
)
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
if summary.bundles_failed:
failed_result = next((result for result in summary.results if not result.success), None)
failed_catalog_id = failed_result.catalog_id if failed_result is not None else "unknown"
page_resume_cursor = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
resume_cursor = page_resume_cursor or catalog_cursor
if dry_run:
cursor_details = (
f"preview_after_catalog_id={resume_cursor or 'none'} "
f"destructive_retry_after_catalog_id={initial_catalog_cursor or 'none'}"
)
else:
cursor_details = f"resume_after_catalog_id={resume_cursor or 'none'}"
click.echo(
click.style(
f"Delete stopped: target_month={target_month} shard={shard or 'all'} "
f"failed_catalog_id={failed_catalog_id} "
f"{cursor_details}",
fg="red",
)
)
raise click.exceptions.Exit(1)
if not all_pages:
break
if summary.bundles_processed == 0:
break
pages_processed += 1
bundles_succeeded += summary.bundles_succeeded
runs_processed += summary.runs_processed
rows_processed += summary.rows_processed
archive_bytes += summary.archive_bytes
next_catalog_id = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
if next_catalog_id is None or (catalog_cursor is not None and next_catalog_id <= catalog_cursor):
click.echo(
click.style(
f"Delete cursor did not advance: target_month={target_month} shard={shard or 'all'} "
f"after_catalog_id={catalog_cursor or 'none'} next_catalog_id={next_catalog_id or 'none'}",
fg="red",
)
)
raise click.exceptions.Exit(1)
catalog_cursor = next_catalog_id
if all_pages:
final_cursor_label = "preview_final_catalog_id" if dry_run else "final_catalog_id"
click.echo(
click.style(
f"Delete all-pages completed successfully. target_month={target_month} shard={shard or 'all'} "
f"pages={pages_processed} bundles_success={bundles_succeeded} runs={runs_processed} "
f"rows={rows_processed} archive_bytes={archive_bytes} "
f"{final_cursor_label}={catalog_cursor or 'none'}",
fg="green",
)
)
def _find_orphaned_draft_variables(batch_size: int = 1000) -> list[str]:
+2
View File
@@ -1,5 +1,6 @@
from configs.extra.agent_backend_config import AgentBackendConfig
from configs.extra.archive_config import ArchiveStorageConfig
from configs.extra.knowledge_fs_config import KnowledgeFSConfig
from configs.extra.notion_config import NotionConfig
from configs.extra.sentry_config import SentryConfig
@@ -8,6 +9,7 @@ class ExtraServiceConfig(
# place the configs in alphabet order
AgentBackendConfig,
ArchiveStorageConfig,
KnowledgeFSConfig,
NotionConfig,
SentryConfig,
):
+134
View File
@@ -0,0 +1,134 @@
"""Configuration for the optional KnowledgeFS control-plane integration."""
from ipaddress import ip_address
from urllib.parse import urlsplit
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
from pydantic_settings import BaseSettings
class KnowledgeFSConfig(BaseSettings):
"""Server-only KnowledgeFS connection and rollout settings."""
KNOWLEDGE_FS_ENABLED: bool = Field(
default=False,
description="Enable the private KnowledgeFS Console bridge.",
)
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
default=False,
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
)
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
default=False,
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
)
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
default=False,
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
)
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
KNOWLEDGE_FS_DIRECT_ORIGIN: str | None = Field(
default=None,
description="Public KnowledgeFS origin returned with direct upload capabilities.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
default=False,
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
)
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
default=None,
description="Identifier for the current asymmetric Capability v2 signing key.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
default=None,
description="Server-only PEM for the current Capability v2 RSA signing key.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
default=None,
description="Optional public-only JWKS JSON retained during key rotation overlap.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
@field_validator(
"KNOWLEDGE_FS_BASE_URL",
"KNOWLEDGE_FS_DIRECT_ORIGIN",
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
mode="before",
)
@classmethod
def normalize_optional_string(cls, value: object) -> object:
if isinstance(value, SecretStr):
normalized = value.get_secret_value().strip()
return SecretStr(normalized) if normalized else None
if isinstance(value, str):
normalized = value.strip()
return normalized or None
return value
@field_validator("KNOWLEDGE_FS_BASE_URL")
@classmethod
def validate_base_url(cls, value: str | None) -> str | None:
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
@field_validator("KNOWLEDGE_FS_DIRECT_ORIGIN")
@classmethod
def validate_direct_origin(cls, value: str | None) -> str | None:
return cls._validate_origin(value, name="KNOWLEDGE_FS_DIRECT_ORIGIN")
@classmethod
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
if value is None:
return None
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
try:
_ = parsed.port
except ValueError as exc:
raise ValueError(f"{name} must include a valid port") from exc
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
return value.rstrip("/")
@model_validator(mode="after")
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
if str(getattr(self, "DEPLOY_ENV", "")).strip().upper() == "PRODUCTION":
for name, value in (
("KNOWLEDGE_FS_BASE_URL", self.KNOWLEDGE_FS_BASE_URL),
("KNOWLEDGE_FS_DIRECT_ORIGIN", self.KNOWLEDGE_FS_DIRECT_ORIGIN),
):
if value and not self._is_secure_or_loopback_origin(value):
raise ValueError(f"{name} must use HTTPS in production unless it targets loopback")
if self.KNOWLEDGE_FS_ENABLED:
if not self.KNOWLEDGE_FS_BASE_URL:
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
):
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
return self
@staticmethod
def _is_secure_or_loopback_origin(value: str) -> bool:
parsed = urlsplit(value)
if parsed.scheme == "https":
return True
hostname = (parsed.hostname or "").rstrip(".").lower()
if hostname == "localhost":
return True
try:
return ip_address(hostname).is_loopback
except ValueError:
return False
+2
View File
@@ -125,6 +125,7 @@ from .explore import (
saved_message,
trial,
)
from .knowledge_fs import resources as knowledge_fs_resources
from .snippets import snippet_workflow, snippet_workflow_draft_variable
from .socketio import workflow as socketio_workflow
@@ -195,6 +196,7 @@ __all__ = [
"human_input_form",
"init_validate",
"installed_app",
"knowledge_fs_resources",
"load_balancing_config",
"login",
"mcp_server",
+2
View File
@@ -105,6 +105,7 @@ class SyncDraftWorkflowPayload(BaseModel):
graph: dict[str, Any]
features: dict[str, Any]
hash: str | None = None
is_collaborative: bool = Field(default=False, alias="_is_collaborative")
environment_variables: list[dict[str, Any]] = Field(
default_factory=list,
)
@@ -610,6 +611,7 @@ class DraftWorkflowApi(Resource):
environment_variables=environment_variables,
conversation_variables=conversation_variables,
session=db.session(),
graph_only=args["is_collaborative"],
)
except WorkflowHashNotEqualError:
raise DraftWorkflowNotSync()
+47 -2
View File
@@ -47,7 +47,9 @@ from controllers.console.explore.error import (
NotWorkflowAppError,
)
from controllers.console.explore.wraps import TrialAppResource, trial_feature_enable
from controllers.console.wraps import with_current_user
from controllers.console.files import FILE_UPLOAD_PARAMS, upload_file_from_request
from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file_from_request
from controllers.console.wraps import cloud_edition_billing_resource_check, with_current_user
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
from core.app.apps.base_app_queue_manager import AppQueueManager
@@ -61,12 +63,13 @@ from extensions.ext_database import db
from extensions.ext_redis import redis_client
from fields.base import ResponseModel
from fields.conversation_variable_fields import WorkflowConversationVariableResponse
from fields.file_fields import FileResponse, FileWithSignedUrl
from fields.message_fields import SuggestedQuestionsResponse
from graphon.graph_engine.manager import GraphEngineManager
from graphon.model_runtime.errors.invoke import InvokeError
from libs import helper
from libs.helper import dump_response, to_timestamp, uuid_value
from models import Account
from models import Account, App
from models.account import TenantStatus
from models.model import AppMode, Site, load_annotation_reply_config
from models.workflow import Workflow
@@ -428,6 +431,36 @@ register_response_schema_models(
simple_account_model = console_ns.models[TrialSimpleAccount.__name__]
class TrialAppFileUploadApi(TrialAppResource):
@trial_feature_enable
@cloud_edition_billing_resource_check("documents")
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
@with_current_user
def post(self, current_user: Account, app_model: App):
"""Upload a file into the tenant that owns the trial app."""
upload_file = upload_file_from_request(
current_user=current_user,
resource_tenant_id=app_model.tenant_id,
)
return dump_response(FileResponse, upload_file), 201
class TrialAppRemoteFileUploadApi(TrialAppResource):
@trial_feature_enable
@cloud_edition_billing_resource_check("documents")
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])
@with_current_user
def post(self, current_user: Account, app_model: App):
"""Upload a remote file into the tenant that owns the trial app."""
remote_file = upload_remote_file_from_request(
current_user=current_user,
resource_tenant_id=app_model.tenant_id,
)
return remote_file.model_dump(mode="json"), 201
class TrialAppWorkflowRunApi(TrialAppResource):
@trial_feature_enable
@console_ns.expect(console_ns.models[WorkflowRunRequest.__name__])
@@ -887,6 +920,18 @@ class DatasetListApi(Resource):
console_ns.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
console_ns.add_resource(
TrialAppFileUploadApi,
"/trial-apps/<uuid:app_id>/files/upload",
endpoint="trial_app_file_upload",
)
console_ns.add_resource(
TrialAppRemoteFileUploadApi,
"/trial-apps/<uuid:app_id>/remote-files/upload",
endpoint="trial_app_remote_file_upload",
)
console_ns.add_resource(
TrialMessageSuggestedQuestionApi,
"/trial-apps/<uuid:app_id>/messages/<uuid:message_id>/suggested-questions",
+41 -35
View File
@@ -29,7 +29,7 @@ from extensions.ext_database import db
from fields.file_fields import FileResponse, UploadConfig
from libs.helper import dump_response
from libs.login import login_required
from models import Account
from models import Account, UploadFile
from services.file_service import FileService
from . import console_ns
@@ -39,7 +39,7 @@ register_response_schema_models(console_ns, AllowedExtensionsResponse, TextConte
PREVIEW_WORDS_LIMIT = 3000
_FILE_UPLOAD_PARAMS = {
FILE_UPLOAD_PARAMS = {
"file": {
"description": "File to upload",
"in": "formData",
@@ -56,6 +56,43 @@ _FILE_UPLOAD_PARAMS = {
}
def upload_file_from_request(*, current_user: Account, resource_tenant_id: str | None = None) -> UploadFile:
"""Validate the multipart request and persist the file under the requested resource tenant."""
source_str = request.form.get("source")
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
if "file" not in request.files:
raise NoFileUploadedError()
if len(request.files) > 1:
raise TooManyFilesError()
file = request.files["file"]
if not file.filename:
raise FilenameNotExistsError
if source == "datasets" and not current_user.is_dataset_editor:
raise Forbidden()
if source not in ("datasets", None):
source = None
try:
return FileService(db.engine).upload_file(
filename=file.filename,
content=file.stream.read(),
mimetype=file.mimetype,
user=current_user,
tenant_id=resource_tenant_id,
source=source,
)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:
raise UnsupportedFileTypeError()
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
raise BlockedFileExtensionError(blocked_extension_error.description)
@console_ns.route("/files/upload")
class FileApi(Resource):
@setup_required
@@ -81,42 +118,11 @@ class FileApi(Resource):
@login_required
@account_initialization_required
@cloud_edition_billing_resource_check("documents")
@console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
@with_current_user
def post(self, current_user: Account):
source_str = request.form.get("source")
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
if "file" not in request.files:
raise NoFileUploadedError()
if len(request.files) > 1:
raise TooManyFilesError()
file = request.files["file"]
if not file.filename:
raise FilenameNotExistsError
if source == "datasets" and not current_user.is_dataset_editor:
raise Forbidden()
if source not in ("datasets", None):
source = None
try:
upload_file = FileService(db.engine).upload_file(
filename=file.filename,
content=file.stream.read(),
mimetype=file.mimetype,
user=current_user,
source=source,
)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:
raise UnsupportedFileTypeError()
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
raise BlockedFileExtensionError(blocked_extension_error.description)
upload_file = upload_file_from_request(current_user=current_user)
return dump_response(FileResponse, upload_file), 201
@@ -0,0 +1,5 @@
"""Typed Dify-owned KnowledgeFS Console product API."""
from . import resources
__all__ = ["resources"]
@@ -0,0 +1,56 @@
"""Stable, non-enumerating Console error contract for KnowledgeFS."""
from libs.exception import BaseHTTPException
class KnowledgeFSSpaceNotFoundHTTPError(BaseHTTPException):
error_code = "knowledge_fs_space_not_found"
description = "KnowledgeFS space was not found."
code = 404
class KnowledgeFSOperationUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_operation_unavailable"
description = "KnowledgeFS operation is not available."
code = 503
class KnowledgeFSUpstreamUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_upstream_unavailable"
description = "KnowledgeFS is unavailable."
code = 502
class KnowledgeFSInvalidRequestHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_request"
description = "KnowledgeFS request is invalid."
code = 400
class KnowledgeFSAccessDeniedHTTPError(BaseHTTPException):
error_code = "knowledge_fs_access_denied"
description = "KnowledgeFS operation is not allowed."
code = 403
class KnowledgeFSRateLimitHTTPError(BaseHTTPException):
error_code = "knowledge_fs_rate_limit_exceeded"
description = "KnowledgeFS operation rate limit exceeded."
code = 429
class KnowledgeFSQuotaExceededHTTPError(BaseHTTPException):
error_code = "knowledge_fs_quota_exceeded"
description = "KnowledgeFS operation quota exceeded."
code = 403
__all__ = [
"KnowledgeFSAccessDeniedHTTPError",
"KnowledgeFSInvalidRequestHTTPError",
"KnowledgeFSOperationUnavailableHTTPError",
"KnowledgeFSQuotaExceededHTTPError",
"KnowledgeFSRateLimitHTTPError",
"KnowledgeFSSpaceNotFoundHTTPError",
"KnowledgeFSUpstreamUnavailableHTTPError",
]
File diff suppressed because it is too large Load Diff
+57 -47
View File
@@ -46,6 +46,61 @@ class GetRemoteFileInfo(Resource):
).model_dump(mode="json")
def upload_remote_file_from_request(
*,
current_user: Account,
resource_tenant_id: str | None = None,
) -> FileWithSignedUrl:
"""Validate the JSON request, fetch its remote file, and persist it under the requested tenant."""
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
url = payload.url
# Try to fetch remote file metadata/content first
try:
resp = remote_fetcher.make_request("HEAD", url=url)
if resp.status_code != httpx.codes.OK:
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
if resp.status_code != httpx.codes.OK:
# Normalize into a user-friendly error message expected by tests
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
except httpx.RequestError as e:
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
file_info = helpers.guess_file_info_from_response(resp)
# Enforce file size limit with 400 (Bad Request) per tests' expectation
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
raise FileTooLargeError()
# Load content if needed
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
try:
upload_file = FileService(db.engine).upload_file(
filename=file_info.filename,
content=content,
mimetype=file_info.mimetype,
user=current_user,
tenant_id=resource_tenant_id,
source_url=url,
)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:
raise UnsupportedFileTypeError()
return FileWithSignedUrl(
id=upload_file.id,
name=upload_file.name,
size=upload_file.size,
extension=upload_file.extension,
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
mime_type=upload_file.mime_type,
created_by=upload_file.created_by,
created_at=int(upload_file.created_at.timestamp()),
)
@console_ns.route("/remote-files/upload")
class RemoteFileUpload(Resource):
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
@@ -53,53 +108,8 @@ class RemoteFileUpload(Resource):
@login_required
@with_current_user
def post(self, current_user: Account):
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
url = payload.url
# Try to fetch remote file metadata/content first
try:
resp = remote_fetcher.make_request("HEAD", url=url)
if resp.status_code != httpx.codes.OK:
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
if resp.status_code != httpx.codes.OK:
# Normalize into a user-friendly error message expected by tests
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
except httpx.RequestError as e:
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
file_info = helpers.guess_file_info_from_response(resp)
# Enforce file size limit with 400 (Bad Request) per tests' expectation
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
raise FileTooLargeError()
# Load content if needed
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
try:
upload_file = FileService(db.engine).upload_file(
filename=file_info.filename,
content=content,
mimetype=file_info.mimetype,
user=current_user,
source_url=url,
)
except services.errors.file.FileTooLargeError as file_too_large_error:
raise FileTooLargeError(file_too_large_error.description)
except services.errors.file.UnsupportedFileTypeError:
raise UnsupportedFileTypeError()
# Success: return created resource with 201 status
remote_file = upload_remote_file_from_request(current_user=current_user)
return (
FileWithSignedUrl(
id=upload_file.id,
name=upload_file.name,
size=upload_file.size,
extension=upload_file.extension,
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
mime_type=upload_file.mime_type,
created_by=upload_file.created_by,
created_at=int(upload_file.created_at.timestamp()),
).model_dump(mode="json"),
remote_file.model_dump(mode="json"),
201,
)
@@ -98,6 +98,7 @@ def handle_collaboration_event(sid, data):
6. workflow_update
7. comments_update
8. node_panel_presence
9. graph_view_state (session reports tab visibility; drives leader election)
"""
return collaboration_service.relay_collaboration_event(sid, data)
+14 -3
View File
@@ -346,7 +346,13 @@ class RBACRoleItemApi(Resource):
def put(self, role_id):
tenant_id, account_id = _current_ids()
request = _payload(_RoleUpsertRequest)
role = svc.RBACService.Roles.update(tenant_id, account_id, str(role_id), request.to_mutation())
role = svc.RBACService.KnowledgeFSRoleMutations.update_role(
tenant_id,
account_id,
str(role_id),
request.to_mutation(),
session=db.session(),
)
return _dump(role)
@login_required
@@ -356,7 +362,12 @@ class RBACRoleItemApi(Resource):
@console_ns.response(200, "Success", console_ns.models[svc.RBACRole.__name__])
def delete(self, role_id):
tenant_id, account_id = _current_ids()
svc.RBACService.Roles.delete(tenant_id, account_id, str(role_id))
svc.RBACService.KnowledgeFSRoleMutations.delete_role(
tenant_id,
account_id,
str(role_id),
session=db.session(),
)
return {"result": "success"}
@@ -915,7 +926,7 @@ class RBACMemberRolesApi(Resource):
tenant_id, account_id = _current_ids()
request = _payload(_ReplaceMemberRolesRequest)
return _dump(
svc.RBACService.MemberRoles.replace(
svc.RBACService.KnowledgeFSRoleMutations.replace_member_roles(
tenant_id,
account_id,
str(member_id),
@@ -8,6 +8,7 @@ from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
from controllers.inner_api.wraps import plugin_inner_api_only
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
from core.plugin.backwards_invocation.datasource import PluginDatasourceBackwardsInvocation
from core.plugin.backwards_invocation.encrypt import PluginEncrypter
from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
@@ -15,10 +16,12 @@ from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
from core.plugin.entities.request import (
RequestFetchAppInfo,
RequestInvokeApp,
RequestInvokeDatasource,
RequestInvokeEncrypt,
RequestInvokeLLM,
RequestInvokeLLMWithStructuredOutput,
RequestInvokeModeration,
RequestInvokeMultimodalEmbedding,
RequestInvokeParameterExtractorNode,
RequestInvokeQuestionClassifierNode,
RequestInvokeRerank,
@@ -27,6 +30,7 @@ from core.plugin.entities.request import (
RequestInvokeTextEmbedding,
RequestInvokeTool,
RequestInvokeTTS,
RequestListModels,
RequestRequestDownloadFile,
RequestRequestUploadFile,
)
@@ -118,6 +122,36 @@ class PluginInvokeTextEmbeddingApi(Resource):
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/multimodal-embedding")
class PluginInvokeMultimodalEmbeddingApi(Resource):
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestInvokeMultimodalEmbedding)
@inner_api_ns.doc("plugin_invoke_multimodal_embedding")
@inner_api_ns.doc(description="Invoke multimodal embedding models through Dify model management")
@inner_api_ns.doc(
responses={
200: "Multimodal embedding successful",
401: "Unauthorized - invalid API key",
404: "Service not available",
}
)
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeMultimodalEmbedding):
try:
return jsonable_encoder(
BaseBackwardsInvocationResponse(
data=PluginModelBackwardsInvocation.invoke_multimodal_embedding(
user_id=user_model.id,
tenant=tenant_model,
payload=payload,
)
)
)
except Exception as e:
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/rerank")
class PluginInvokeRerankApi(Resource):
@get_user_tenant
@@ -144,6 +178,70 @@ class PluginInvokeRerankApi(Resource):
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/model-catalog")
class PluginModelCatalogApi(Resource):
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestListModels)
@inner_api_ns.doc("plugin_model_catalog")
@inner_api_ns.doc(description="List tenant-active models managed by Dify")
@inner_api_ns.doc(
responses={
200: "Model catalog lookup successful",
401: "Unauthorized - invalid API key",
404: "Service not available",
}
)
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestListModels):
try:
return jsonable_encoder(
BaseBackwardsInvocationResponse(
data=PluginModelBackwardsInvocation.list_models(
tenant_id=tenant_model.id,
user_id=user_model.id,
payload=payload,
)
)
)
except Exception as e:
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/datasource")
class PluginInvokeDatasourceApi(Resource):
"""Invoke an installed datasource with credentials resolved inside Dify."""
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestInvokeDatasource)
@inner_api_ns.doc("plugin_invoke_datasource")
@inner_api_ns.doc(description="Invoke datasource plugins through Dify credential management")
@inner_api_ns.doc(
responses={
200: "Datasource invocation successful (streaming response)",
401: "Unauthorized - invalid API key",
404: "Datasource provider, datasource, or credential not found",
}
)
def post(
self,
user_model: Account | EndUser,
tenant_model: Tenant,
payload: RequestInvokeDatasource,
):
response = PluginDatasourceBackwardsInvocation.invoke(
user_id=user_model.id,
tenant=tenant_model,
payload=payload,
)
return length_prefixed_response(
0xF,
PluginDatasourceBackwardsInvocation.convert_to_event_stream(response),
)
@inner_api_ns.route("/invoke/tts")
class PluginInvokeTTSApi(Resource):
@get_user_tenant
+2
View File
@@ -38,6 +38,7 @@ from .dataset import (
)
from .dataset.rag_pipeline import rag_pipeline_workflow
from .end_user import end_user
from .knowledge_fs import resources as knowledge_fs_resources
from .workspace import models
__all__ = [
@@ -54,6 +55,7 @@ __all__ = [
"hit_testing",
"human_input_form",
"index",
"knowledge_fs_resources",
"message",
"metadata",
"models",
@@ -0,0 +1,5 @@
"""KnowledgeFS-specific Service API authenticated by resource credentials."""
from . import resources
__all__ = ["resources"]
@@ -0,0 +1,49 @@
"""Stable KnowledgeFS Service API error contract."""
from libs.exception import BaseHTTPException
class KnowledgeFSInvalidCredentialHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_credential"
description = "Invalid KnowledgeFS service credential."
code = 401
class KnowledgeFSServiceOperationUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_operation_unavailable"
description = "KnowledgeFS operation is not available."
code = 503
class KnowledgeFSServiceUpstreamUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_upstream_unavailable"
description = "KnowledgeFS is unavailable."
code = 502
class KnowledgeFSServiceInvalidRequestHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_request"
description = "KnowledgeFS request is invalid."
code = 400
class KnowledgeFSServiceRateLimitHTTPError(BaseHTTPException):
error_code = "knowledge_fs_rate_limit_exceeded"
description = "KnowledgeFS operation rate limit exceeded."
code = 429
class KnowledgeFSServiceQuotaExceededHTTPError(BaseHTTPException):
error_code = "knowledge_fs_quota_exceeded"
description = "KnowledgeFS operation quota exceeded."
code = 403
__all__ = [
"KnowledgeFSInvalidCredentialHTTPError",
"KnowledgeFSServiceInvalidRequestHTTPError",
"KnowledgeFSServiceOperationUnavailableHTTPError",
"KnowledgeFSServiceQuotaExceededHTTPError",
"KnowledgeFSServiceRateLimitHTTPError",
"KnowledgeFSServiceUpstreamUnavailableHTTPError",
]
File diff suppressed because it is too large Load Diff
+12
View File
@@ -14,7 +14,10 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
from core.app.apps.base_app_runner import AppRunner
from core.app.entities.app_invoke_entities import (
AgentChatAppGenerateEntity,
DifyRunContext,
InvokeFrom,
ModelConfigWithCredentialsEntity,
UserFrom,
)
from core.app.file_access import DatabaseFileAccessController
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
@@ -148,6 +151,15 @@ class BaseAgentRunner(AppRunner):
user_id=self.user_id,
invoke_from=self.application_generate_entity.invoke_from,
)
invoke_from = self.application_generate_entity.invoke_from
if isinstance(invoke_from, InvokeFrom):
tool_entity.runtime.dify_run_context = DifyRunContext(
tenant_id=self.tenant_id,
app_id=self.app_config.app_id,
user_id=self.user_id,
user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER),
invoke_from=invoke_from,
)
assert tool_entity.entity.description
message_tool = PromptMessageTool(
name=tool.tool_name,
@@ -159,6 +159,7 @@ class AgentAppRuntimeRequestBuilder:
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
agent_mode="agent_app",
trace_id=context.dify_context.trace_session_id,
),
# ENG-616: expand slash-menu mention tokens to canonical names so
# no frontend-internal {{#…#}} marker ever reaches the model.
@@ -38,6 +38,11 @@ class DatasourcePluginProviderController(ABC):
):
raise ToolProviderCredentialValidationError("Invalid credentials")
def validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
"""Validate credential shape and value against this installed provider declaration."""
self.validate_credentials_format(credentials)
self._validate_credentials(user_id, credentials)
@property
def provider_type(self) -> DatasourceProviderType:
"""
+91 -2
View File
@@ -47,6 +47,24 @@ class MaxRetriesExceededError(ValueError):
pass
class ResponseLimitError(ValueError):
"""Base error for responses that cannot be safely bounded."""
pass
class ResponseTooLargeError(ResponseLimitError):
"""Raised when an identity response exceeds the configured byte limit."""
pass
class UnsupportedResponseEncodingError(ResponseLimitError):
"""Raised when response encoding prevents safe decoded-size enforcement."""
pass
request_error = httpx.RequestError
max_retries_exceeded_error = MaxRetriesExceededError
@@ -142,7 +160,31 @@ def _inject_trace_headers(headers: Headers | None) -> Headers:
return headers
def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
def make_request(
method: str,
url: str,
max_retries: int = SSRF_DEFAULT_MAX_RETRIES,
stream_response: bool = False,
**kwargs: Any,
) -> httpx.Response:
"""Send one SSRF-protected request with optional streaming.
Args:
method: HTTP method sent through the configured SSRF client.
url: Absolute request URL.
max_retries: Number of retry attempts after the initial request.
stream_response: Return an open streaming response that the caller must close.
**kwargs: Additional keyword arguments forwarded to ``httpx.Client``.
Returns:
A buffered response, or an open response when ``stream_response`` is true.
Raises:
ToolSSRFError: The configured SSRF proxy rejects the destination.
MaxRetriesExceededError: All configured request attempts fail.
httpx.RequestError: A request fails while retries are disabled.
ValueError: The SSL verification option or request headers are invalid.
"""
# Convert requests-style allow_redirects to httpx-style follow_redirects
if "allow_redirects" in kwargs:
allow_redirects = kwargs.pop("allow_redirects")
@@ -175,6 +217,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
# When using a forward proxy, httpx may override the Host header based on the URL.
# We extract and preserve any explicitly set Host header to support virtual hosting.
user_provided_host = _get_user_provided_host_header(headers)
send_kwargs: dict[str, Any] = {}
if "auth" in kwargs:
send_kwargs["auth"] = kwargs.pop("auth")
if "follow_redirects" in kwargs:
send_kwargs["follow_redirects"] = kwargs.pop("follow_redirects")
retries = 0
while retries <= max_retries:
@@ -185,7 +232,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
if user_provided_host is not None:
headers["host"] = user_provided_host
kwargs["headers"] = headers
response = client.request(method=method, url=url, **kwargs)
request = client.build_request(method=method, url=url, **kwargs)
if stream_response:
response = client.send(request, stream=True, **send_kwargs)
else:
response = client.send(request, **send_kwargs)
# Check for SSRF protection by Squid proxy
if response.status_code in (401, 403):
@@ -195,6 +246,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
# Squid typically identifies itself in Server or Via headers
if "squid" in server_header or "squid" in via_header:
response.close()
raise ToolSSRFError(
f"Access to '{url}' was blocked by SSRF protection. "
f"The URL may point to a private or local network address. "
@@ -208,6 +260,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
response.status_code,
url,
)
response.close()
except httpx.RequestError as e:
logger.warning("Request to URL %s failed on attempt %s: %s", url, retries + 1, e)
@@ -220,6 +273,42 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
raise MaxRetriesExceededError(f"Reached maximum retries ({max_retries}) for URL {url}")
def buffer_response(response: httpx.Response, *, max_response_bytes: int) -> httpx.Response:
"""Consume one open identity response under a decoded byte limit and close its stream."""
if max_response_bytes <= 0:
raise ValueError("max_response_bytes must be positive")
try:
content_encoding = response.headers.get("content-encoding", "identity").strip().lower()
if content_encoding not in {"", "identity"}:
raise UnsupportedResponseEncodingError(f"content encoding {content_encoding} cannot be safely bounded")
content = bytearray()
for chunk in response.iter_bytes():
if len(content) + len(chunk) > max_response_bytes:
raise ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
content.extend(chunk)
decoded_headers = {
name: value
for name, value in response.headers.items()
if name.lower() not in {"content-encoding", "content-length", "transfer-encoding"}
}
try:
request = response.request
except RuntimeError:
request = None
return httpx.Response(
response.status_code,
headers=decoded_headers,
content=bytes(content),
request=request,
extensions=response.extensions,
history=response.history,
default_encoding=response.default_encoding,
)
finally:
response.close()
def get(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
return make_request("GET", url, max_retries=max_retries, **kwargs)
@@ -0,0 +1,110 @@
"""Datasource backward invocation through Dify's tenant-bound runtime.
This module is the internal service boundary used by trusted callers such as
KnowledgeFS. It resolves installed provider declarations and Dify-owned
credential references before reaching ``PluginDatasourceManager``; callers
must never provide raw datasource credentials or a plugin-daemon API key.
"""
from collections.abc import Generator
from typing import Any, cast
from pydantic import BaseModel
from core.datasource.datasource_manager import DatasourceManager
from core.datasource.entities.datasource_entities import (
OnlineDriveBrowseFilesRequest,
OnlineDriveDownloadFileRequest,
)
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
from core.plugin.entities.request import RequestInvokeDatasource
from models.account import Tenant
from models.provider_ids import DatasourceProviderID
from services.datasource_provider_service import DatasourceProviderService
class PluginDatasourceBackwardsInvocation(BaseBackwardsInvocation):
"""Resolve and invoke a datasource without exposing credential material to the caller."""
@classmethod
def invoke(
cls,
*,
user_id: str,
tenant: Tenant,
payload: RequestInvokeDatasource,
) -> Generator[BaseModel | dict[str, Any], None, None]:
"""Yield datasource messages for one validated inner-runtime request."""
provider_id = DatasourceProviderID(payload.provider)
canonical_provider_id = str(provider_id)
controller = DatasourceManager.get_datasource_plugin_provider(
provider_id=canonical_provider_id,
tenant_id=tenant.id,
datasource_type=payload.datasource_type,
)
if controller.entity.provider_type != payload.datasource_type:
raise ValueError("Datasource provider type mismatch")
# Resolving the datasource from the installed declaration prevents a caller
# from dispatching an arbitrary datasource name under a valid plugin ID.
runtime = controller.get_datasource(payload.datasource)
credentials = DatasourceProviderService().get_datasource_credentials(
tenant_id=tenant.id,
provider=provider_id.provider_name,
plugin_id=provider_id.plugin_id,
credential_id=payload.credential_id,
)
if controller.need_credentials and not credentials:
raise ValueError("Datasource credential not found")
if payload.operation == "validate_credentials":
controller.validate_credentials(user_id=user_id, credentials=credentials)
yield {"result": True}
return
runtime.runtime.credentials = credentials
provider_type = runtime.datasource_provider_type()
match payload.operation:
case "get_website_crawl":
website = cast(WebsiteCrawlDatasourcePlugin, runtime)
yield from website.get_website_crawl(
user_id=user_id,
datasource_parameters=payload.datasource_parameters,
provider_type=provider_type,
)
case "get_online_document_pages":
document = cast(OnlineDocumentDatasourcePlugin, runtime)
yield from document.get_online_document_pages(
user_id=user_id,
datasource_parameters=payload.datasource_parameters,
provider_type=provider_type,
)
case "get_online_document_page_content":
if payload.page is None:
raise ValueError("Online-document page input is required")
document = cast(OnlineDocumentDatasourcePlugin, runtime)
yield from document.get_online_document_page_content(
user_id=user_id,
datasource_parameters=payload.page,
provider_type=provider_type,
)
case "online_drive_browse_files":
drive = cast(OnlineDriveDatasourcePlugin, runtime)
yield from drive.online_drive_browse_files(
user_id=user_id,
request=OnlineDriveBrowseFilesRequest.model_validate(payload.request),
provider_type=provider_type,
)
case "online_drive_download_file":
drive = cast(OnlineDriveDatasourcePlugin, runtime)
yield from drive.online_drive_download_file(
user_id=user_id,
request=OnlineDriveDownloadFileRequest.model_validate(payload.request),
provider_type=provider_type,
)
case _:
raise ValueError(f"Unsupported datasource operation: {payload.operation}")
+109 -2
View File
@@ -1,22 +1,31 @@
import tempfile
from binascii import hexlify, unhexlify
from collections.abc import Generator
from collections.abc import Generator, Mapping
from enum import Enum
from typing import Any
from pydantic import BaseModel
from core.app.llm import deduct_llm_quota
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
from core.model_manager import ModelManager
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
from core.plugin.entities.request import (
InvokableModelCatalogItem,
InvokableModelCatalogPage,
RequestInvokeLLM,
RequestInvokeLLMWithStructuredOutput,
RequestInvokeModeration,
RequestInvokeMultimodalEmbedding,
RequestInvokeRerank,
RequestInvokeSpeech2Text,
RequestInvokeSummary,
RequestInvokeTextEmbedding,
RequestInvokeTTS,
RequestListModels,
)
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from core.plugin.plugin_service import PluginService
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.utils.model_invocation_utils import ModelInvocationUtils
from graphon.model_runtime.entities.llm_entities import (
@@ -33,6 +42,20 @@ from graphon.model_runtime.entities.message_entities import (
)
from graphon.model_runtime.entities.model_entities import ModelType
from models.account import Tenant
from models.provider_ids import ModelProviderID
def _json_compatible(value: Any) -> Any:
"""Convert model-runtime metadata into stable JSON-compatible values."""
if isinstance(value, BaseModel):
return value.model_dump(mode="json")
if isinstance(value, Enum):
return value.value
if isinstance(value, Mapping):
return {str(_json_compatible(key)): _json_compatible(child) for key, child in value.items()}
if isinstance(value, list | tuple | set):
return [_json_compatible(child) for child in value]
return value
class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
@@ -183,7 +206,30 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
)
# invoke model
response = model_instance.invoke_text_embedding(texts=payload.texts)
response = model_instance.invoke_text_embedding(texts=payload.texts, input_type=payload.input_type)
return response
@classmethod
def invoke_multimodal_embedding(
cls,
user_id: str,
tenant: Tenant,
payload: RequestInvokeMultimodalEmbedding,
):
"""Invoke multimodal embedding through the tenant-bound model instance."""
model_instance = cls._get_bound_model_instance(
tenant_id=tenant.id,
user_id=user_id,
provider=payload.provider,
model_type=payload.model_type,
model=payload.model,
)
response = model_instance.invoke_multimodal_embedding(
multimodel_documents=[document.model_dump(exclude_none=True) for document in payload.documents],
input_type=payload.input_type,
)
return response
@@ -210,6 +256,67 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
return response
@classmethod
def list_models(
cls,
tenant_id: str,
user_id: str,
payload: RequestListModels,
) -> InvokableModelCatalogPage:
"""List only models that are active for the tenant's Dify configuration."""
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id, user_id=user_id)
active_models = provider_manager.get_configurations(tenant_id).get_models(
model_type=payload.model_type,
only_active=True,
)
installed_identities: dict[str, str] = {}
for plugin in PluginService.list(tenant_id):
existing = installed_identities.get(plugin.plugin_id)
if existing is not None and existing != plugin.plugin_unique_identifier:
raise ValueError(f"Ambiguous installed identity for model plugin {plugin.plugin_id}")
installed_identities[plugin.plugin_id] = plugin.plugin_unique_identifier
requested_provider = str(ModelProviderID(payload.provider)) if payload.provider else None
matched_models = [
model
for model in active_models
if (requested_provider is None or model.provider.provider == requested_provider)
and (payload.model is None or model.model == payload.model)
]
matched_models.sort(key=lambda model: (model.provider.provider, model.model))
page_models = matched_models[payload.offset : payload.offset + payload.limit]
items: list[InvokableModelCatalogItem] = []
for model in page_models:
provider_id = ModelProviderID(model.provider.provider)
unique_identifier = installed_identities.get(provider_id.plugin_id)
if unique_identifier is None:
raise ValueError(f"Installed identity not found for active model plugin {provider_id.plugin_id}")
items.append(
InvokableModelCatalogItem(
plugin_id=provider_id.plugin_id,
plugin_unique_identifier=unique_identifier,
provider=provider_id.provider_name,
model=model.model,
model_type=model.model_type,
capabilities={
"deprecated": model.deprecated,
"features": _json_compatible(model.features or []),
"fetchFrom": _json_compatible(model.fetch_from),
"modelProperties": _json_compatible(model.model_properties),
"modelType": model.model_type.value,
"status": _json_compatible(model.status),
},
)
)
next_offset = payload.offset + len(page_models)
return InvokableModelCatalogPage(
items=items,
next_offset=next_offset if next_offset < len(matched_models) else None,
)
@classmethod
def invoke_tts(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTTS):
"""
+113 -2
View File
@@ -6,6 +6,11 @@ from typing import Any, Literal
from flask import Response
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from core.datasource.entities.datasource_entities import (
DatasourceProviderType,
GetOnlineDocumentPageContentRequest,
)
from core.entities.embedding_type import EmbeddingInputType
from core.entities.provider_entities import BasicProviderConfig
from core.plugin.utils.http_parser import deserialize_response
from core.workflow.file_reference import is_canonical_file_reference
@@ -54,6 +59,61 @@ class RequestInvokeTool(BaseModel):
credential_id: str | None = None
DatasourceInvocationOperation = Literal[
"get_online_document_page_content",
"get_online_document_pages",
"get_website_crawl",
"online_drive_browse_files",
"online_drive_download_file",
"validate_credentials",
]
class RequestInvokeDatasource(BaseModel):
"""Invoke one installed datasource using a Dify-owned credential reference.
Raw credentials are intentionally not part of this contract. ``tenant_id`` and
``user_id`` are consumed by the inner-API request context, while the remaining
fields select an installed provider declaration and an operation-specific input.
"""
tenant_id: str = Field(min_length=1, max_length=512)
user_id: str = Field(min_length=1, max_length=512)
provider: str = Field(min_length=1, max_length=768)
datasource: str = Field(min_length=1, max_length=256)
datasource_type: DatasourceProviderType
credential_id: str = Field(min_length=1, max_length=512)
operation: DatasourceInvocationOperation
datasource_parameters: dict[str, Any] = Field(default_factory=dict)
page: GetOnlineDocumentPageContentRequest | None = None
request: dict[str, Any] | None = None
model_config = ConfigDict(extra="forbid")
@model_validator(mode="after")
def validate_operation_payload(self) -> "RequestInvokeDatasource":
expected_type = {
"get_online_document_page_content": DatasourceProviderType.ONLINE_DOCUMENT,
"get_online_document_pages": DatasourceProviderType.ONLINE_DOCUMENT,
"get_website_crawl": DatasourceProviderType.WEBSITE_CRAWL,
"online_drive_browse_files": DatasourceProviderType.ONLINE_DRIVE,
"online_drive_download_file": DatasourceProviderType.ONLINE_DRIVE,
"validate_credentials": self.datasource_type,
}[self.operation]
if self.datasource_type != expected_type:
raise ValueError(f"{self.operation} requires datasource_type {expected_type.value}")
page_required = self.operation == "get_online_document_page_content"
if page_required != (self.page is not None):
raise ValueError("page is required only for get_online_document_page_content")
request_required = self.operation in {"online_drive_browse_files", "online_drive_download_file"}
if request_required != (self.request is not None):
raise ValueError("request is required only for online-drive operations")
return self
class BaseRequestInvokeModel(BaseModel):
provider: str
model: str
@@ -115,6 +175,25 @@ class RequestInvokeTextEmbedding(BaseRequestInvokeModel):
model_type: ModelType = ModelType.TEXT_EMBEDDING
texts: list[str]
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
class MultimodalEmbeddingDocument(BaseModel):
"""A document accepted by a multimodal text-embedding model."""
content: str
content_type: str
file_id: str | None = None
model_config = ConfigDict(extra="forbid")
class RequestInvokeMultimodalEmbedding(BaseRequestInvokeModel):
"""Request to invoke a multimodal text-embedding model."""
model_type: ModelType = ModelType.TEXT_EMBEDDING
documents: list[MultimodalEmbeddingDocument] = Field(min_length=1)
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
class RequestInvokeRerank(BaseRequestInvokeModel):
@@ -125,8 +204,40 @@ class RequestInvokeRerank(BaseRequestInvokeModel):
model_type: ModelType = ModelType.RERANK
query: str
docs: list[str]
score_threshold: float
top_n: int
score_threshold: float | None = None
top_n: int | None = None
class RequestListModels(BaseModel):
"""Tenant-scoped query for models that Dify can invoke."""
model_type: Literal[ModelType.LLM, ModelType.TEXT_EMBEDDING, ModelType.RERANK]
provider: str | None = None
model: str | None = None
offset: int = Field(default=0, ge=0)
limit: int = Field(default=50, ge=1, le=100)
model_config = ConfigDict(protected_namespaces=())
class InvokableModelCatalogItem(BaseModel):
"""Installed identity and active Dify capability metadata for one model."""
plugin_id: str
plugin_unique_identifier: str
provider: str
model: str
model_type: ModelType
capabilities: dict[str, Any] = Field(default_factory=dict)
model_config = ConfigDict(protected_namespaces=())
class InvokableModelCatalogPage(BaseModel):
"""Offset page returned by the internal model catalog endpoint."""
items: list[InvokableModelCatalogItem] = Field(default_factory=list)
next_offset: int | None = None
class RequestInvokeTTS(BaseRequestInvokeModel):
+10
View File
@@ -10,6 +10,7 @@ class RBACResourceScope(StrEnum):
APP = "app"
DATASET = "dataset"
KNOWLEDGE_FS = "knowledge_space"
WORKSPACE = "workspace"
@@ -57,6 +58,15 @@ class RBACPermission(StrEnum):
DATASET_EXTERNAL_CONNECT = "dataset_external_connect"
DATASET_IMPORT_EXPORT_DSL = "dataset_import_export_dsl"
KNOWLEDGE_FS_READ = "knowledge_space_read"
KNOWLEDGE_FS_CREATE = "knowledge_space_create"
KNOWLEDGE_FS_EDIT = "knowledge_space_edit"
KNOWLEDGE_FS_DELETE = "knowledge_space_delete"
KNOWLEDGE_FS_ACCESS_CONFIG = "knowledge_space_access_config"
KNOWLEDGE_FS_API_KEY_MANAGE = "knowledge_space_api_key_manage"
KNOWLEDGE_FS_DOCUMENT_WRITE = "knowledge_space_document_write"
KNOWLEDGE_FS_QUERY = "knowledge_space_query"
WORKSPACE_MEMBER_MANAGE = "workspace_member_manage"
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
API_EXTENSION_MANAGE = "api_extension_manage"
+2 -1
View File
@@ -2,7 +2,7 @@ from typing import Any
from pydantic import BaseModel, Field
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
from core.plugin.entities.plugin_daemon import CredentialType
from core.tools.entities.tool_entities import ToolInvokeFrom
@@ -20,6 +20,7 @@ class ToolRuntime(BaseModel):
tool_id: str | None = None
invoke_from: InvokeFrom | None = None
tool_invoke_from: ToolInvokeFrom | None = None
dify_run_context: DifyRunContext | None = Field(default=None, exclude=True, repr=False)
credentials: dict[str, Any] = Field(default_factory=dict)
credential_type: CredentialType = Field(default=CredentialType.API_KEY)
runtime_parameters: dict[str, Any] = Field(default_factory=dict)
@@ -1,4 +1,5 @@
- audio
- code
- knowledge_fs
- time
- webscraper
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="none">
<rect width="64" height="64" rx="14" fill="#155EEF"/>
<path d="M17 17h20c5.5 0 10 4.5 10 10v20H27c-5.5 0-10-4.5-10-10V17Z" fill="white" fill-opacity=".96"/>
<path d="M26 27h12M26 34h12M26 41h7" stroke="#155EEF" stroke-width="4" stroke-linecap="round"/>
</svg>

After

Width:  |  Height:  |  Size: 340 B

@@ -0,0 +1,9 @@
from typing import Any, override
from core.tools.builtin_tool.provider import BuiltinToolProviderController
class KnowledgeFSProvider(BuiltinToolProviderController):
@override
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
_ = (user_id, credentials)
@@ -0,0 +1,13 @@
identity:
author: Dify
name: knowledge_fs
label:
en_US: KnowledgeFS
zh_Hans: KnowledgeFS
description:
en_US: Run explicitly bound KnowledgeFS operations from an Agent or Workflow.
zh_Hans: 从 Agent 或 Workflow 执行显式绑定的 KnowledgeFS 操作。
icon: icon.svg
tags:
- rag
@@ -0,0 +1,60 @@
from collections.abc import Generator
from typing import Any, override
from pydantic import ValidationError
from sqlalchemy.orm import Session, sessionmaker
from core.tools.builtin_tool.tool import BuiltinTool
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage
from core.tools.errors import ToolInvokeError
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
from services.knowledge_fs.app_execution_capability import KnowledgeResourceRef
from services.knowledge_fs.product_dto import KnowledgeFSResearchTaskCreatePayload
from services.knowledge_fs.runtime import create_knowledge_fs_runtime
class KnowledgeFSCreateResearchTaskTool(BuiltinTool):
@override
def _invoke(
self,
session: Session,
user_id: str,
tool_parameters: dict[str, Any],
conversation_id: str | None = None,
app_id: str | None = None,
message_id: str | None = None,
) -> Generator[ToolInvokeMessage, None, None]:
_ = (user_id, conversation_id, app_id, message_id)
run_context = self.runtime.dify_run_context
if run_context is None or self.runtime.tenant_id != run_context.tenant_id:
raise ToolInvokeError("KnowledgeFS requires a trusted Dify run context")
match self.runtime.tool_invoke_from:
case ToolInvokeFrom.AGENT:
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
case ToolInvokeFrom.WORKFLOW:
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
case _:
raise ToolInvokeError("KnowledgeFS is only available to Agent and Workflow callers")
try:
resource = KnowledgeResourceRef.model_validate(tool_parameters.get("resource"))
payload_data: dict[str, object] = {
"query": tool_parameters.get("query"),
}
mode = tool_parameters.get("mode")
if mode:
payload_data["mode"] = mode
payload = KnowledgeFSResearchTaskCreatePayload.model_validate(payload_data)
runtime = create_knowledge_fs_runtime(sessionmaker(bind=session.get_bind(), expire_on_commit=False))
response = runtime.app_capabilities.create_research_task(
run_context=run_context,
caller_kind=caller_kind,
resource=resource,
payload=payload,
)
except ToolInvokeError:
raise
except (ValidationError, RuntimeError, ValueError) as exc:
raise ToolInvokeError(str(exc)) from exc
yield self.create_json_message(response.model_dump(mode="json", by_alias=True))
@@ -0,0 +1,75 @@
identity:
name: create_research_task
author: Dify
label:
en_US: Create KnowledgeFS Research Task
zh_Hans: 创建 KnowledgeFS Research 任务
description:
human:
en_US: Create a Research task in an explicitly bound KnowledgeFS space.
zh_Hans: 在显式绑定的 KnowledgeFS 空间中创建 Research 任务。
llm: Create a durable research task using an explicitly configured KnowledgeFS resource.
parameters:
- name: resource
type: object
required: true
label:
en_US: KnowledgeFS resource
zh_Hans: KnowledgeFS 资源
human_description:
en_US: A typed KnowledgeFS control-space reference configured by the app author.
zh_Hans: 由应用作者配置的类型化 KnowledgeFS control-space 引用。
form: form
input_schema:
type: object
additionalProperties: false
properties:
kind:
type: string
const: knowledge_fs
control_space_id:
type: string
minLength: 1
required:
- kind
- control_space_id
- name: query
type: string
required: true
label:
en_US: Research query
zh_Hans: Research 查询
human_description:
en_US: The question the Research task should investigate.
zh_Hans: Research 任务需要调查的问题。
llm_description: The question to investigate with KnowledgeFS.
form: llm
- name: mode
type: select
required: false
label:
en_US: Mode
zh_Hans: 模式
human_description:
en_US: Optional KnowledgeFS retrieval mode.
zh_Hans: 可选的 KnowledgeFS 检索模式。
llm_description: Optional retrieval mode. Use auto unless the task needs a specific mode.
form: llm
options:
- value: auto
label:
en_US: Auto
zh_Hans: 自动
- value: fast
label:
en_US: Fast
zh_Hans: 快速
- value: deep
label:
en_US: Deep
zh_Hans: 深度
- value: research
label:
en_US: Research
zh_Hans: Research
+1
View File
@@ -493,6 +493,7 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):
self._run_context.invoke_from,
variable_pool,
)
tool_runtime.runtime.dify_run_context = self._run_context
except ToolNodeError:
raise
except Exception as exc:
@@ -250,6 +250,7 @@ class WorkflowAgentRuntimeRequestBuilder:
agent_config_version_kind="snapshot",
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
trace_id=context.dify_context.trace_session_id,
),
agent_soul_prompt=soul_prompt or None,
workflow_node_job_prompt=workflow_job_prompt,
+346
View File
@@ -0,0 +1,346 @@
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
The primary threshold aggregates statement and branch coverage for every Dify
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
that only contain narrow integration hooks are checked with changed-line
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import re
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import TypedDict, cast
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
{
"api/dev/check_knowledge_fs_coverage.py",
"api/dev/generate_knowledge_fs_contract.py",
"api/dev/knowledge_fs_product_contract.py",
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
}
)
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
logger = logging.getLogger(__name__)
class CoverageSummary(TypedDict):
"""Coverage.py counts required by the aggregate gate."""
covered_lines: int
num_statements: int
covered_branches: int
num_branches: int
class CoverageFile(TypedDict):
"""Per-file coverage data emitted by ``coverage json``."""
executed_lines: list[int]
missing_lines: list[int]
summary: CoverageSummary
class CoverageReport(TypedDict):
"""Relevant top-level shape of a coverage.py JSON report."""
files: dict[str, CoverageFile]
@dataclass(frozen=True, slots=True)
class CoverageTotals:
"""Covered and measurable units for one gate surface."""
covered: int
total: int
@property
def percent(self) -> float:
return 100.0 if self.total == 0 else self.covered * 100 / self.total
class CoverageGateError(RuntimeError):
"""Raised when coverage input is incomplete or below its threshold."""
def main() -> None:
"""Validate focused module coverage and changed integration glue."""
logging.basicConfig(level=logging.INFO, format="%(message)s")
parser = argparse.ArgumentParser()
parser.add_argument("--coverage-json", type=Path, required=True)
parser.add_argument("--glue-manifest", type=Path, required=True)
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
parser.add_argument("--base", default="")
parser.add_argument("--minimum", type=float, default=90.0)
parser.add_argument("--glue-minimum", type=float, default=90.0)
args = parser.parse_args()
workspace_root = args.workspace_root.resolve()
report = load_coverage_report(args.coverage_json)
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
base = resolve_diff_base(workspace_root, args.base)
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
glue_totals = validate_changed_glue_coverage(
report,
changed_lines=changed_lines,
minimum=args.glue_minimum,
)
logger.info(
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
core_totals.percent,
core_totals.covered,
core_totals.total,
glue_totals.percent,
glue_totals.covered,
glue_totals.total,
)
def load_coverage_report(path: Path) -> CoverageReport:
"""Load the detailed JSON report used by both coverage checks."""
if not path.is_file():
raise CoverageGateError(f"coverage JSON does not exist: {path}")
document = json.loads(path.read_text())
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
raise CoverageGateError(f"coverage JSON has no files object: {path}")
return cast(CoverageReport, document)
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
if not path.is_file():
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
try:
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
except UnicodeDecodeError as error:
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
if not paths:
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
if len(paths) != len(set(paths)):
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
invalid_paths = [
candidate
for candidate in paths
if not candidate.startswith("api/")
or not candidate.endswith(".py")
or not (workspace_root / candidate).is_file()
]
if invalid_paths:
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
return paths
def is_core_coverage_path(path: str) -> bool:
"""Return whether a repository-relative path belongs to the focused aggregate."""
if not path.endswith(".py"):
return False
if path in {
"api/commands/knowledge_fs.py",
"api/configs/extra/knowledge_fs_config.py",
"api/extensions/ext_knowledge_fs_observability.py",
"api/services/knowledge_fs_capability.py",
}:
return True
if path.startswith(
(
"api/controllers/console/knowledge_fs/",
"api/controllers/service_api/knowledge_fs/",
"api/core/tools/builtin_tool/providers/knowledge_fs/",
"api/services/knowledge_fs/",
)
):
return True
filename = path.rsplit("/", maxsplit=1)[-1]
return (
path.startswith("api/models/")
and filename.startswith("knowledge_fs")
or path.startswith("api/repositories/")
and "knowledge_fs" in filename
or path.startswith("api/tasks/")
and "knowledge_fs" in filename
)
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
"""Classify every KnowledgeFS-named production file or fail closed."""
named_paths = discover_knowledge_fs_production_paths(workspace_root)
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
if unclassified_paths:
raise CoverageGateError(
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
+ ", ".join(sorted(unclassified_paths))
)
if not core_paths:
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
return tuple(sorted(core_paths))
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
api_root = workspace_root / "api"
if not api_root.is_dir():
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
paths: set[str] = set()
for directory, child_directories, filenames in os.walk(api_root):
current_directory = Path(directory)
if current_directory == api_root:
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
for filename in filenames:
path = (current_directory / filename).relative_to(workspace_root).as_posix()
if filename.endswith(".py") and "knowledge_fs" in path:
paths.add(path)
if not paths:
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
return tuple(sorted(paths))
def validate_core_coverage(
report: CoverageReport,
*,
workspace_root: Path,
minimum: float,
) -> CoverageTotals:
"""Require the exact combined line-and-branch percentage for all core files."""
paths = discover_core_coverage_paths(workspace_root)
missing_paths = [path for path in paths if path not in report["files"]]
if missing_paths:
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
covered = 0
total = 0
for path in paths:
summary = report["files"][path]["summary"]
covered += summary["covered_lines"] + summary["covered_branches"]
total += summary["num_statements"] + summary["num_branches"]
if total == 0:
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
return totals
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
base = preferred.strip()
if not base or set(base) == {"0"}:
base = "HEAD^"
result = subprocess.run(
["git", "cat-file", "-e", f"{base}^{{commit}}"],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "commit is unavailable"
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
return base
def collect_changed_glue_lines(
workspace_root: Path,
base: str,
*,
glue_paths: tuple[str, ...],
) -> dict[str, set[int]]:
"""Return added line numbers in the narrow Dify modules touched by this integration."""
result = subprocess.run(
[
"git",
"diff",
"--no-ext-diff",
"--no-color",
"--unified=0",
base,
"--",
*glue_paths,
],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "git diff failed"
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
return parse_added_lines(result.stdout)
def parse_added_lines(diff: str) -> dict[str, set[int]]:
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
changed_lines: dict[str, set[int]] = {}
current_path: str | None = None
current_line: int | None = None
for raw_line in diff.splitlines():
if raw_line.startswith("diff --git "):
current_line = None
continue
if raw_line.startswith("+++ "):
candidate = raw_line[4:]
current_path = candidate[2:] if candidate.startswith("b/") else None
if current_path is not None:
changed_lines.setdefault(current_path, set())
current_line = None
continue
if raw_line.startswith("@@ "):
match = HUNK_HEADER.match(raw_line)
current_line = int(match.group(1)) if match is not None else None
continue
if current_path is None or current_line is None:
continue
if raw_line.startswith("+"):
changed_lines[current_path].add(current_line)
current_line += 1
elif raw_line.startswith("-") or raw_line.startswith("\\"):
continue
else:
current_line += 1
return changed_lines
def validate_changed_glue_coverage(
report: CoverageReport,
*,
changed_lines: dict[str, set[int]],
minimum: float,
) -> CoverageTotals:
"""Require added executable glue lines to be exercised by the focused unit suite."""
covered = 0
total = 0
for path, lines in sorted(changed_lines.items()):
if not lines:
continue
file_coverage = report["files"].get(path)
if file_coverage is None:
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
executed_lines = set(file_coverage["executed_lines"])
executable_lines = executed_lines | set(file_coverage["missing_lines"])
changed_executable_lines = lines & executable_lines
covered += len(changed_executable_lines & executed_lines)
total += len(changed_executable_lines)
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
return totals
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
if totals.percent + 1e-12 < minimum:
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
if __name__ == "__main__":
main()
+732
View File
@@ -0,0 +1,732 @@
"""Pin the in-repository KnowledgeFS contract and validate every Dify product operation.
The lock is intentionally independent of the enclosing Dify commit: it records the staged ``knowledge-fs/`` tree,
the complete generated OpenAPI document, both explicit product-operation manifests, and the active Capability v2
profile and deterministic public-key vector. The full OpenAPI hash
covers request/response schemas, status codes, security, deprecation, and stream metadata. Field-level validation
cross-checks the Dify product registry, Python Capability issuer, TypeScript request guard, and exported OpenAPI;
each product operation must be ready or an explicit gap, and KFS-only activation remains explicitly internal.
Contract export reads the working tree only after proving it matches the staged KnowledgeFS index. This keeps the
OpenAPI bytes and auth manifest aligned with the exact subtree tree ID that will be reviewed and committed.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Literal, TypedDict, cast
import jwt
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
from jwt.algorithms import RSAAlgorithm
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
from dev.knowledge_fs_product_contract import (
capability_operation_runtime_contracts,
parse_capability_operation_policy,
parse_product_operation_gap_manifest,
parse_product_operation_manifest,
product_operation_runtime_contracts,
validate_product_operation_contracts,
)
WORKSPACE_ROOT = API_ROOT.parent
KNOWLEDGE_FS_DIRECTORY = "knowledge-fs"
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
UPSTREAM_PROVENANCE_RELATIVE_PATH = Path("upstream-provenance.json")
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
LOCK_SCHEMA_VERSION = 5
class ContractDeclaration(TypedDict):
"""KnowledgeFS transport contract declared by one Dify Console registry entry."""
operation_id: str
method: str
path: str
required_scope: str | None
response_kind: str
max_response_bytes: int
request_headers: tuple[str, ...]
response_headers: tuple[str, ...]
response_media_types: tuple[str, ...]
class ContractLock(TypedDict):
"""Content-addressed contract inputs that must move together."""
schemaVersion: int
subtreeTree: str
openapiSha256: str
capabilityV2AuthManifestSha256: str
capabilityV2AuthTestVectorSha256: str
productOperationManifestSha256: str
productOperationGapManifestSha256: str
type DeclarationField = Literal[
"method",
"path",
"required_scope",
"response_kind",
"max_response_bytes",
"request_headers",
"response_headers",
"response_media_types",
]
DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
"method",
"path",
"required_scope",
"response_kind",
"max_response_bytes",
"request_headers",
"response_headers",
"response_media_types",
)
def main() -> None:
"""Update or verify the monorepo pin and validate Dify product declarations."""
parser = argparse.ArgumentParser()
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--check", action="store_true")
mode.add_argument("--update-lock", action="store_true")
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
args = parser.parse_args()
workspace_root = args.workspace_root.resolve()
knowledge_fs_root = workspace_root / KNOWLEDGE_FS_DIRECTORY
lock_path = workspace_root / LOCK_RELATIVE_PATH
capability_v2_auth_manifest_path = knowledge_fs_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH
capability_v2_auth_test_vector_path = knowledge_fs_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH
product_operations_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
product_operation_gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
upstream_provenance_path = knowledge_fs_root / UPSTREAM_PROVENANCE_RELATIVE_PATH
ensure_clean_knowledge_fs_worktree(workspace_root)
ensure_contract_inputs_exist(
knowledge_fs_root=knowledge_fs_root,
capability_v2_auth_manifest_path=capability_v2_auth_manifest_path,
capability_v2_auth_test_vector_path=capability_v2_auth_test_vector_path,
product_operations_path=product_operations_path,
product_operation_gaps_path=product_operation_gaps_path,
upstream_provenance_path=upstream_provenance_path,
)
subtree_tree = staged_subtree_tree(workspace_root)
capability_v2_auth_manifest_content = capability_v2_auth_manifest_path.read_bytes()
capability_v2_auth_test_vector_content = capability_v2_auth_test_vector_path.read_bytes()
product_operation_manifest_content = product_operations_path.read_bytes()
product_operation_gap_manifest_content = product_operation_gaps_path.read_bytes()
capability_v2_auth_manifest = load_json_object(capability_v2_auth_manifest_path)
validate_capability_v2_auth_manifest(capability_v2_auth_manifest)
validate_capability_v2_auth_test_vector(
load_json_object(capability_v2_auth_test_vector_path),
capability_v2_auth_manifest,
)
validate_upstream_provenance(load_json_object(upstream_provenance_path))
product_manifest = parse_product_operation_manifest(load_json_object(product_operations_path))
product_gap_manifest = parse_product_operation_gap_manifest(load_json_object(product_operation_gaps_path))
product_runtime_operations = product_operation_runtime_contracts()
capability_runtime_operations = capability_operation_runtime_contracts()
declarations = console_contract_declarations()
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
capability_policy_path = Path(directory) / "dify-capability-v2-operations.json"
subprocess.run(
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
cwd=knowledge_fs_root,
check=True,
)
subprocess.run(
["pnpm", "capability:export", "--", "--output", str(capability_policy_path)],
cwd=knowledge_fs_root,
check=True,
)
openapi_content = openapi_path.read_bytes()
capability_policy = parse_capability_operation_policy(load_json_object(capability_policy_path))
document: dict[str, Any] = json.loads(openapi_content)
validate_product_operation_contracts(
capability_operations=capability_runtime_operations,
capability_policy=capability_policy,
document=document,
gap_manifest=product_gap_manifest,
manifest=product_manifest,
product_operations=product_runtime_operations,
)
validate_declarations(document, declarations)
expected_lock: ContractLock = {
"schemaVersion": LOCK_SCHEMA_VERSION,
"subtreeTree": subtree_tree,
"openapiSha256": sha256(openapi_content),
"capabilityV2AuthManifestSha256": sha256(capability_v2_auth_manifest_content),
"capabilityV2AuthTestVectorSha256": sha256(capability_v2_auth_test_vector_content),
"productOperationManifestSha256": sha256(product_operation_manifest_content),
"productOperationGapManifestSha256": sha256(product_operation_gap_manifest_content),
}
if args.update_lock:
lock_path.write_text(json.dumps(expected_lock, indent=2) + "\n")
return
received_lock = parse_contract_lock(load_json_object(lock_path))
lock_fields = (
(
"capabilityV2AuthManifestSha256",
received_lock["capabilityV2AuthManifestSha256"],
expected_lock["capabilityV2AuthManifestSha256"],
),
(
"capabilityV2AuthTestVectorSha256",
received_lock["capabilityV2AuthTestVectorSha256"],
expected_lock["capabilityV2AuthTestVectorSha256"],
),
(
"productOperationManifestSha256",
received_lock["productOperationManifestSha256"],
expected_lock["productOperationManifestSha256"],
),
(
"productOperationGapManifestSha256",
received_lock["productOperationGapManifestSha256"],
expected_lock["productOperationGapManifestSha256"],
),
("schemaVersion", received_lock["schemaVersion"], expected_lock["schemaVersion"]),
("subtreeTree", received_lock["subtreeTree"], expected_lock["subtreeTree"]),
("openapiSha256", received_lock["openapiSha256"], expected_lock["openapiSha256"]),
)
for field, received_value, expected_value in lock_fields:
if received_value != expected_value:
raise RuntimeError(
f"KnowledgeFS contract lock field {field} drifted: "
f"expected {expected_value!r}, received {received_value!r}. "
"Run --update-lock intentionally after reviewing the staged subtree and contract changes."
)
def ensure_contract_inputs_exist(
*,
knowledge_fs_root: Path,
capability_v2_auth_manifest_path: Path,
capability_v2_auth_test_vector_path: Path,
product_operations_path: Path,
product_operation_gaps_path: Path,
upstream_provenance_path: Path,
) -> None:
"""Fail with a stable error before invoking package tooling when a contract input is absent."""
required_paths = (
knowledge_fs_root / "package.json",
capability_v2_auth_manifest_path,
capability_v2_auth_test_vector_path,
product_operations_path,
product_operation_gaps_path,
upstream_provenance_path,
)
missing_paths = [path for path in required_paths if not path.is_file()]
if missing_paths:
missing = ", ".join(str(path) for path in missing_paths)
raise RuntimeError(f"KnowledgeFS contract input is missing: {missing}")
def ensure_clean_knowledge_fs_worktree(workspace_root: Path) -> None:
"""Require exported KnowledgeFS files to exactly match the staged index tree.
Staged changes are expected during intentional lock updates. Unstaged tracked changes and untracked files are
rejected because the export process reads the working tree while the tree ID is calculated from the index.
"""
subtree_path = f"{KNOWLEDGE_FS_DIRECTORY}/"
unstaged = subprocess.run(
["git", "diff", "--quiet", "--", subtree_path],
cwd=workspace_root,
check=False,
)
if unstaged.returncode > 1:
raise RuntimeError("git diff failed while validating the staged KnowledgeFS subtree")
untracked = run(
"git",
"ls-files",
"--others",
"--exclude-standard",
"--",
subtree_path,
cwd=workspace_root,
).strip()
if unstaged.returncode != 0 or untracked:
raise RuntimeError(
"knowledge-fs/ contains unstaged or untracked changes; stage or remove them before contract export"
)
def staged_subtree_tree(workspace_root: Path) -> str:
"""Return the Git tree object for the staged ``knowledge-fs/`` subtree."""
return run("git", "write-tree", f"--prefix={KNOWLEDGE_FS_DIRECTORY}/", cwd=workspace_root).strip()
def load_json_object(path: Path) -> dict[str, Any]:
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
return cast(dict[str, Any], value)
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
"""Validate the compact, non-self-referential contract lock schema."""
expected_fields = {
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"schemaVersion",
"subtreeTree",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
for field in (
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"subtreeTree",
):
field_value = value.get(field)
expected_length = 40 if field == "subtreeTree" else 64
if (
not isinstance(field_value, str)
or len(field_value) != expected_length
or any(character not in "0123456789abcdef" for character in field_value)
):
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
return cast(ContractLock, value)
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
available_operation_ids = [
operation_id
for path_item in document.get("paths", {}).values()
for method in OPENAPI_METHODS
if isinstance(path_item, dict)
for operation in (path_item.get(method),)
if isinstance(operation, dict)
for operation_id in (operation.get("operationId"),)
if isinstance(operation_id, str) and operation_id
]
for operation_id in required_operation_ids:
count = available_operation_ids.count(operation_id)
if count != 1:
raise ValueError(
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
)
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
expected_fields = {
"active",
"audience",
"callerProfiles",
"claimBindings",
"issuer",
"lifecycle",
"maxTtlSeconds",
"productionReady",
"profileId",
"protectedHeader",
"requiredClaims",
"resourceContract",
"runtimeAssembly",
"schemaVersion",
"signatureAlgorithms",
"tokenKind",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
fixed_values = {
"active": True,
"audience": "knowledge-fs",
"issuer": "dify-control-plane",
"lifecycle": "active",
"maxTtlSeconds": 60,
"productionReady": True,
"profileId": "dify-capability-v2",
"schemaVersion": 3,
"signatureAlgorithms": ["RS256"],
"tokenKind": "jwt",
}
for field, expected in fixed_values.items():
if value.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
if value.get("protectedHeader") != {
"algorithm": "RS256",
"keyIdClaim": "kid",
"keyIdRequired": True,
"type": "JWT",
}:
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
required_claims = [
"action",
"actor",
"aud",
"authz_revision",
"azp",
"caller_kind",
"cap_ver",
"content_policy_revision",
"content_scope_ids",
"control_space_id",
"exp",
"grant_id",
"iat",
"iss",
"jti",
"namespace_id",
"nbf",
"resource",
"sub",
"trace_id",
]
if value.get("requiredClaims") != required_claims:
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
if value.get("claimBindings") != {
"action": "action",
"callerKind": "caller_kind",
"controlSpace": "control_space_id",
"namespace": "namespace_id",
"resource": "resource",
"resourceParent": "resource.parent_id",
"subject": "sub",
}:
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
if value.get("resourceContract") != {
"fields": ["id", "parent_id", "type"],
"parentForbiddenFor": ["namespace", "knowledge_space"],
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
}:
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
if value.get("callerProfiles") != {
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-kfs-credential:"},
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
}:
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
if value.get("runtimeAssembly") != {
"failClosed": True,
"keySelection": "kid",
"maximumPublishedKeys": 3,
"verificationKeySource": "jwks",
}:
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
def validate_capability_v2_auth_test_vector(
value: dict[str, Any],
manifest: dict[str, Any],
) -> None:
"""Verify the deterministic public-key vector and every security-sensitive binding."""
expected_fields = {
"algorithm",
"audience",
"expectedClaims",
"expectedPrincipal",
"issuer",
"operation",
"profileId",
"protectedHeader",
"publicJwk",
"schemaVersion",
"testOnly",
"token",
"ttlSeconds",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
if (
value.get("schemaVersion") != 2
or value.get("profileId") != manifest.get("profileId")
or value.get("testOnly") is not True
or value.get("algorithm") != "RS256"
or value.get("issuer") != manifest.get("issuer")
or value.get("audience") != manifest.get("audience")
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
):
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
protected_header = value.get("protectedHeader")
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
kid = protected_header.get("kid")
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
public_jwk = value.get("publicJwk")
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
if (
public_jwk.get("alg") != "RS256"
or public_jwk.get("kid") != kid
or public_jwk.get("kty") != "RSA"
or public_jwk.get("use") != "sig"
or not _is_non_blank(public_jwk.get("e"))
or not _is_non_blank(public_jwk.get("n"))
):
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
claims = value.get("expectedClaims")
required_claims = manifest.get("requiredClaims")
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
operation = value.get("operation")
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
resource = claims.get("resource")
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
expected_operation = {
"action": "documents.read",
"method": "GET",
"operationId": "getDocument",
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
}
if operation != expected_operation:
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
exact_claims = {
"action": operation["action"],
"aud": value["audience"],
"caller_kind": "interactive",
"cap_ver": 2,
"control_space_id": "control-space-contract-vector",
"iss": value["issuer"],
"namespace_id": "workspace-contract-vector",
"resource": {
"id": "document-contract-vector",
"parent_id": "space-contract-vector",
"type": "document",
},
"sub": "dify-account:account-contract-vector",
}
for field, expected in exact_claims.items():
if claims.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
issued_at = claims.get("iat")
not_before = claims.get("nbf")
expires_at = claims.get("exp")
if (
not isinstance(issued_at, int)
or isinstance(issued_at, bool)
or not_before != issued_at
or not isinstance(expires_at, int)
or isinstance(expires_at, bool)
or expires_at - issued_at != value["ttlSeconds"]
):
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
expected_principal = {
"callerKind": claims["caller_kind"],
"subject": {
"scopes": ["knowledge-spaces:read"],
"subjectId": claims["sub"],
"tenantId": claims["namespace_id"],
},
}
if value.get("expectedPrincipal") != expected_principal:
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
token = value.get("token")
if not isinstance(token, str) or not _is_non_blank(token):
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
try:
verification_key = RSAAlgorithm.from_jwk(public_jwk)
if not isinstance(verification_key, RSAPublicKey):
raise ValueError("Capability vector verification key is not RSA public material")
header = jwt.get_unverified_header(token)
decoded_claims = jwt.decode(
token,
verification_key,
algorithms=["RS256"],
audience=cast(str, value["audience"]),
issuer=cast(str, value["issuer"]),
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
)
except (jwt.PyJWTError, TypeError, ValueError) as exc:
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
if header != protected_header or decoded_claims != claims:
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
def _is_non_blank(value: object) -> bool:
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
def validate_upstream_provenance(value: dict[str, Any]) -> None:
"""Validate the imported-source provenance that is itself covered by the subtree tree ID."""
expected_fields = {"commit", "release", "repository", "schemaVersion"}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS upstream provenance fields must be exactly {sorted(expected_fields)}")
if value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS upstream provenance must use schemaVersion 1")
repository = value.get("repository")
commit = value.get("commit")
if not isinstance(repository, str) or not repository.startswith("https://"):
raise ValueError("KnowledgeFS upstream provenance repository must be an HTTPS URL")
if (
not isinstance(commit, str)
or len(commit) != 40
or any(character not in "0123456789abcdef" for character in commit)
):
raise ValueError("KnowledgeFS upstream provenance commit must be a lowercase full Git SHA")
if value.get("release") is not None and not isinstance(value["release"], str):
raise ValueError("KnowledgeFS upstream provenance release must be null or a string")
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
for path, path_item in document.get("paths", {}).items():
for method in OPENAPI_METHODS:
operation = path_item.get(method)
if operation is None:
continue
operation_id = operation.get("operationId")
if isinstance(operation_id, str) and operation_id:
operations_by_id.setdefault(operation_id, []).append((method, path, path_item, operation))
declared_ids: set[str] = set()
for declaration in declarations:
operation_id = declaration["operation_id"]
if operation_id in declared_ids:
raise ValueError(f"Dify Console registry has duplicate operationId: {operation_id}")
declared_ids.add(operation_id)
matches = operations_by_id.get(operation_id, [])
if not matches:
raise ValueError(f"KnowledgeFS OpenAPI has no operationId: {operation_id}")
if len(matches) > 1:
raise ValueError(f"KnowledgeFS OpenAPI has duplicate operationId: {operation_id}")
method, path, path_item, operation = matches[0]
if not path.startswith("/"):
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
if method not in PROXY_METHODS:
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
expected: ContractDeclaration = {
"operation_id": operation_id,
"method": method.upper(),
"path": path[1:],
"required_scope": required_scope(document, operation),
"response_kind": response_kind(operation),
"max_response_bytes": required_max_response_bytes(operation),
"request_headers": request_header_names(path_item, operation),
"response_headers": response_header_names(operation),
"response_media_types": response_media_types(operation),
}
for field in DECLARATION_FIELDS:
expected_value = expected[field]
received_value = declaration[field]
if received_value != expected_value:
raise ValueError(
f"KnowledgeFS operation {operation_id} field {field} drifted: "
f"expected {expected_value!r}, received {received_value!r}"
)
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
return ()
def response_kind(operation: dict[str, Any]) -> str:
media_types = response_media_types(operation)
if "text/event-stream" in media_types:
return "stream"
if "application/octet-stream" in media_types:
return "binary"
return "buffered"
def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
media_types: set[str] = set()
for status, response in operation.get("responses", {}).items():
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit()):
media_types.update(response.get("content", {}))
return tuple(sorted(media_types))
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
scope = operation.get("x-knowledge-fs-required-scope")
security = operation["security"] if "security" in operation else document.get("security")
if security == []:
if scope is not None:
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
return None
if security != [{"bearerAuth": []}]:
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
return scope
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
def required_max_response_bytes(operation: dict[str, Any]) -> int:
value = operation.get("x-knowledge-fs-max-response-bytes")
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError(f"KnowledgeFS operation has no valid response byte limit: {value}")
return value
def request_header_names(path_item: dict[str, Any], operation: dict[str, Any]) -> tuple[str, ...]:
names: set[str] = set()
for parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]:
if "$ref" in parameter:
raise ValueError(f"KnowledgeFS request header references are not supported: {parameter['$ref']}")
if parameter.get("in") == "header":
names.add(parameter["name"].lower())
return tuple(sorted(names))
def response_header_names(operation: dict[str, Any]) -> tuple[str, ...]:
return tuple(
sorted(
{
name.lower()
for response in operation.get("responses", {}).values()
for name in response.get("headers", {})
}
)
)
def sha256(content: bytes) -> str:
return hashlib.sha256(content).hexdigest()
def run(*command: str, cwd: Path) -> str:
return subprocess.run(command, cwd=cwd, check=True, capture_output=True, text=True).stdout
if __name__ == "__main__":
main()
+494
View File
@@ -0,0 +1,494 @@
"""Cross-service KnowledgeFS product-operation contract validation.
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
policy remain executable sources of truth. A product operation must appear exactly once as ready
or as an explicit gap; internal KFS-only operations require a named exclusion.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any, Literal, NamedTuple, TypedDict, cast
class ProductOperationLimits(TypedDict):
kfsMaxResponseBytes: int
productMaxRequestBytes: int
productMaxResponseBytes: int
class ProductOperationStream(TypedDict):
kfsResponseKind: str
productKind: str
class ProductOperationManifestEntry(TypedDict):
action: str
kfsOperationId: str
limits: ProductOperationLimits
method: str
path: str
productOperationId: str
resource: str
stream: ProductOperationStream
transport: str
type ProductOperationManifestField = Literal[
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
]
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
)
class ProductOperationManifest(TypedDict):
operations: list[ProductOperationManifestEntry]
schemaVersion: int
class ProductOperationGapEntry(ProductOperationManifestEntry):
reason: str
reasonCode: str
replacementProductOperationIds: list[str]
class InternalKfsOperationExclusion(TypedDict):
kfsOperationId: str
reason: str
reasonCode: str
class ProductOperationGapManifest(TypedDict):
gaps: list[ProductOperationGapEntry]
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
schemaVersion: int
class ProductOperationRuntimeContract(NamedTuple):
action: str
kfs_operation_id: str
max_request_bytes: int
max_response_bytes: int
method: str
path: str
product_operation_id: str
ready: bool
resource: str
stream_kind: str
transport: str
class CapabilityOperationRuntimeContract(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
path: str
resource: str
class CapabilityOperationPolicy(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
parent_resource_binding: dict[str, object] | None
path: str
resource: str
resource_binding: dict[str, object]
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
"""Read the complete typed Dify product registry without normalizing away gaps."""
from services.knowledge_fs.product_operations import (
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
is_product_operation_ready,
)
contracts: list[ProductOperationRuntimeContract] = []
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
contracts.append(
ProductOperationRuntimeContract(
action=operation.action,
kfs_operation_id=operation.capability_operation_id,
max_request_bytes=operation.max_request_bytes,
max_response_bytes=operation.max_response_bytes,
method=operation.method,
path=operation.kfs_path,
product_operation_id=product_operation_id,
ready=is_product_operation_ready(product_operation_id),
resource=operation.resource_resolver,
stream_kind=operation.stream_kind,
transport=operation.transport,
)
)
return tuple(contracts)
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
"""Read the Python issuer policy used to mint each operation Capability."""
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
return tuple(
CapabilityOperationRuntimeContract(
action=operation.action,
allowed_caller_kinds=operation.allowed_caller_kinds,
method=operation.method,
operation_id=operation_id,
path=operation.path,
resource=operation.resource_type,
)
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
)
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
"""Parse the ready product-operation inventory with a closed schema."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
return {
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
"schemaVersion": 2,
}
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
if set(value) != expected_fields or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
gaps = value.get("gaps")
exclusions = value.get("internalKfsOperationExclusions")
if not isinstance(gaps, list) or not gaps or not isinstance(exclusions, list):
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
parsed_exclusions: list[InternalKfsOperationExclusion] = []
for raw in exclusions:
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
for field in ("kfsOperationId", "reason", "reasonCode"):
_required_string(raw.get(field), f"internal exclusion {field}")
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
return {
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
"internalKfsOperationExclusions": parsed_exclusions,
"schemaVersion": 2,
}
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
"""Parse the runtime-exported TypeScript request-guard registry."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
parsed: list[CapabilityOperationPolicy] = []
for raw in operations:
expected_fields = {
"action",
"allowedCallerKinds",
"method",
"operationId",
"parentResourceBinding",
"path",
"resourceBinding",
"resourceType",
}
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
callers = raw.get("allowedCallerKinds")
resource_binding = raw.get("resourceBinding")
parent_binding = raw.get("parentResourceBinding")
if (
not isinstance(callers, list)
or not callers
or not all(isinstance(caller, str) and caller for caller in callers)
or not isinstance(resource_binding, dict)
or not resource_binding
or (parent_binding is not None and not isinstance(parent_binding, dict))
):
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
for field in ("action", "method", "operationId", "path", "resourceType"):
_required_string(raw.get(field), f"Capability operation {field}")
parsed.append(
CapabilityOperationPolicy(
action=cast(str, raw["action"]),
allowed_caller_kinds=tuple(callers),
method=cast(str, raw["method"]),
operation_id=cast(str, raw["operationId"]),
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
path=cast(str, raw["path"]),
resource=cast(str, raw["resourceType"]),
resource_binding=cast(dict[str, object], resource_binding),
)
)
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
return tuple(parsed)
def validate_product_operation_contracts(
*,
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
capability_policy: tuple[CapabilityOperationPolicy, ...],
document: dict[str, Any],
gap_manifest: ProductOperationGapManifest,
manifest: ProductOperationManifest,
product_operations: tuple[ProductOperationRuntimeContract, ...],
) -> None:
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
if len(ready_entries) != len(manifest["operations"]):
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
if len(gap_entries) != len(gap_manifest["gaps"]):
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
if set(ready_entries) & set(gap_entries):
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
openapi_operations = _openapi_operations(document)
for product_operation_id, runtime in runtime_by_product.items():
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
if runtime.ready != (product_operation_id in ready_entries):
state = "ready" if runtime.ready else "gap"
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
if product_operation_id in gap_entries:
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
if not replacements or any(replacement not in ready_entries for replacement in replacements):
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
exclusions = gap_manifest["internalKfsOperationExclusions"]
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
if len(exclusion_by_id) != len(exclusions):
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
lifecycle_operation_ids = {
"activateDifyWorkspaceIntegration",
"freezeDifyWorkspaceIntegration",
}
if set(exclusion_by_id) != lifecycle_operation_ids:
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
for operation_id in exclusion_by_id:
if operation_id in product_capability_ids:
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
_validate_capability_alignment(
operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
python_operation = python_capabilities[operation_id]
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
base_fields = {
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
}
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS product operation entry fields are invalid")
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
_required_string(raw.get(field), f"product operation {field}")
limits = raw.get("limits")
stream = raw.get("stream")
if not isinstance(limits, dict) or set(limits) != {
"kfsMaxResponseBytes",
"productMaxRequestBytes",
"productMaxResponseBytes",
}:
raise ValueError("KnowledgeFS product operation limits are invalid")
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
raise ValueError("KnowledgeFS product operation stream contract is invalid")
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
_required_string(stream.get("productKind"), "product stream kind")
if gap:
_required_string(raw.get("reason"), "product gap reason")
_required_string(raw.get("reasonCode"), "product gap reasonCode")
replacements = raw.get("replacementProductOperationIds")
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
return cast(ProductOperationManifestEntry, raw)
def _validate_product_entry(
entry: ProductOperationManifestEntry,
runtime: ProductOperationRuntimeContract,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
_validate_capability_alignment(
runtime.kfs_operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
expected: ProductOperationManifestEntry = {
"action": runtime.action,
"kfsOperationId": runtime.kfs_operation_id,
"limits": {
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
"productMaxRequestBytes": runtime.max_request_bytes,
"productMaxResponseBytes": runtime.max_response_bytes,
},
"method": runtime.method,
"path": runtime.path,
"productOperationId": runtime.product_operation_id,
"resource": runtime.resource,
"stream": {
"kfsResponseKind": _openapi_response_kind(openapi_operation),
"productKind": runtime.stream_kind,
},
"transport": runtime.transport,
}
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
expected_value = expected[field]
if entry[field] != expected_value:
raise ValueError(
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
f"expected {expected_value!r}, received {entry[field]!r}"
)
def _validate_capability_alignment(
operation_id: str,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
python_operation = python_capabilities.get(operation_id)
guard_operation = guard_capabilities.get(operation_id)
openapi_operation = openapi_operations.get(operation_id)
if python_operation is None or guard_operation is None or openapi_operation is None:
raise ValueError(
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
)
method, path, _ = openapi_operation
expected = (
python_operation.action,
python_operation.method,
python_operation.path,
python_operation.resource,
)
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
if received_guard != expected:
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
if (method, path) != (python_operation.method, python_operation.path):
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
duplicates: set[str] = set()
for path, path_item in document.get("paths", {}).items():
if not isinstance(path, str) or not isinstance(path_item, dict):
continue
for method in ("delete", "get", "patch", "post", "put"):
operation = path_item.get(method)
if not isinstance(operation, dict):
continue
operation_id = operation.get("operationId")
if not isinstance(operation_id, str) or not operation_id:
continue
if operation_id in operations:
duplicates.add(operation_id)
operations[operation_id] = (method.upper(), path, operation)
if duplicates:
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
return operations
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
media_types = {
media_type
for status, response in operation.get("responses", {}).items()
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
for media_type in response.get("content", {})
}
if "text/event-stream" in media_types:
return "stream"
if "application/octet-stream" in media_types:
return "binary"
return "buffered"
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
value = operation.get("x-knowledge-fs-max-response-bytes")
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
return value
def _required_string(value: object, field: str) -> str:
if not isinstance(value, str) or not value.strip() or value != value.strip():
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
return value
def _unique(values: Iterable[str], label: str) -> None:
sequence = tuple(values)
if len(sequence) != len(set(sequence)):
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
__all__ = [
"CapabilityOperationPolicy",
"CapabilityOperationRuntimeContract",
"ProductOperationGapManifest",
"ProductOperationManifest",
"ProductOperationRuntimeContract",
"capability_operation_runtime_contracts",
"parse_capability_operation_policy",
"parse_product_operation_gap_manifest",
"parse_product_operation_manifest",
"product_operation_runtime_contracts",
"validate_product_operation_contracts",
]
+8
View File
@@ -164,6 +164,14 @@ def init_app(app: DifyApp) -> Celery:
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
if get_configured_knowledge_fs_lifecycle_worker_readiness().ready:
imports.append("tasks.knowledge_fs_lifecycle_tasks")
beat_schedule["knowledge_fs_lifecycle_worker"] = {
"task": "tasks.knowledge_fs_lifecycle_tasks.run_knowledge_fs_lifecycle_worker",
"schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
}
if dify_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK:
imports.append("schedule.clean_embedding_cache_task")
beat_schedule["clean_embedding_cache_task"] = {
+2
View File
@@ -27,6 +27,7 @@ def init_app(app: DifyApp):
import_migration_data,
install_plugins,
install_rag_pipeline_plugins,
knowledge_fs_control_space,
migrate_data_for_plugin,
migrate_dataset_permissions_to_rbac,
migrate_member_roles_to_rbac,
@@ -88,6 +89,7 @@ def init_app(app: DifyApp):
export_migration_data_template,
import_migration_data,
migration_data_wizard,
knowledge_fs_control_space,
]
for cmd in cmds_to_register:
app.cli.add_command(cmd)
@@ -0,0 +1,24 @@
"""Production assembly for KnowledgeFS database-backed metrics."""
from __future__ import annotations
import logging
from flask import Flask
from core.db.session_factory import session_factory
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
)
from services.knowledge_fs.observability import get_knowledge_fs_operational_metrics
logger = logging.getLogger(__name__)
def init_app(app: Flask) -> None:
del app
try:
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_factory.get_session_maker())
get_knowledge_fs_operational_metrics().register_control_space_state_gauge(reader)
except Exception:
logger.exception("KnowledgeFS operational metric registration failed; continuing application startup")
+9
View File
@@ -0,0 +1,9 @@
{
"schemaVersion": 5,
"subtreeTree": "04f9a6189c215f4fcccdccbb3ae7fc810b927463",
"openapiSha256": "df702b15769f79c561e34d72472a2ec8dddcdefe4bd4c82858f06a98cd6eae8c",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
"productOperationManifestSha256": "5ceef85419dfba9c803732f78f31c803a0836e883972bb67f26ae43b0c78d341",
"productOperationGapManifestSha256": "ccbae37fe658177a77529822211a2cf02e72b5815cbc9b7e7e65e6817ba5e0a9"
}
@@ -0,0 +1,44 @@
{
"schemaVersion": 2,
"gaps": [
{
"productOperationId": "createDocument",
"kfsOperationId": "uploadDocument",
"method": "POST",
"path": "/knowledge-spaces/{id}/documents",
"action": "documents.create",
"resource": "knowledge_space",
"transport": "multipart",
"stream": {
"productKind": "buffered-multipart",
"kfsResponseKind": "buffered"
},
"limits": {
"productMaxRequestBytes": 0,
"productMaxResponseBytes": 0,
"kfsMaxResponseBytes": 1048576
},
"reasonCode": "NON_HOMOMORPHIC_MULTIPART_TRANSPORT",
"reason": "Legacy buffered multipart document creation has no homomorphic Dify product transport; use Capability v2 direct-upload operations.",
"replacementProductOperationIds": [
"createUploadSession",
"presignUploadSessionPart",
"uploadSmallFile",
"completeUploadSession",
"abortUploadSession"
]
}
],
"internalKfsOperationExclusions": [
{
"kfsOperationId": "activateDifyWorkspaceIntegration",
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
"reason": "Workspace activation is an internal cutover control-plane operation and is never exposed as a product operation."
},
{
"kfsOperationId": "freezeDifyWorkspaceIntegration",
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
"reason": "Workspace maintenance freeze is an internal cutover control-plane operation and is never exposed as a product operation."
}
]
}
+53
View File
@@ -0,0 +1,53 @@
{
"schemaVersion": 2,
"operations": [
{"productOperationId":"batchSpaceSummaries","kfsOperationId":"batchKnowledgeSpaceProductSummaries","method":"POST","path":"/internal/knowledge-spaces/product-summaries/batch","action":"knowledge_spaces.status.batch","resource":"namespace","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSpace","kfsOperationId":"getKnowledgeSpace","method":"GET","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSpace","kfsOperationId":"updateKnowledgeSpace","method":"PATCH","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSettings","kfsOperationId":"getKnowledgeSpaceProductSettings","method":"GET","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocument","kfsOperationId":"getDocument","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocumentOutline","kfsOperationId":"getDocumentOutline","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/outline","action":"documents.outline.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocumentRevisions","kfsOperationId":"listDocumentRevisions","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions","action":"documents.revisions.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateDocumentMetadata","kfsOperationId":"patchDocumentMetadata","method":"PATCH","path":"/knowledge-spaces/{id}/documents/{documentId}/metadata","action":"documents.metadata.update","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocumentChunks","kfsOperationId":"listDocumentChunks","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks","action":"documents.chunks.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocumentChunk","kfsOperationId":"getDocumentChunk","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}","action":"documents.chunks.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"deleteDocument","kfsOperationId":"requestDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.delete","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"bulkDeleteDocuments","kfsOperationId":"requestBulkDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/bulk","action":"documents.bulk.delete","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"reindexDocuments","kfsOperationId":"bulkReindexDocuments","method":"POST","path":"/knowledge-spaces/{id}/documents/bulk/reindex","action":"documents.bulk.reindex","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getCompilationJob","kfsOperationId":"getDocumentCompilationJob","method":"GET","path":"/jobs/{id}","action":"document_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"cancelCompilationJob","kfsOperationId":"cancelDocumentCompilationJob","method":"DELETE","path":"/jobs/{id}","action":"document_jobs.cancel","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"retryCompilationJob","kfsOperationId":"retryDocumentCompilationJob","method":"POST","path":"/jobs/{id}/retry","action":"document_jobs.retry","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getBulkJob","kfsOperationId":"getBulkOperation","method":"GET","path":"/bulk-jobs/{id}","action":"bulk_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSources","kfsOperationId":"listKnowledgeSpaceSources","method":"GET","path":"/knowledge-spaces/{id}/sources","action":"sources.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createSource","kfsOperationId":"createKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources","action":"sources.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSource","kfsOperationId":"getKnowledgeSpaceSource","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.read","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSource","kfsOperationId":"updateKnowledgeSpaceSource","method":"PATCH","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"deleteSource","kfsOperationId":"requestSourceDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.delete","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"testSource","kfsOperationId":"testKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/test","action":"sources.test","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"crawlSource","kfsOperationId":"crawlKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl","action":"sources.crawl","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourcePages","kfsOperationId":"listKnowledgeSpaceSourcePages","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/pages","action":"sources.pages.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"importSourcePages","kfsOperationId":"importKnowledgeSpaceSourcePages","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import","action":"sources.pages.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourceFiles","kfsOperationId":"listKnowledgeSpaceSourceFiles","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/files","action":"sources.files.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"importSourceFiles","kfsOperationId":"importKnowledgeSpaceSourceFiles","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import-files","action":"sources.files.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createQuery","kfsOperationId":"createQuery","method":"POST","path":"/queries","action":"queries.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}},
{"productOperationId":"listResearchTasks","kfsOperationId":"listKnowledgeSpaceResearchTasks","method":"GET","path":"/knowledge-spaces/{id}/research-tasks","action":"research_tasks.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createResearchTask","kfsOperationId":"createResearchTask","method":"POST","path":"/research-tasks","action":"research_tasks.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"planResearchTask","kfsOperationId":"planResearchTask","method":"POST","path":"/research-tasks/plan","action":"research_tasks.plan","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getResearchTask","kfsOperationId":"getResearchTask","method":"GET","path":"/research-tasks/{id}","action":"research_tasks.read","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listResearchTaskPartials","kfsOperationId":"listResearchTaskPartials","method":"GET","path":"/research-tasks/{id}/partials","action":"research_tasks.partials.list","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"cancelResearchTask","kfsOperationId":"cancelResearchTask","method":"DELETE","path":"/research-tasks/{id}","action":"research_tasks.cancel","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraces","kfsOperationId":"listKnowledgeSpaceQualityTraces","method":"GET","path":"/knowledge-spaces/{id}/quality/traces","action":"quality.traces.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getTrace","kfsOperationId":"getAnswerTrace","method":"GET","path":"/queries/{traceId}","action":"queries.read","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceEvidence","kfsOperationId":"listQueryEvidence","method":"GET","path":"/queries/{traceId}/evidence","action":"queries.evidence.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceConflicts","kfsOperationId":"listQueryConflicts","method":"GET","path":"/queries/{traceId}/conflicts","action":"queries.conflicts.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceMissing","kfsOperationId":"listQueryMissing","method":"GET","path":"/queries/{traceId}/missing","action":"queries.missing.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createUploadSession","kfsOperationId":"createUploadSession","method":"POST","path":"/knowledge-spaces/{id}/upload-sessions","action":"upload_sessions.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"presignUploadSessionPart","kfsOperationId":"presignUploadSessionPart","method":"POST","path":"/upload-sessions/{id}/parts/{partNumber}/presign","action":"upload_sessions.write","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"uploadSmallFile","kfsOperationId":"uploadSmallFile","method":"POST","path":"/upload-sessions/{id}/small-file","action":"upload_sessions.write","resource":"upload_session","transport":"binary","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":8388608,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"completeUploadSession","kfsOperationId":"completeUploadSession","method":"POST","path":"/upload-sessions/{id}/complete","action":"upload_sessions.complete","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"abortUploadSession","kfsOperationId":"abortUploadSession","method":"POST","path":"/upload-sessions/{id}/abort","action":"upload_sessions.abort","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"streamResearchTask","kfsOperationId":"streamResearchTaskProgress","method":"GET","path":"/research-tasks/{id}/events","action":"research_tasks.stream","resource":"research_task","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}}
]
}
+33 -12
View File
@@ -16,12 +16,19 @@ from urllib.parse import quote
import boto3
import orjson
from botocore.client import Config
from botocore.exceptions import ClientError
from botocore.exceptions import BotoCoreError, ClientError
from configs import dify_config
logger = logging.getLogger(__name__)
_OBJECT_NOT_FOUND_ERROR_CODES = frozenset({"404", "NoSuchKey", "NotFound"})
def _is_object_not_found_error(error: ClientError) -> bool:
error_code = str(error.response.get("Error", {}).get("Code", ""))
return error_code in _OBJECT_NOT_FOUND_ERROR_CODES
class ArchiveStorageError(Exception):
"""Base exception for archive storage operations."""
@@ -138,10 +145,11 @@ class ArchiveStorage:
response = self.client.get_object(Bucket=self.bucket, Key=key)
return response["Body"].read()
except ClientError as e:
error_code = e.response.get("Error", {}).get("Code")
if error_code == "NoSuchKey":
raise FileNotFoundError(f"Archive object not found: {key}")
raise ArchiveStorageError(f"Failed to download object '{key}': {e}")
if _is_object_not_found_error(e):
raise FileNotFoundError(f"Archive object not found: {key}") from e
raise ArchiveStorageError(f"Failed to download object '{key}': {e}") from e
except BotoCoreError as e:
raise ArchiveStorageError(f"Failed to download object '{key}': {e}") from e
def get_object_stream(self, key: str) -> Generator[bytes, None, None]:
"""
@@ -161,10 +169,11 @@ class ArchiveStorage:
response = self.client.get_object(Bucket=self.bucket, Key=key)
yield from response["Body"].iter_chunks()
except ClientError as e:
error_code = e.response.get("Error", {}).get("Code")
if error_code == "NoSuchKey":
raise FileNotFoundError(f"Archive object not found: {key}")
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}")
if _is_object_not_found_error(e):
raise FileNotFoundError(f"Archive object not found: {key}") from e
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}") from e
except BotoCoreError as e:
raise ArchiveStorageError(f"Failed to stream object '{key}': {e}") from e
def object_exists(self, key: str) -> bool:
"""
@@ -175,12 +184,19 @@ class ArchiveStorage:
Returns:
True if object exists, False otherwise
Raises:
ArchiveStorageError: If storage cannot authoritatively determine object existence
"""
try:
self.client.head_object(Bucket=self.bucket, Key=key)
return True
except ClientError:
return False
except ClientError as e:
if _is_object_not_found_error(e):
return False
raise ArchiveStorageError(f"Failed to check archive object '{key}': {e}") from e
except BotoCoreError as e:
raise ArchiveStorageError(f"Failed to check archive object '{key}': {e}") from e
def delete_object(self, key: str) -> None:
"""
@@ -196,7 +212,12 @@ class ArchiveStorage:
self.client.delete_object(Bucket=self.bucket, Key=key)
logger.debug("Deleted object: %s", key)
except ClientError as e:
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}")
if _is_object_not_found_error(e):
logger.debug("Archive object was already absent: %s", key)
return
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}") from e
except BotoCoreError as e:
raise ArchiveStorageError(f"Failed to delete object '{key}': {e}") from e
def generate_presigned_url(
self,
@@ -0,0 +1,45 @@
"""add workflow-run archive bundle cursor indexes
Revision ID: 3c9f8e2a1d7b
Revises: 7a1c2d9e4b60
Create Date: 2026-07-15 16:00:00.000000
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = "3c9f8e2a1d7b"
down_revision = "7a1c2d9e4b60"
branch_labels = None
depends_on = None
_TABLE_NAME = "workflow_run_archive_bundles"
_INDEXES = (
("workflow_run_archive_bundle_month_id_idx", ("year", "month", "id")),
("workflow_run_archive_bundle_month_shard_id_idx", ("year", "month", "shard", "id")),
)
def _is_postgresql() -> bool:
return op.get_bind().dialect.name == "postgresql"
def upgrade() -> None:
if _is_postgresql():
with op.get_context().autocommit_block():
for index_name, columns in _INDEXES:
op.create_index(index_name, _TABLE_NAME, columns, postgresql_concurrently=True)
return
for index_name, columns in _INDEXES:
op.create_index(index_name, _TABLE_NAME, columns)
def downgrade() -> None:
if _is_postgresql():
with op.get_context().autocommit_block():
for index_name, _ in reversed(_INDEXES):
op.drop_index(index_name, table_name=_TABLE_NAME, postgresql_concurrently=True)
return
for index_name, _ in reversed(_INDEXES):
op.drop_index(index_name, table_name=_TABLE_NAME)
@@ -0,0 +1,387 @@
"""add independent KnowledgeFS control-plane tables
Revision ID: a4e7c2f91b30
Revises: 3c9f8e2a1d7b
Create Date: 2026-07-21 12:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "a4e7c2f91b30"
down_revision = "3c9f8e2a1d7b"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def _timestamps() -> tuple[sa.Column, sa.Column]:
return (
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
)
def upgrade() -> None:
op.create_table(
"knowledge_fs_control_spaces",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("knowledge_space_id", models.types.StringUUID(), nullable=True),
sa.Column("knowledge_space_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("owner_account_id", models.types.StringUUID(), nullable=False),
sa.Column("visibility", sa.String(length=32), server_default=sa.text("'only_me'"), nullable=False),
sa.Column("provisioning_key", sa.String(length=255), nullable=False),
sa.Column("lifecycle_operation_id", models.types.StringUUID(), nullable=True),
sa.Column("state", sa.String(length=32), server_default=sa.text("'provisioning'"), nullable=False),
sa.Column("resource_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
sa.Column("last_error_code", sa.String(length=255), nullable=True),
sa.Column("last_error_message", models.types.LongText(), nullable=True),
sa.Column("last_synced_at", sa.DateTime(), nullable=True),
sa.Column("deletion_irreversible_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
sa.UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
sa.UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id"],
["tenants.id"],
name="kfs_control_space_workspace_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("resource_version >= 0", name="kfs_control_space_resource_version_ck"),
sa.CheckConstraint("attempt_count >= 0", name="kfs_control_space_attempt_count_ck"),
sa.CheckConstraint(
"state != 'active' OR knowledge_space_id IS NOT NULL",
name="kfs_control_space_active_registration_ck",
),
sa.CheckConstraint(
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
name="kfs_control_space_irreversible_state_ck",
),
sa.CheckConstraint(
"knowledge_space_revision >= 0",
name="kfs_control_space_remote_revision_ck",
),
)
if op.get_bind().dialect.name == "postgresql":
op.create_index(
"kfs_control_space_tenant_space_uq",
"knowledge_fs_control_spaces",
["tenant_id", "knowledge_space_id"],
unique=True,
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
)
else:
op.create_index(
"kfs_control_space_tenant_space_uq",
"knowledge_fs_control_spaces",
["tenant_id", "knowledge_space_id"],
unique=True,
)
op.create_index(
"kfs_control_space_state_updated_idx",
"knowledge_fs_control_spaces",
["state", "updated_at"],
)
op.create_index(
"kfs_control_space_tenant_state_updated_idx",
"knowledge_fs_control_spaces",
["tenant_id", "state", "updated_at"],
)
op.create_index(
"kfs_control_space_tenant_owner_state_idx",
"knowledge_fs_control_spaces",
["tenant_id", "owner_account_id", "state"],
)
op.create_table(
"knowledge_fs_control_space_permissions",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("account_id", models.types.StringUUID(), nullable=False),
sa.Column("role", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("granted_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
sa.UniqueConstraint(
"tenant_id",
"control_space_id",
"account_id",
name="kfs_control_space_permission_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_control_space_permission_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_control_space_permission_revision_ck"),
)
op.create_index(
"kfs_control_space_permission_account_idx",
"knowledge_fs_control_space_permissions",
["tenant_id", "account_id", "status"],
)
op.create_table(
"knowledge_fs_external_access_policies",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("service_api_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("agent_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("workflow_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("mcp_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("updated_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_external_access_policy_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_external_access_policy_revision_ck"),
)
op.create_table(
"knowledge_fs_api_credentials",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("credential_hash", sa.String(length=255), nullable=False),
sa.Column("credential_prefix", sa.String(length=32), nullable=False),
sa.Column("credential_last4", sa.String(length=4), nullable=False),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("allowed_actions", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("expires_at", sa.DateTime(), nullable=True),
sa.Column("last_used_at", sa.DateTime(), nullable=True),
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoke_reason", sa.String(length=255), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
sa.UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_api_credential_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_api_credential_revision_ck"),
)
op.create_index(
"kfs_api_credential_tenant_space_status_idx",
"knowledge_fs_api_credentials",
["tenant_id", "control_space_id", "status"],
)
op.create_index(
"kfs_api_credential_tenant_prefix_idx",
"knowledge_fs_api_credentials",
["tenant_id", "credential_prefix"],
)
op.create_table(
"app_knowledge_fs_space_joins",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("app_id", models.types.StringUUID(), nullable=False),
sa.Column("join_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
sa.UniqueConstraint(
"tenant_id",
"app_id",
"control_space_id",
"join_type",
name="app_kfs_space_join_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="app_kfs_space_join_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="app_kfs_space_join_revision_ck"),
)
op.create_index(
"app_kfs_space_join_app_status_idx",
"app_knowledge_fs_space_joins",
["tenant_id", "app_id", "status"],
)
op.create_table(
"knowledge_fs_authorization_revisions",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("membership_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("space_acl_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("external_access_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("content_policy_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("revoke_sequence", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_authorization_revision_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("membership_epoch >= 0", name="kfs_authorization_membership_epoch_ck"),
sa.CheckConstraint("space_acl_epoch >= 0", name="kfs_authorization_space_acl_epoch_ck"),
sa.CheckConstraint("external_access_epoch >= 0", name="kfs_authorization_external_access_epoch_ck"),
sa.CheckConstraint("content_policy_revision >= 0", name="kfs_authorization_content_policy_revision_ck"),
sa.CheckConstraint("revoke_sequence >= 0", name="kfs_authorization_revoke_sequence_ck"),
)
op.create_table(
"knowledge_fs_capability_issuance_audits",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("trace_id", sa.String(length=128), nullable=False),
sa.Column("jti_hash", sa.String(length=80), nullable=False),
sa.Column("claims_summary", sa.JSON(), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
sa.UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_audit_space_fk",
ondelete="RESTRICT",
),
)
op.create_index(
"kfs_capability_issuance_audit_space_created_idx",
"knowledge_fs_capability_issuance_audits",
["tenant_id", "control_space_id", "created_at"],
)
op.create_index(
"kfs_capability_issuance_audit_trace_idx",
"knowledge_fs_capability_issuance_audits",
["tenant_id", "trace_id"],
)
op.create_table(
"knowledge_fs_lifecycle_outbox",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("operation_id", models.types.StringUUID(), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("operation", sa.String(length=32), nullable=False),
sa.Column("command_payload", sa.JSON(), nullable=False),
sa.Column("expected_control_space_version", sa.BigInteger(), nullable=False),
sa.Column("expected_knowledge_space_revision", sa.BigInteger(), nullable=True),
sa.Column("command_schema_version", sa.Integer(), server_default=sa.text("1"), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'pending'"), nullable=False),
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("next_attempt_at", sa.DateTime(), nullable=True),
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
sa.Column("lease_owner", sa.String(length=255), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(), nullable=True),
sa.Column("last_error_code", sa.String(length=255), nullable=True),
sa.Column("last_error_message", models.types.LongText(), nullable=True),
sa.Column("completed_at", sa.DateTime(), nullable=True),
sa.Column("retain_until", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
sa.UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
sa.UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_lifecycle_outbox_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("command_schema_version >= 1", name="kfs_lifecycle_outbox_schema_version_ck"),
sa.CheckConstraint("expected_control_space_version >= 0", name="kfs_lifecycle_outbox_expected_version_ck"),
sa.CheckConstraint("attempt_count >= 0", name="kfs_lifecycle_outbox_attempt_count_ck"),
sa.CheckConstraint(
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
name="kfs_lifecycle_outbox_status_ck",
),
sa.CheckConstraint(
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
name="kfs_lifecycle_outbox_lease_state_ck",
),
sa.CheckConstraint(
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
name="kfs_lifecycle_outbox_terminal_state_ck",
),
)
op.create_index(
"kfs_lifecycle_outbox_dispatch_idx",
"knowledge_fs_lifecycle_outbox",
["status", "next_attempt_at", "id"],
)
op.create_index(
"kfs_lifecycle_outbox_space_created_idx",
"knowledge_fs_lifecycle_outbox",
["tenant_id", "control_space_id", "created_at"],
)
def downgrade() -> None:
op.drop_index("kfs_lifecycle_outbox_space_created_idx", table_name="knowledge_fs_lifecycle_outbox")
op.drop_index("kfs_lifecycle_outbox_dispatch_idx", table_name="knowledge_fs_lifecycle_outbox")
op.drop_table("knowledge_fs_lifecycle_outbox")
op.drop_index(
"kfs_capability_issuance_audit_trace_idx",
table_name="knowledge_fs_capability_issuance_audits",
)
op.drop_index(
"kfs_capability_issuance_audit_space_created_idx",
table_name="knowledge_fs_capability_issuance_audits",
)
op.drop_table("knowledge_fs_capability_issuance_audits")
op.drop_table("knowledge_fs_authorization_revisions")
op.drop_index("app_kfs_space_join_app_status_idx", table_name="app_knowledge_fs_space_joins")
op.drop_table("app_knowledge_fs_space_joins")
op.drop_index("kfs_api_credential_tenant_prefix_idx", table_name="knowledge_fs_api_credentials")
op.drop_index("kfs_api_credential_tenant_space_status_idx", table_name="knowledge_fs_api_credentials")
op.drop_table("knowledge_fs_api_credentials")
op.drop_table("knowledge_fs_external_access_policies")
op.drop_index(
"kfs_control_space_permission_account_idx",
table_name="knowledge_fs_control_space_permissions",
)
op.drop_table("knowledge_fs_control_space_permissions")
op.drop_index("kfs_control_space_tenant_owner_state_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_tenant_state_updated_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_state_updated_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_tenant_space_uq", table_name="knowledge_fs_control_spaces")
op.drop_table("knowledge_fs_control_spaces")
@@ -0,0 +1,205 @@
"""add KnowledgeFS workspace cutover ledger and migration evidence
Revision ID: b7f2a9d41c60
Revises: a4e7c2f91b30
Create Date: 2026-07-21 13:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "b7f2a9d41c60"
down_revision = "a4e7c2f91b30"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def _timestamps() -> tuple[sa.Column, sa.Column]:
return (
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
)
def upgrade() -> None:
op.create_table(
"knowledge_fs_workspace_cutover_ledgers",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("phase", sa.String(length=32), server_default=sa.text("'inventory'"), nullable=False),
sa.Column("source_revision_watermark", sa.JSON(), nullable=False),
sa.Column("final_revision_watermark", sa.JSON(), nullable=True),
sa.Column("applied_revision_watermark", sa.JSON(), nullable=False),
sa.Column("source_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("final_task_watermark", sa.BigInteger(), nullable=True),
sa.Column("applied_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("freeze_at", sa.DateTime(), nullable=True),
sa.Column("cutover_at", sa.DateTime(), nullable=True),
sa.Column("rolled_back_at", sa.DateTime(), nullable=True),
sa.Column("rollback_cutoff_at", sa.DateTime(), nullable=True),
sa.Column("observation_started_at", sa.DateTime(), nullable=True),
sa.Column("observation_window_ends_at", sa.DateTime(), nullable=True),
sa.Column("observation_completed_at", sa.DateTime(), nullable=True),
sa.Column("maximum_task_expires_at", sa.DateTime(), nullable=True),
sa.Column("irreversible_cleanup_at", sa.DateTime(), nullable=True),
sa.Column("product_routes_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("capability_v2_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("integrated_mode_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("legacy_acl_read_only", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("smoke_results", sa.JSON(), nullable=True),
sa.Column("legacy_dependency_report", sa.JSON(), nullable=True),
sa.Column("legacy_dependency_checked_at", sa.DateTime(), nullable=True),
sa.Column("legacy_dependency_ready", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("cas_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
sa.UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
sa.UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
sa.CheckConstraint("cas_version >= 0", name="kfs_workspace_cutover_cas_version_ck"),
sa.CheckConstraint(
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
name="kfs_workspace_cutover_task_watermark_ck",
),
sa.CheckConstraint(
"cutover_at IS NULL OR freeze_at IS NOT NULL",
name="kfs_workspace_cutover_freeze_before_cutover_ck",
),
sa.CheckConstraint(
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
name="kfs_workspace_cutover_rollback_cutoff_ck",
),
)
op.create_index(
"kfs_workspace_cutover_ledger_phase_updated_idx",
"knowledge_fs_workspace_cutover_ledgers",
["phase", "updated_at"],
)
op.create_table(
"knowledge_fs_migration_issues",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("issue_key", sa.String(length=255), nullable=False),
sa.Column("kind", sa.String(length=48), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'open'"), nullable=False),
sa.Column("resource_type", sa.String(length=64), nullable=True),
sa.Column("resource_id", sa.String(length=255), nullable=True),
sa.Column("details", sa.JSON(), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
sa.UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_issue_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_migration_issue_gate_idx",
"knowledge_fs_migration_issues",
["tenant_id", "ledger_id", "status", "kind"],
)
op.create_table(
"knowledge_fs_migration_quarantine",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("source_kind", sa.String(length=32), nullable=False),
sa.Column("source_id", sa.String(length=255), nullable=False),
sa.Column("reason_code", sa.String(length=64), nullable=False),
sa.Column("disposition", sa.String(length=32), nullable=False),
sa.Column("details", sa.JSON(), nullable=False),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"source_kind",
"source_id",
name="kfs_migration_quarantine_source_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_quarantine_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_migration_quarantine_disposition_idx",
"knowledge_fs_migration_quarantine",
["tenant_id", "ledger_id", "disposition"],
)
op.create_table(
"knowledge_fs_shadow_authorization_diffs",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("diff_key", sa.String(length=255), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
sa.Column("decision", sa.String(length=32), nullable=False),
sa.Column("reason", models.types.LongText(), nullable=False),
sa.Column("observed_revision", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
sa.UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_diff_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_shadow_authorization_diff_gate_idx",
"knowledge_fs_shadow_authorization_diffs",
["tenant_id", "ledger_id", "status", "decision"],
)
def downgrade() -> None:
op.drop_index(
"kfs_shadow_authorization_diff_gate_idx",
table_name="knowledge_fs_shadow_authorization_diffs",
)
op.drop_table("knowledge_fs_shadow_authorization_diffs")
op.drop_index(
"kfs_migration_quarantine_disposition_idx",
table_name="knowledge_fs_migration_quarantine",
)
op.drop_table("knowledge_fs_migration_quarantine")
op.drop_index("kfs_migration_issue_gate_idx", table_name="knowledge_fs_migration_issues")
op.drop_table("knowledge_fs_migration_issues")
op.drop_index(
"kfs_workspace_cutover_ledger_phase_updated_idx",
table_name="knowledge_fs_workspace_cutover_ledgers",
)
op.drop_table("knowledge_fs_workspace_cutover_ledgers")
@@ -0,0 +1,365 @@
"""add guarded cleanup, cutover evidence, and capability issuance reservations
Revision ID: c8e31b7d52a4
Revises: b7f2a9d41c60
Create Date: 2026-07-21 14:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "c8e31b7d52a4"
down_revision = "b7f2a9d41c60"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def upgrade() -> None:
op.create_table(
"knowledge_fs_cleanup_authorizations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("request_id", models.types.StringUUID(), nullable=False),
sa.Column("plan_schema_version", sa.String(length=32), nullable=False),
sa.Column("plan_digest", sa.String(length=71), nullable=False),
sa.Column("targets", sa.JSON(), nullable=False),
sa.Column("readiness_evidence", sa.JSON(), nullable=False),
sa.Column("requested_by_account_id", models.types.StringUUID(), nullable=False),
sa.Column("requested_at", sa.DateTime(), nullable=False),
sa.Column("readiness_ledger_cas_version", sa.BigInteger(), nullable=False),
sa.Column("status", sa.String(length=16), server_default=sa.text("'requested'"), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("approval_expires_at", sa.DateTime(), nullable=True),
sa.Column("approved_ledger_cas_version", sa.BigInteger(), nullable=True),
sa.Column("started_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("started_ledger_cas_version", sa.BigInteger(), nullable=True),
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"request_id",
name="kfs_cleanup_authorization_request_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_cleanup_authorization_ledger_fk",
),
sa.CheckConstraint("row_version >= 0", name="kfs_cleanup_authorization_version_ck"),
sa.CheckConstraint(
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
name="kfs_cleanup_authorization_ledger_versions_ck",
),
sa.CheckConstraint(
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL)",
name="kfs_cleanup_authorization_status_fields_ck",
),
sa.CheckConstraint(
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
name="kfs_cleanup_authorization_approval_window_ck",
),
)
op.create_index(
"kfs_cleanup_authorization_status_idx",
"knowledge_fs_cleanup_authorizations",
["tenant_id", "status", "updated_at"],
)
op.create_table(
"knowledge_fs_capability_issuance_reservations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("grant_id", models.types.StringUUID(), nullable=False),
sa.Column("trace_id", sa.String(length=128), nullable=False),
sa.Column("subject", sa.String(length=255), nullable=False),
sa.Column("caller_kind", sa.String(length=32), nullable=False),
sa.Column("request_summary", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=16), server_default=sa.text("'reserved'"), nullable=False),
sa.Column("issued_at", sa.DateTime(), nullable=True),
sa.Column("token_expires_at", sa.DateTime(), nullable=True),
sa.Column("failed_at", sa.DateTime(), nullable=True),
sa.Column("failure_code", sa.String(length=128), nullable=True),
sa.Column("cleanup_after", sa.DateTime(), nullable=True),
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
sa.UniqueConstraint(
"tenant_id",
"grant_id",
name="kfs_capability_issuance_reservation_grant_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_reservation_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint(
"row_version >= 0",
name="kfs_capability_issuance_reservation_version_ck",
),
sa.CheckConstraint(
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
name="kfs_capability_issuance_reservation_status_fields_ck",
),
sa.CheckConstraint(
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
name="kfs_capability_issuance_reservation_cleanup_window_ck",
),
)
op.create_index(
"kfs_capability_issuance_reservation_subject_idx",
"knowledge_fs_capability_issuance_reservations",
["tenant_id", "control_space_id", "subject", "caller_kind"],
)
op.create_index(
"kfs_capability_issuance_reservation_trace_idx",
"knowledge_fs_capability_issuance_reservations",
["tenant_id", "trace_id"],
)
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
batch_op.add_column(sa.Column("resolved_by_operator", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
batch_op.create_check_constraint(
"kfs_migration_quarantine_version_ck",
"row_version >= 0",
)
batch_op.create_check_constraint(
"kfs_migration_quarantine_resolution_fields_ck",
"(disposition = 'resolved' "
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
"(disposition <> 'resolved' "
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
"AND evidence IS NULL AND resolved_at IS NULL)",
)
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.add_column(sa.Column("shadow_started_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_evidence_digest", sa.String(length=71), nullable=True))
batch_op.add_column(
sa.Column("shadow_observation_count", sa.BigInteger(), server_default=sa.text("0"), nullable=False)
)
batch_op.add_column(sa.Column("shadow_window_started_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_window_ended_at", sa.DateTime(), nullable=True))
batch_op.add_column(
sa.Column("shadow_traffic_zero", sa.Boolean(), server_default=sa.text("false"), nullable=False)
)
batch_op.add_column(sa.Column("shadow_traffic_zero_evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("shadow_latest_observed_revision", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("shadow_producer", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_by_operator", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_id", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("remote_activation_revision", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("remote_activation_control_space_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_activated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_updated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_acknowledged_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_applied", sa.Boolean(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_replayed", sa.Boolean(), nullable=True))
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_count_ck",
"shadow_observation_count >= 0",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_window_ck",
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_completion_fields_ck",
"(shadow_completed_at IS NULL "
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
"AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_activation_fields_ck",
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
"AND remote_activation_replayed IS NULL) OR "
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
"AND remote_activation_acknowledged_at IS NOT NULL "
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_activation_time_ck",
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
)
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.add_column(sa.Column("current_evidence_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("last_observed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
batch_op.create_check_constraint(
"kfs_shadow_authorization_diff_version_ck",
"row_version >= 0",
)
op.execute(
sa.text(
"UPDATE knowledge_fs_shadow_authorization_diffs "
"SET current_evidence_digest = 'sha256:legacy-unversioned', last_observed_at = created_at "
"WHERE current_evidence_digest IS NULL OR last_observed_at IS NULL"
)
)
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.alter_column("current_evidence_digest", existing_type=sa.String(length=71), nullable=False)
batch_op.alter_column("last_observed_at", existing_type=sa.DateTime(), nullable=False)
op.create_table(
"knowledge_fs_shadow_authorization_observations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("diff_key", sa.String(length=255), nullable=False),
sa.Column("producer", sa.String(length=255), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
sa.Column("decision", sa.String(length=32), nullable=False),
sa.Column("reason", models.types.LongText(), nullable=False),
sa.Column("observed_revision", sa.JSON(), nullable=False),
sa.Column("observed_at", sa.DateTime(), nullable=False),
sa.Column("evidence_digest", sa.String(length=71), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"diff_key",
"evidence_digest",
name="kfs_shadow_authorization_observation_evidence_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_observation_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_shadow_authorization_observation_window_idx",
"knowledge_fs_shadow_authorization_observations",
["tenant_id", "ledger_id", "observed_at"],
)
def downgrade() -> None:
op.drop_index(
"kfs_shadow_authorization_observation_window_idx",
table_name="knowledge_fs_shadow_authorization_observations",
)
op.drop_table("knowledge_fs_shadow_authorization_observations")
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.drop_constraint("kfs_shadow_authorization_diff_version_ck", type_="check")
batch_op.drop_column("row_version")
batch_op.drop_column("last_observed_at")
batch_op.drop_column("current_evidence_digest")
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_time_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_fields_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_completion_fields_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_window_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_count_ck", type_="check")
batch_op.drop_column("remote_activation_replayed")
batch_op.drop_column("remote_activation_applied")
batch_op.drop_column("remote_activation_acknowledged_at")
batch_op.drop_column("remote_activation_updated_at")
batch_op.drop_column("remote_activation_activated_at")
batch_op.drop_column("remote_activation_control_space_id")
batch_op.drop_column("remote_activation_digest")
batch_op.drop_column("remote_activation_revision")
batch_op.drop_column("remote_activation_id")
batch_op.drop_column("shadow_completed_by_account_id")
batch_op.drop_column("shadow_completed_by_operator")
batch_op.drop_column("shadow_producer")
batch_op.drop_column("shadow_latest_observed_revision")
batch_op.drop_column("shadow_traffic_zero_evidence")
batch_op.drop_column("shadow_traffic_zero")
batch_op.drop_column("shadow_window_ended_at")
batch_op.drop_column("shadow_window_started_at")
batch_op.drop_column("shadow_observation_count")
batch_op.drop_column("shadow_evidence_digest")
batch_op.drop_column("shadow_completed_at")
batch_op.drop_column("shadow_started_at")
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
batch_op.drop_constraint(
"kfs_migration_quarantine_resolution_fields_ck",
type_="check",
)
batch_op.drop_constraint("kfs_migration_quarantine_version_ck", type_="check")
batch_op.drop_column("row_version")
batch_op.drop_column("evidence")
batch_op.drop_column("resolved_by_account_id")
batch_op.drop_column("resolved_by_operator")
op.drop_index(
"kfs_capability_issuance_reservation_trace_idx",
table_name="knowledge_fs_capability_issuance_reservations",
)
op.drop_index(
"kfs_capability_issuance_reservation_subject_idx",
table_name="knowledge_fs_capability_issuance_reservations",
)
op.drop_table("knowledge_fs_capability_issuance_reservations")
op.drop_index(
"kfs_cleanup_authorization_status_idx",
table_name="knowledge_fs_cleanup_authorizations",
)
op.drop_table("knowledge_fs_cleanup_authorizations")
@@ -0,0 +1,66 @@
"""add durable KnowledgeFS remote freeze evidence
Revision ID: d4f6e8a1c305
Revises: c8e31b7d52a4
Create Date: 2026-07-21 15:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "d4f6e8a1c305"
down_revision = "c8e31b7d52a4"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.add_column(sa.Column("remote_freeze_id", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_revision", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_task_watermark", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_control_space_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_frozen_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_updated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_acknowledged_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_applied", sa.Boolean(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_replayed", sa.Boolean(), nullable=True))
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_freeze_fields_ck",
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_freeze_time_ck",
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
)
def downgrade() -> None:
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_time_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_fields_ck", type_="check")
batch_op.drop_column("remote_freeze_replayed")
batch_op.drop_column("remote_freeze_applied")
batch_op.drop_column("remote_freeze_acknowledged_at")
batch_op.drop_column("remote_freeze_updated_at")
batch_op.drop_column("remote_freeze_frozen_at")
batch_op.drop_column("remote_freeze_control_space_id")
batch_op.drop_column("remote_freeze_task_watermark")
batch_op.drop_column("remote_freeze_digest")
batch_op.drop_column("remote_freeze_revision")
batch_op.drop_column("remote_freeze_id")
@@ -0,0 +1,113 @@
"""add global KnowledgeFS cleanup completion evidence
Revision ID: e5a7c9b2d416
Revises: d4f6e8a1c305
Create Date: 2026-07-21 16:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "e5a7c9b2d416"
down_revision = "d4f6e8a1c305"
branch_labels = None
depends_on = None
_LEDGER_VERSIONS_CHECK = (
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)"
)
_STATUS_FIELDS_CHECK = (
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'completed' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)"
)
def upgrade() -> None:
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
batch_op.add_column(sa.Column("completed_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("completed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("completion_evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("completed_ledger_cas_version", sa.BigInteger(), nullable=True))
batch_op.create_check_constraint(
"kfs_cleanup_authorization_ledger_versions_ck",
_LEDGER_VERSIONS_CHECK,
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_status_fields_ck",
_STATUS_FIELDS_CHECK,
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_completion_time_ck",
"completed_at IS NULL OR completed_at >= started_at",
)
def downgrade() -> None:
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
batch_op.drop_constraint("kfs_cleanup_authorization_completion_time_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
batch_op.create_check_constraint(
"kfs_cleanup_authorization_ledger_versions_ck",
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_status_fields_ck",
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL)",
)
batch_op.drop_column("completed_ledger_cas_version")
batch_op.drop_column("completion_evidence")
batch_op.drop_column("completed_at")
batch_op.drop_column("completed_by_account_id")
+72
View File
@@ -66,6 +66,45 @@ from .enums import (
)
from .execution_extra_content import ExecutionExtraContent, HumanInputContent
from .human_input import HumanInputForm, HumanInputFormUploadFile, HumanInputFormUploadToken
from .knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSApiCredential,
KnowledgeFSApiCredentialStatus,
KnowledgeFSAppSpaceJoinStatus,
KnowledgeFSAppSpaceJoinType,
KnowledgeFSAuthorizationRevision,
KnowledgeFSCapabilityIssuanceAudit,
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSCapabilityIssuanceReservationStatus,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpacePermissionStatus,
KnowledgeFSControlSpaceState,
KnowledgeFSControlSpaceVisibility,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
)
from .knowledge_fs_cleanup import (
KnowledgeFSCleanupAuthorization,
KnowledgeFSCleanupAuthorizationStatus,
KnowledgeFSCleanupTarget,
)
from .knowledge_fs_cutover import (
KnowledgeFSMigrationIssue,
KnowledgeFSMigrationIssueKind,
KnowledgeFSMigrationIssueStatus,
KnowledgeFSMigrationQuarantine,
KnowledgeFSMigrationQuarantineDisposition,
KnowledgeFSMigrationQuarantineKind,
KnowledgeFSShadowAuthorizationDecision,
KnowledgeFSShadowAuthorizationDiff,
KnowledgeFSShadowAuthorizationObservation,
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
)
from .model import (
AccountTrialAppRecord,
ApiRequest,
@@ -180,6 +219,7 @@ __all__ = [
"AppAnnotationHitHistory",
"AppAnnotationSetting",
"AppDatasetJoin",
"AppKnowledgeFSSpaceJoin",
"AppMCPServer",
"AppMode",
"AppModelConfig",
@@ -224,6 +264,38 @@ __all__ = [
"IconType",
"InstalledApp",
"InvitationCode",
"KnowledgeFSApiCredential",
"KnowledgeFSApiCredentialStatus",
"KnowledgeFSAppSpaceJoinStatus",
"KnowledgeFSAppSpaceJoinType",
"KnowledgeFSAuthorizationRevision",
"KnowledgeFSCapabilityIssuanceAudit",
"KnowledgeFSCapabilityIssuanceReservation",
"KnowledgeFSCapabilityIssuanceReservationStatus",
"KnowledgeFSCleanupAuthorization",
"KnowledgeFSCleanupAuthorizationStatus",
"KnowledgeFSCleanupTarget",
"KnowledgeFSControlSpace",
"KnowledgeFSControlSpacePermission",
"KnowledgeFSControlSpacePermissionRole",
"KnowledgeFSControlSpacePermissionStatus",
"KnowledgeFSControlSpaceState",
"KnowledgeFSControlSpaceVisibility",
"KnowledgeFSExternalAccessPolicy",
"KnowledgeFSLifecycleOperation",
"KnowledgeFSLifecycleOutbox",
"KnowledgeFSLifecycleOutboxStatus",
"KnowledgeFSMigrationIssue",
"KnowledgeFSMigrationIssueKind",
"KnowledgeFSMigrationIssueStatus",
"KnowledgeFSMigrationQuarantine",
"KnowledgeFSMigrationQuarantineDisposition",
"KnowledgeFSMigrationQuarantineKind",
"KnowledgeFSShadowAuthorizationDecision",
"KnowledgeFSShadowAuthorizationDiff",
"KnowledgeFSShadowAuthorizationObservation",
"KnowledgeFSWorkspaceCutoverLedger",
"KnowledgeFSWorkspaceCutoverPhase",
"LoadBalancingModelConfig",
"Message",
"MessageAgentThought",
+713
View File
@@ -0,0 +1,713 @@
"""Dify-owned control-plane persistence for the KnowledgeFS product.
These models intentionally have no dependency on Dify Dataset or Document
models. KnowledgeFS data-plane objects remain owned by KnowledgeFS; these rows
only describe Dify authorization, registration, and durable lifecycle intent.
"""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import NotRequired, TypedDict
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, LongText, StringUUID
KnowledgeFSAllowedActions = list[str]
class KnowledgeFSCommandEnvelope(TypedDict):
schema_version: int
idempotency_key: str
expected_revision: int
class KnowledgeFSModelSelectionIntentPayload(TypedDict):
pluginId: str
provider: str
model: str
class KnowledgeFSRerankIntentPayload(TypedDict):
enabled: bool
model: NotRequired[KnowledgeFSModelSelectionIntentPayload]
class KnowledgeFSScoreThresholdIntentPayload(TypedDict):
enabled: bool
stage: str
value: NotRequired[float]
class KnowledgeFSRetrievalProfileIntentPayload(TypedDict):
defaultMode: str
reasoningModel: KnowledgeFSModelSelectionIntentPayload
rerank: KnowledgeFSRerankIntentPayload
scoreThreshold: KnowledgeFSScoreThresholdIntentPayload
topK: int
class KnowledgeFSProvisionCommandPayload(KnowledgeFSCommandEnvelope):
provisioning_key: str
name: str
icon: str | None
description: str | None
slug: str
model_intent: KnowledgeFSModelSelectionIntentPayload
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
class KnowledgeFSMetadataUpdateCommandPayload(KnowledgeFSCommandEnvelope):
metadata: dict[str, object]
class KnowledgeFSDeleteCommandPayload(KnowledgeFSCommandEnvelope):
knowledge_space_id: str | None
provisioning_key: str
class KnowledgeFSRevokeCommandPayload(KnowledgeFSCommandEnvelope):
event_id: str
grant_id: str
knowledge_space_id: str
principal: str
reason_code: str
revoke_sequence: int
class KnowledgeFSRepairCommandPayload(KnowledgeFSCommandEnvelope):
repair_reason: str
KnowledgeFSCommandPayload = (
KnowledgeFSProvisionCommandPayload
| KnowledgeFSMetadataUpdateCommandPayload
| KnowledgeFSDeleteCommandPayload
| KnowledgeFSRevokeCommandPayload
| KnowledgeFSRepairCommandPayload
)
class KnowledgeFSCapabilityAuthzRevisionSummary(TypedDict):
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
credential_revision: int | None
class KnowledgeFSCapabilityClaimsSummary(TypedDict):
action: str
actor: str
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
caller_kind: str
content_policy_revision: int
content_scope_ids: list[str]
control_space_id: str
expires_at: str
grant_id: str
issued_at: str
namespace_id: str
operation_id: str
resource_id: str
resource_parent_id: str | None
resource_type: str
subject: str
class KnowledgeFSCapabilityReservationSummary(TypedDict):
"""Sanitized, exact input bound before a capability can be signed."""
action: str
actor: str
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
caller_kind: str
content_policy_revision: int
content_scope_ids: list[str]
control_space_id: str
grant_id: str
namespace_id: str
operation_id: str
resource_id: str
resource_parent_id: str | None
resource_type: str
subject: str
trace_id: str
class KnowledgeFSControlSpaceVisibility(StrEnum):
ONLY_ME = "only_me"
ALL_TEAM_MEMBERS = "all_team_members"
PARTIAL_MEMBERS = "partial_members"
class KnowledgeFSControlSpaceState(StrEnum):
PROVISIONING = "provisioning"
ACTIVE = "active"
DELETING = "deleting"
DELETED = "deleted"
ERROR = "error"
class KnowledgeFSControlSpacePermissionRole(StrEnum):
OWNER = "owner"
EDITOR = "editor"
VIEWER = "viewer"
class KnowledgeFSControlSpacePermissionStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
class KnowledgeFSApiCredentialStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
EXPIRED = "expired"
class KnowledgeFSAppSpaceJoinType(StrEnum):
AGENT = "agent"
WORKFLOW = "workflow"
class KnowledgeFSAppSpaceJoinStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
class KnowledgeFSCapabilityIssuanceReservationStatus(StrEnum):
RESERVED = "reserved"
ISSUED = "issued"
FAILED = "failed"
class KnowledgeFSLifecycleOperation(StrEnum):
PROVISION = "provision"
METADATA_UPDATE = "metadata_update"
DELETE = "delete"
REVOKE = "revoke"
REPAIR = "repair"
class KnowledgeFSLifecycleOutboxStatus(StrEnum):
PENDING = "pending"
PROCESSING = "processing"
SUCCEEDED = "succeeded"
RETRY = "retry"
DEAD_LETTER = "dead_letter"
class KnowledgeFSControlSpace(DefaultFieldsDCMixin, TypeBase):
"""Dify product resource registered to at most one KnowledgeFS Space."""
__tablename__ = "knowledge_fs_control_spaces"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id"],
["tenants.id"],
name="kfs_control_space_workspace_fk",
ondelete="RESTRICT",
),
Index(
"kfs_control_space_tenant_space_uq",
"tenant_id",
"knowledge_space_id",
unique=True,
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
),
Index("kfs_control_space_state_updated_idx", "state", "updated_at"),
Index("kfs_control_space_tenant_state_updated_idx", "tenant_id", "state", "updated_at"),
Index("kfs_control_space_tenant_owner_state_idx", "tenant_id", "owner_account_id", "state"),
sa.CheckConstraint(
"resource_version >= 0",
name=sa.schema.conv("kfs_control_space_resource_version_ck"),
),
sa.CheckConstraint("attempt_count >= 0", name=sa.schema.conv("kfs_control_space_attempt_count_ck")),
sa.CheckConstraint(
"state != 'active' OR knowledge_space_id IS NOT NULL",
name=sa.schema.conv("kfs_control_space_active_registration_ck"),
),
sa.CheckConstraint(
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
name=sa.schema.conv("kfs_control_space_irreversible_state_ck"),
),
sa.CheckConstraint(
"knowledge_space_revision >= 0",
name=sa.schema.conv("kfs_control_space_remote_revision_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
owner_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provisioning_key: Mapped[str] = mapped_column(String(255), nullable=False)
knowledge_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
knowledge_space_revision: Mapped[int] = mapped_column(
sa.BigInteger,
nullable=False,
server_default=sa.text("0"),
default=0,
)
visibility: Mapped[KnowledgeFSControlSpaceVisibility] = mapped_column(
EnumText(KnowledgeFSControlSpaceVisibility, length=32),
nullable=False,
server_default=sa.text("'only_me'"),
default=KnowledgeFSControlSpaceVisibility.ONLY_ME,
)
lifecycle_operation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
state: Mapped[KnowledgeFSControlSpaceState] = mapped_column(
EnumText(KnowledgeFSControlSpaceState, length=32),
nullable=False,
server_default=sa.text("'provisioning'"),
default=KnowledgeFSControlSpaceState.PROVISIONING,
)
resource_version: Mapped[int] = mapped_column(
sa.BigInteger,
nullable=False,
server_default=sa.text("0"),
default=0,
)
attempt_count: Mapped[int] = mapped_column(
sa.Integer,
nullable=False,
server_default=sa.text("0"),
default=0,
)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
last_synced_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
deletion_irreversible_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
class KnowledgeFSControlSpacePermission(DefaultFieldsDCMixin, TypeBase):
"""Account-level product authorization for one control-space."""
__tablename__ = "knowledge_fs_control_space_permissions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
UniqueConstraint(
"tenant_id",
"control_space_id",
"account_id",
name="kfs_control_space_permission_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_control_space_permission_space_fk",
ondelete="RESTRICT",
),
Index("kfs_control_space_permission_account_idx", "tenant_id", "account_id", "status"),
sa.CheckConstraint(
"revision >= 0",
name=sa.schema.conv("kfs_control_space_permission_revision_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
role: Mapped[KnowledgeFSControlSpacePermissionRole] = mapped_column(
EnumText(KnowledgeFSControlSpacePermissionRole, length=32), nullable=False
)
status: Mapped[KnowledgeFSControlSpacePermissionStatus] = mapped_column(
EnumText(KnowledgeFSControlSpacePermissionStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSControlSpacePermissionStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
granted_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSExternalAccessPolicy(DefaultFieldsDCMixin, TypeBase):
"""Fail-closed caller-channel policy for a control-space."""
__tablename__ = "knowledge_fs_external_access_policies"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_external_access_policy_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_external_access_policy_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
service_api_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
agent_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
workflow_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
mcp_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
updated_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSApiCredential(DefaultFieldsDCMixin, TypeBase):
"""Hashed, resource-bound KnowledgeFS API credential metadata."""
__tablename__ = "knowledge_fs_api_credentials"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_api_credential_space_fk",
ondelete="RESTRICT",
),
Index("kfs_api_credential_tenant_space_status_idx", "tenant_id", "control_space_id", "status"),
Index("kfs_api_credential_tenant_prefix_idx", "tenant_id", "credential_prefix"),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_api_credential_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
credential_hash: Mapped[str] = mapped_column(String(255), nullable=False)
credential_prefix: Mapped[str] = mapped_column(String(32), nullable=False)
credential_last4: Mapped[str] = mapped_column(String(4), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
allowed_actions: Mapped[KnowledgeFSAllowedActions] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSApiCredentialStatus] = mapped_column(
EnumText(KnowledgeFSApiCredentialStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSApiCredentialStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoke_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
class AppKnowledgeFSSpaceJoin(DefaultFieldsDCMixin, TypeBase):
"""Explicit Agent or Workflow app authorization for a control-space."""
__tablename__ = "app_knowledge_fs_space_joins"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
UniqueConstraint(
"tenant_id",
"app_id",
"control_space_id",
"join_type",
name="app_kfs_space_join_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="app_kfs_space_join_space_fk",
ondelete="RESTRICT",
),
Index("app_kfs_space_join_app_status_idx", "tenant_id", "app_id", "status"),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("app_kfs_space_join_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
join_type: Mapped[KnowledgeFSAppSpaceJoinType] = mapped_column(
EnumText(KnowledgeFSAppSpaceJoinType, length=32), nullable=False
)
status: Mapped[KnowledgeFSAppSpaceJoinStatus] = mapped_column(
EnumText(KnowledgeFSAppSpaceJoinStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSAppSpaceJoinStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSAuthorizationRevision(DefaultFieldsDCMixin, TypeBase):
"""Monotonic authorization epochs included in future Capability claims."""
__tablename__ = "knowledge_fs_authorization_revisions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_authorization_revision_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint(
"membership_epoch >= 0",
name=sa.schema.conv("kfs_authorization_membership_epoch_ck"),
),
sa.CheckConstraint(
"space_acl_epoch >= 0",
name=sa.schema.conv("kfs_authorization_space_acl_epoch_ck"),
),
sa.CheckConstraint(
"external_access_epoch >= 0",
name=sa.schema.conv("kfs_authorization_external_access_epoch_ck"),
),
sa.CheckConstraint(
"content_policy_revision >= 0",
name=sa.schema.conv("kfs_authorization_content_policy_revision_ck"),
),
sa.CheckConstraint(
"revoke_sequence >= 0",
name=sa.schema.conv("kfs_authorization_revoke_sequence_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
membership_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
space_acl_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
external_access_epoch: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
content_policy_revision: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
revoke_sequence: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSCapabilityIssuanceAudit(DefaultFieldsDCMixin, TypeBase):
"""Sanitized durable evidence for a Capability v2 issuance."""
__tablename__ = "knowledge_fs_capability_issuance_audits"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_audit_space_fk",
ondelete="RESTRICT",
),
Index(
"kfs_capability_issuance_audit_space_created_idx",
"tenant_id",
"control_space_id",
"created_at",
),
Index("kfs_capability_issuance_audit_trace_idx", "tenant_id", "trace_id"),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
jti_hash: Mapped[str] = mapped_column(String(80), nullable=False)
claims_summary: Mapped[KnowledgeFSCapabilityClaimsSummary] = mapped_column(sa.JSON, nullable=False)
class KnowledgeFSCapabilityIssuanceReservation(DefaultFieldsDCMixin, TypeBase):
"""Durable authorization fence written before signing starts.
The summary intentionally excludes bearer tokens, signing material, and raw JTI values.
Revocation producers scan these rows as well as completed issuance audits, closing the
authorization-to-signing race without holding a database transaction across signing.
"""
__tablename__ = "knowledge_fs_capability_issuance_reservations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
UniqueConstraint(
"tenant_id",
"grant_id",
name="kfs_capability_issuance_reservation_grant_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_reservation_space_fk",
ondelete="RESTRICT",
),
Index(
"kfs_capability_issuance_reservation_subject_idx",
"tenant_id",
"control_space_id",
"subject",
"caller_kind",
),
Index(
"kfs_capability_issuance_reservation_trace_idx",
"tenant_id",
"trace_id",
),
sa.CheckConstraint(
"row_version >= 0",
name=sa.schema.conv("kfs_capability_issuance_reservation_version_ck"),
),
sa.CheckConstraint(
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
name=sa.schema.conv("kfs_capability_issuance_reservation_status_fields_ck"),
),
sa.CheckConstraint(
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
name=sa.schema.conv("kfs_capability_issuance_reservation_cleanup_window_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
grant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
subject: Mapped[str] = mapped_column(String(255), nullable=False)
caller_kind: Mapped[str] = mapped_column(String(32), nullable=False)
request_summary: Mapped[KnowledgeFSCapabilityReservationSummary] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSCapabilityIssuanceReservationStatus] = mapped_column(
EnumText(KnowledgeFSCapabilityIssuanceReservationStatus, length=16),
nullable=False,
server_default=sa.text("'reserved'"),
default=KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
)
issued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
token_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
failed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
failure_code: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
cleanup_after: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSLifecycleOutbox(DefaultFieldsDCMixin, TypeBase):
"""Durable command snapshot; it is never a product read-model source.
Provision payload writers must snapshot name, icon, description, slug,
model/profile intent, schema version, idempotency key, and expected
revision before enabling the later P1B dispatcher.
"""
__tablename__ = "knowledge_fs_lifecycle_outbox"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_lifecycle_outbox_space_fk",
ondelete="RESTRICT",
),
Index("kfs_lifecycle_outbox_dispatch_idx", "status", "next_attempt_at", "id"),
Index("kfs_lifecycle_outbox_space_created_idx", "tenant_id", "control_space_id", "created_at"),
sa.CheckConstraint(
"command_schema_version >= 1",
name=sa.schema.conv("kfs_lifecycle_outbox_schema_version_ck"),
),
sa.CheckConstraint(
"expected_control_space_version >= 0",
name=sa.schema.conv("kfs_lifecycle_outbox_expected_version_ck"),
),
sa.CheckConstraint(
"attempt_count >= 0",
name=sa.schema.conv("kfs_lifecycle_outbox_attempt_count_ck"),
),
sa.CheckConstraint(
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
name=sa.schema.conv("kfs_lifecycle_outbox_status_ck"),
),
sa.CheckConstraint(
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
name=sa.schema.conv("kfs_lifecycle_outbox_lease_state_ck"),
),
sa.CheckConstraint(
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
name=sa.schema.conv("kfs_lifecycle_outbox_terminal_state_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
operation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
operation: Mapped[KnowledgeFSLifecycleOperation] = mapped_column(
EnumText(KnowledgeFSLifecycleOperation, length=32), nullable=False
)
command_payload: Mapped[KnowledgeFSCommandPayload] = mapped_column(sa.JSON, nullable=False)
expected_control_space_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
expected_knowledge_space_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
command_schema_version: Mapped[int] = mapped_column(
sa.Integer, nullable=False, server_default=sa.text("1"), default=1
)
status: Mapped[KnowledgeFSLifecycleOutboxStatus] = mapped_column(
EnumText(KnowledgeFSLifecycleOutboxStatus, length=32),
nullable=False,
server_default=sa.text("'pending'"),
default=KnowledgeFSLifecycleOutboxStatus.PENDING,
)
attempt_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"), default=0)
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
lease_owner: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
retain_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
__all__ = [
"AppKnowledgeFSSpaceJoin",
"KnowledgeFSAllowedActions",
"KnowledgeFSApiCredential",
"KnowledgeFSApiCredentialStatus",
"KnowledgeFSAppSpaceJoinStatus",
"KnowledgeFSAppSpaceJoinType",
"KnowledgeFSAuthorizationRevision",
"KnowledgeFSCapabilityAuthzRevisionSummary",
"KnowledgeFSCapabilityClaimsSummary",
"KnowledgeFSCapabilityIssuanceAudit",
"KnowledgeFSCapabilityIssuanceReservation",
"KnowledgeFSCapabilityIssuanceReservationStatus",
"KnowledgeFSCapabilityReservationSummary",
"KnowledgeFSCommandEnvelope",
"KnowledgeFSCommandPayload",
"KnowledgeFSControlSpace",
"KnowledgeFSControlSpacePermission",
"KnowledgeFSControlSpacePermissionRole",
"KnowledgeFSControlSpacePermissionStatus",
"KnowledgeFSControlSpaceState",
"KnowledgeFSControlSpaceVisibility",
"KnowledgeFSDeleteCommandPayload",
"KnowledgeFSExternalAccessPolicy",
"KnowledgeFSLifecycleOperation",
"KnowledgeFSLifecycleOutbox",
"KnowledgeFSLifecycleOutboxStatus",
"KnowledgeFSMetadataUpdateCommandPayload",
"KnowledgeFSModelSelectionIntentPayload",
"KnowledgeFSProvisionCommandPayload",
"KnowledgeFSRepairCommandPayload",
"KnowledgeFSRerankIntentPayload",
"KnowledgeFSRetrievalProfileIntentPayload",
"KnowledgeFSRevokeCommandPayload",
"KnowledgeFSScoreThresholdIntentPayload",
]
+130
View File
@@ -0,0 +1,130 @@
"""Persistent authorization for the irreversible P9 KnowledgeFS cleanup window."""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, StringUUID
class KnowledgeFSCleanupAuthorizationStatus(StrEnum):
REQUESTED = "requested"
APPROVED = "approved"
STARTED = "started"
COMPLETED = "completed"
class KnowledgeFSCleanupTarget(StrEnum):
LEGACY_SNAPSHOT_FOREIGN_KEYS = "legacy_snapshot_foreign_keys"
LEGACY_ACL_ROUTES = "legacy_acl_routes"
LEGACY_ACL_SCHEMA = "legacy_acl_schema"
LEGACY_API_KEY_SCHEMA = "legacy_api_key_schema"
LEGACY_V1_AUTH = "legacy_v1_auth"
RAW_LIST_CREATE_PROXY = "raw_list_create_proxy"
class KnowledgeFSCleanupAuthorization(DefaultFieldsDCMixin, TypeBase):
"""Four-eyes approval and immutable evidence for one cleanup start fence."""
__tablename__ = "knowledge_fs_cleanup_authorizations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "request_id", name="kfs_cleanup_authorization_request_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_cleanup_authorization_ledger_fk",
),
Index("kfs_cleanup_authorization_status_idx", "tenant_id", "status", "updated_at"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_cleanup_authorization_version_ck")),
sa.CheckConstraint(
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)",
name=sa.schema.conv("kfs_cleanup_authorization_ledger_versions_ck"),
),
sa.CheckConstraint(
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'completed' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)",
name=sa.schema.conv("kfs_cleanup_authorization_status_fields_ck"),
),
sa.CheckConstraint(
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
name=sa.schema.conv("kfs_cleanup_authorization_approval_window_ck"),
),
sa.CheckConstraint(
"completed_at IS NULL OR completed_at >= started_at",
name=sa.schema.conv("kfs_cleanup_authorization_completion_time_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
request_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
plan_schema_version: Mapped[str] = mapped_column(String(32), nullable=False)
plan_digest: Mapped[str] = mapped_column(String(71), nullable=False)
targets: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False)
readiness_evidence: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False)
requested_by_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
requested_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
readiness_ledger_cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
status: Mapped[KnowledgeFSCleanupAuthorizationStatus] = mapped_column(
EnumText(KnowledgeFSCleanupAuthorizationStatus, length=16),
nullable=False,
server_default=sa.text("'requested'"),
default=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
approval_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
approved_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
started_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
started_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
completion_evidence: Mapped[dict[str, object] | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
completed_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
__all__ = [
"KnowledgeFSCleanupAuthorization",
"KnowledgeFSCleanupAuthorizationStatus",
"KnowledgeFSCleanupTarget",
]
+507
View File
@@ -0,0 +1,507 @@
"""Workspace-scoped KnowledgeFS migration and cutover persistence.
The ledger is Dify control-plane state only. It deliberately stores opaque
KnowledgeFS identifiers and authorization watermarks without foreign keys or
queries to legacy product tables. Cutover feature switches live on one
row so the repository can change them with a single CAS statement. Quarantine
remediation is separately CAS-versioned and retains immutable operator evidence.
"""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import TypedDict
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, LongText, StringUUID
class KnowledgeFSCutoverRevisionWatermark(TypedDict):
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
content_policy_revision: int
class KnowledgeFSCutoverSmokeChecks(TypedDict):
authorization: bool
list_spaces: bool
create_space: bool
query: bool
upload: bool
stream: bool
deletion: bool
class KnowledgeFSCutoverSmokeEvidenceReferences(TypedDict):
authorization: str
list_spaces: str
create_space: str
query: str
upload: str
stream: str
deletion: str
class KnowledgeFSCutoverSmokeResults(TypedDict):
schema_version: str
tenant_id: str
environment: str
operator: str
operator_account_id: str
observed_at: str
checks: KnowledgeFSCutoverSmokeChecks
evidence_references: KnowledgeFSCutoverSmokeEvidenceReferences
def knowledge_fs_cutover_smoke_results_passed(value: KnowledgeFSCutoverSmokeResults | None) -> bool:
"""Reject incomplete or pre-production JSON evidence instead of treating metadata as booleans."""
if not isinstance(value, dict):
return False
checks = value.get("checks")
references = value.get("evidence_references")
expected = {
"authorization",
"list_spaces",
"create_space",
"query",
"upload",
"stream",
"deletion",
}
return bool(
value.get("schema_version") == "knowledge-fs-p8-cutover-smoke/v1"
and value.get("environment") == "production"
and isinstance(checks, dict)
and set(checks) == expected
and all(checks.get(name) is True for name in expected)
and isinstance(references, dict)
and set(references) == expected
and all(isinstance(reference := references.get(name), str) and bool(reference) for name in expected)
)
class KnowledgeFSWorkspaceCutoverPhase(StrEnum):
INVENTORY = "inventory"
BACKFILL = "backfill"
SHADOW = "shadow"
FROZEN = "frozen"
CUTOVER = "cutover"
OBSERVING = "observing"
READY_FOR_CLEANUP = "ready_for_cleanup"
class KnowledgeFSMigrationIssueStatus(StrEnum):
OPEN = "open"
APPROVED_FAIL_CLOSED = "approved_fail_closed"
RESOLVED = "resolved"
class KnowledgeFSMigrationIssueKind(StrEnum):
REGISTRATION_CONFLICT = "registration_conflict"
UNRESOLVED_SUBJECT = "unresolved_subject"
UNKNOWN_EXTERNAL_ACCESS = "unknown_external_access"
REVISION_DRIFT = "revision_drift"
TASK_QUARANTINE = "task_quarantine"
LEGACY_SNAPSHOT_DEPENDENCY = "legacy_snapshot_dependency"
LEGACY_FOREIGN_KEY_DEPENDENCY = "legacy_foreign_key_dependency"
class KnowledgeFSMigrationQuarantineKind(StrEnum):
CONTROL_SPACE = "control_space"
SUBJECT = "subject"
TASK = "task"
LEGACY_API_KEY = "legacy_api_key"
ORPHAN_RESOURCE = "orphan_resource"
class KnowledgeFSMigrationQuarantineDisposition(StrEnum):
PENDING = "pending"
MIGRATABLE = "migratable"
WAIT_FOR_COMPLETION = "wait_for_completion"
CANCEL = "cancel"
ISOLATE = "isolate"
ROTATE_CREDENTIAL = "rotate_credential"
RESOLVED = "resolved"
class KnowledgeFSShadowAuthorizationDecision(StrEnum):
MATCH = "match"
TIGHTENED = "tightened"
EXPANDED = "expanded"
UNKNOWN = "unknown"
class KnowledgeFSWorkspaceCutoverLedger(DefaultFieldsDCMixin, TypeBase):
"""Single-Workspace migration state, watermarks, switches, and time fences."""
__tablename__ = "knowledge_fs_workspace_cutover_ledgers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
Index("kfs_workspace_cutover_ledger_phase_updated_idx", "phase", "updated_at"),
sa.CheckConstraint("cas_version >= 0", name=sa.schema.conv("kfs_workspace_cutover_cas_version_ck")),
sa.CheckConstraint(
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
name=sa.schema.conv("kfs_workspace_cutover_task_watermark_ck"),
),
sa.CheckConstraint(
"cutover_at IS NULL OR freeze_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_freeze_before_cutover_ck"),
),
sa.CheckConstraint(
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_rollback_cutoff_ck"),
),
sa.CheckConstraint(
"shadow_observation_count >= 0",
name=sa.schema.conv("kfs_workspace_cutover_shadow_count_ck"),
),
sa.CheckConstraint(
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_shadow_window_ck"),
),
sa.CheckConstraint(
"(shadow_completed_at IS NULL "
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
"AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
name=sa.schema.conv("kfs_workspace_cutover_shadow_completion_fields_ck"),
),
sa.CheckConstraint(
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_fields_ck"),
),
sa.CheckConstraint(
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_time_ck"),
),
sa.CheckConstraint(
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
"AND remote_activation_replayed IS NULL) OR "
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
"AND remote_activation_acknowledged_at IS NOT NULL "
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_fields_ck"),
),
sa.CheckConstraint(
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_time_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
source_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
applied_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
phase: Mapped[KnowledgeFSWorkspaceCutoverPhase] = mapped_column(
EnumText(KnowledgeFSWorkspaceCutoverPhase, length=32),
nullable=False,
server_default=sa.text("'inventory'"),
default=KnowledgeFSWorkspaceCutoverPhase.INVENTORY,
)
final_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
sa.JSON, nullable=True, default=None
)
source_task_watermark: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
final_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
applied_task_watermark: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
shadow_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_evidence_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
shadow_observation_count: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
shadow_window_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_window_ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_traffic_zero: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
shadow_traffic_zero_evidence: Mapped[dict[str, object] | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
shadow_latest_observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
shadow_producer: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
shadow_completed_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
shadow_completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_freeze_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
remote_freeze_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_freeze_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
remote_freeze_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_freeze_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_freeze_frozen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_freeze_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_activation_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
remote_activation_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_activation_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
remote_activation_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_activation_activated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_activation_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
freeze_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
cutover_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
rollback_cutoff_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_window_ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
maximum_task_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
irreversible_cleanup_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
product_routes_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
capability_v2_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
integrated_mode_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
legacy_acl_read_only: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
smoke_results: Mapped[KnowledgeFSCutoverSmokeResults | None] = mapped_column(sa.JSON, nullable=True, default=None)
legacy_dependency_report: Mapped[list[dict[str, object]] | None] = mapped_column(
sa.JSON, nullable=True, default=None
)
legacy_dependency_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
legacy_dependency_ready: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSMigrationIssue(DefaultFieldsDCMixin, TypeBase):
"""Auditable blocker; unknown access can only be approved as fail-closed."""
__tablename__ = "knowledge_fs_migration_issues"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_issue_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_migration_issue_gate_idx", "tenant_id", "ledger_id", "status", "kind"),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
issue_key: Mapped[str] = mapped_column(String(255), nullable=False)
kind: Mapped[KnowledgeFSMigrationIssueKind] = mapped_column(
EnumText(KnowledgeFSMigrationIssueKind, length=48), nullable=False
)
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
EnumText(KnowledgeFSMigrationIssueStatus, length=32),
nullable=False,
server_default=sa.text("'open'"),
default=KnowledgeFSMigrationIssueStatus.OPEN,
)
resource_type: Mapped[str | None] = mapped_column(String(64), nullable=True, default=None)
resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
class KnowledgeFSMigrationQuarantine(DefaultFieldsDCMixin, TypeBase):
"""Fail-closed inventory item whose operator resolution is immutable and CAS-versioned."""
__tablename__ = "knowledge_fs_migration_quarantine"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
UniqueConstraint(
"tenant_id",
"ledger_id",
"source_kind",
"source_id",
name="kfs_migration_quarantine_source_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_quarantine_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_migration_quarantine_disposition_idx", "tenant_id", "ledger_id", "disposition"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_migration_quarantine_version_ck")),
sa.CheckConstraint(
"(disposition = 'resolved' "
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
"(disposition <> 'resolved' "
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
"AND evidence IS NULL AND resolved_at IS NULL)",
name=sa.schema.conv("kfs_migration_quarantine_resolution_fields_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
source_kind: Mapped[KnowledgeFSMigrationQuarantineKind] = mapped_column(
EnumText(KnowledgeFSMigrationQuarantineKind, length=32), nullable=False
)
source_id: Mapped[str] = mapped_column(String(255), nullable=False)
reason_code: Mapped[str] = mapped_column(String(64), nullable=False)
disposition: Mapped[KnowledgeFSMigrationQuarantineDisposition] = mapped_column(
EnumText(KnowledgeFSMigrationQuarantineDisposition, length=32), nullable=False
)
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
resolved_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
evidence: Mapped[dict[str, object] | None] = mapped_column(sa.JSON(none_as_null=True), nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSShadowAuthorizationDiff(DefaultFieldsDCMixin, TypeBase):
"""Current disposition for a diff key; append-only observations retain its history."""
__tablename__ = "knowledge_fs_shadow_authorization_diffs"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_diff_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_shadow_authorization_diff_gate_idx", "tenant_id", "ledger_id", "status", "decision"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_shadow_authorization_diff_version_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
)
reason: Mapped[str] = mapped_column(LongText, nullable=False)
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
EnumText(KnowledgeFSMigrationIssueStatus, length=32), nullable=False
)
current_evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
last_observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSShadowAuthorizationObservation(DefaultFieldsDCMixin, TypeBase):
"""Append-only producer evidence used to derive a shadow diff disposition."""
__tablename__ = "knowledge_fs_shadow_authorization_observations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
UniqueConstraint(
"tenant_id",
"ledger_id",
"diff_key",
"evidence_digest",
name="kfs_shadow_authorization_observation_evidence_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_observation_ledger_fk",
ondelete="CASCADE",
),
Index(
"kfs_shadow_authorization_observation_window_idx",
"tenant_id",
"ledger_id",
"observed_at",
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
producer: Mapped[str] = mapped_column(String(255), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
)
reason: Mapped[str] = mapped_column(LongText, nullable=False)
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
__all__ = [
"KnowledgeFSCutoverRevisionWatermark",
"KnowledgeFSCutoverSmokeChecks",
"KnowledgeFSCutoverSmokeEvidenceReferences",
"KnowledgeFSCutoverSmokeResults",
"KnowledgeFSMigrationIssue",
"KnowledgeFSMigrationIssueKind",
"KnowledgeFSMigrationIssueStatus",
"KnowledgeFSMigrationQuarantine",
"KnowledgeFSMigrationQuarantineDisposition",
"KnowledgeFSMigrationQuarantineKind",
"KnowledgeFSShadowAuthorizationDecision",
"KnowledgeFSShadowAuthorizationDiff",
"KnowledgeFSShadowAuthorizationObservation",
"KnowledgeFSWorkspaceCutoverLedger",
"KnowledgeFSWorkspaceCutoverPhase",
"knowledge_fs_cutover_smoke_results_passed",
]
+2
View File
@@ -1476,6 +1476,8 @@ class WorkflowRunArchiveBundle(DefaultFieldsDCMixin, TypeBase):
name="workflow_run_archive_bundle_identity_uq",
),
sa.Index("workflow_run_archive_bundle_tenant_month_idx", "tenant_id", "year", "month"),
sa.Index("workflow_run_archive_bundle_month_id_idx", "year", "month", "id"),
sa.Index("workflow_run_archive_bundle_month_shard_id_idx", "year", "month", "shard", "id"),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+43
View File
@@ -9639,6 +9639,27 @@ Bedrock retrieval test (internal use only)
| ---- | ----------- | ------ |
| 200 | Success | **application/json**: [TrialDatasetListResponse](#trialdatasetlistresponse)<br> |
### [POST] /trial-apps/{app_id}/files/upload
**Upload a file into the tenant that owns the trial app**
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| app_id | path | | Yes | string (uuid) |
#### Request Body
| Required | Schema |
| -------- | ------ |
| Yes | **multipart/form-data**: { **"file"**: binary, **"source"**: string, <br>**Available values:** "datasets" }<br> |
#### Responses
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | File uploaded successfully | **application/json**: [FileResponse](#fileresponse)<br> |
### [GET] /trial-apps/{app_id}/messages/{message_id}/suggested-questions
#### Parameters
@@ -9668,6 +9689,27 @@ Bedrock retrieval test (internal use only)
| ---- | ----------- | ------ |
| 200 | Success | **application/json**: [Parameters](#parameters)<br> |
### [POST] /trial-apps/{app_id}/remote-files/upload
**Upload a remote file into the tenant that owns the trial app**
#### Parameters
| Name | Located in | Description | Required | Schema |
| ---- | ---------- | ----------- | -------- | ------ |
| app_id | path | | Yes | string (uuid) |
#### Request Body
| Required | Schema |
| -------- | ------ |
| Yes | **application/json**: [RemoteFileUploadPayload](#remotefileuploadpayload)<br> |
#### Responses
| Code | Description | Schema |
| ---- | ----------- | ------ |
| 201 | File uploaded successfully | **application/json**: [FileWithSignedUrl](#filewithsignedurl)<br> |
### [GET] /trial-apps/{app_id}/site
**Retrieve app site info**
@@ -21859,6 +21901,7 @@ The subscription constructor of the trigger provider
| Name | Type | Description | Required |
| ---- | ---- | ----------- | -------- |
| _is_collaborative | boolean | | No |
| conversation_variables | [ object ] | | No |
| environment_variables | [ object ] | | No |
| features | object | | Yes |
@@ -0,0 +1,53 @@
"""Persistence contract for irreversible KnowledgeFS cleanup authorization."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs_cleanup import (
KnowledgeFSCleanupAuthorization,
KnowledgeFSCleanupAuthorizationStatus,
)
@dataclass(frozen=True, slots=True)
class KnowledgeFSCleanupAuthorizationCASUpdate:
tenant_id: str
ledger_id: str
request_id: str
expected_status: KnowledgeFSCleanupAuthorizationStatus
expected_row_version: int
new_status: KnowledgeFSCleanupAuthorizationStatus
approved_by_account_id: str | None = None
approved_at: datetime | None = None
approval_expires_at: datetime | None = None
approved_ledger_cas_version: int | None = None
started_by_account_id: str | None = None
started_at: datetime | None = None
started_ledger_cas_version: int | None = None
completed_by_account_id: str | None = None
completed_at: datetime | None = None
completion_evidence: dict[str, object] | None = None
completed_ledger_cas_version: int | None = None
class KnowledgeFSCleanupAuthorizationRepository(Protocol):
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization: ...
def get(
self,
*,
tenant_id: str,
ledger_id: str,
request_id: str,
) -> KnowledgeFSCleanupAuthorization | None: ...
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool: ...
__all__ = [
"KnowledgeFSCleanupAuthorizationCASUpdate",
"KnowledgeFSCleanupAuthorizationRepository",
]
@@ -0,0 +1,60 @@
"""Persistence boundary for tenant-scoped KnowledgeFS lifecycle operations."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
@dataclass(frozen=True, slots=True)
class KnowledgeFSControlSpaceCASUpdate:
"""Values applied only when the stored state and resource version match."""
tenant_id: str
control_space_id: str
expected_resource_version: int
expected_state: KnowledgeFSControlSpaceState
new_state: KnowledgeFSControlSpaceState
lifecycle_operation_id: str
knowledge_space_id: str | None = None
knowledge_space_revision: int | None = None
attempted_at: datetime | None = None
last_error_code: str | None = None
last_error_message: str | None = None
class KnowledgeFSControlSpaceRepository(Protocol):
"""Repository operations that always include the owning Dify tenant."""
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace: ...
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None: ...
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None: ...
def find_by_knowledge_space_id(
self, *, tenant_id: str, knowledge_space_id: str
) -> KnowledgeFSControlSpace | None: ...
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]: ...
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]: ...
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool: ...
def mark_deletion_irreversible(
self,
*,
tenant_id: str,
control_space_id: str,
lifecycle_operation_id: str,
irreversible_at: datetime,
) -> bool: ...
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]: ...
__all__ = ["KnowledgeFSControlSpaceCASUpdate", "KnowledgeFSControlSpaceRepository"]
@@ -0,0 +1,229 @@
"""Persistence contract for KnowledgeFS workspace migration and cutover state."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs_cutover import (
KnowledgeFSCutoverRevisionWatermark,
KnowledgeFSCutoverSmokeResults,
KnowledgeFSMigrationIssue,
KnowledgeFSMigrationIssueStatus,
KnowledgeFSMigrationQuarantine,
KnowledgeFSMigrationQuarantineDisposition,
KnowledgeFSMigrationQuarantineKind,
KnowledgeFSShadowAuthorizationDecision,
KnowledgeFSShadowAuthorizationDiff,
KnowledgeFSShadowAuthorizationObservation,
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
)
@dataclass(frozen=True, slots=True)
class KnowledgeFSCutoverCASUpdate:
"""Optional values changed with the phase and CAS version in one SQL statement."""
tenant_id: str
expected_phase: KnowledgeFSWorkspaceCutoverPhase
expected_cas_version: int
new_phase: KnowledgeFSWorkspaceCutoverPhase
source_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
final_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
applied_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
source_task_watermark: int | None = None
final_task_watermark: int | None = None
applied_task_watermark: int | None = None
shadow_started_at: datetime | None = None
shadow_completed_at: datetime | None = None
shadow_evidence_digest: str | None = None
shadow_observation_count: int | None = None
shadow_window_started_at: datetime | None = None
shadow_window_ended_at: datetime | None = None
shadow_traffic_zero: bool | None = None
shadow_traffic_zero_evidence: dict[str, object] | None = None
shadow_latest_observed_revision: KnowledgeFSCutoverRevisionWatermark | None = None
shadow_producer: str | None = None
shadow_completed_by_operator: str | None = None
shadow_completed_by_account_id: str | None = None
remote_freeze_id: str | None = None
remote_freeze_revision: int | None = None
remote_freeze_digest: str | None = None
remote_freeze_task_watermark: int | None = None
remote_freeze_control_space_id: str | None = None
remote_freeze_frozen_at: datetime | None = None
remote_freeze_updated_at: datetime | None = None
remote_freeze_acknowledged_at: datetime | None = None
remote_freeze_applied: bool | None = None
remote_freeze_replayed: bool | None = None
remote_activation_id: str | None = None
remote_activation_revision: int | None = None
remote_activation_digest: str | None = None
remote_activation_control_space_id: str | None = None
remote_activation_activated_at: datetime | None = None
remote_activation_updated_at: datetime | None = None
remote_activation_acknowledged_at: datetime | None = None
remote_activation_applied: bool | None = None
remote_activation_replayed: bool | None = None
clear_smoke_results: bool = False
freeze_at: datetime | None = None
cutover_at: datetime | None = None
rolled_back_at: datetime | None = None
rollback_cutoff_at: datetime | None = None
observation_started_at: datetime | None = None
observation_window_ends_at: datetime | None = None
observation_completed_at: datetime | None = None
maximum_task_expires_at: datetime | None = None
irreversible_cleanup_at: datetime | None = None
smoke_results: KnowledgeFSCutoverSmokeResults | None = None
legacy_dependency_report: list[dict[str, object]] | None = None
legacy_dependency_checked_at: datetime | None = None
legacy_dependency_ready: bool | None = None
product_routes_enabled: bool | None = None
capability_v2_enabled: bool | None = None
integrated_mode_enabled: bool | None = None
legacy_acl_read_only: bool | None = None
@dataclass(frozen=True, slots=True)
class KnowledgeFSQuarantineCASUpdate:
"""Tenant/item-scoped transition from an operator disposition to resolved."""
tenant_id: str
ledger_id: str
source_kind: KnowledgeFSMigrationQuarantineKind
source_id: str
expected_disposition: KnowledgeFSMigrationQuarantineDisposition
expected_row_version: int
resolved_by_operator: str
resolved_by_account_id: str
evidence: dict[str, object]
resolved_at: datetime
@dataclass(frozen=True, slots=True)
class KnowledgeFSShadowDiffCASUpdate:
"""Replace one current diff disposition after append-only safe re-evaluation."""
tenant_id: str
ledger_id: str
diff_key: str
expected_row_version: int
control_space_id: str | None
principal: str
legacy_allowed: bool | None
dify_allowed: bool
decision: KnowledgeFSShadowAuthorizationDecision
reason: str
observed_revision: KnowledgeFSCutoverRevisionWatermark
status: KnowledgeFSMigrationIssueStatus
current_evidence_digest: str
last_observed_at: datetime
class KnowledgeFSCutoverRepository(Protocol):
"""Tenant-scoped storage; callers own the surrounding transaction."""
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger: ...
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None: ...
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]: ...
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool: ...
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue: ...
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None: ...
def set_issue_status(
self,
*,
tenant_id: str,
ledger_id: str,
issue_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool: ...
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine: ...
def get_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kind: KnowledgeFSMigrationQuarantineKind,
source_id: str,
) -> KnowledgeFSMigrationQuarantine | None: ...
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool: ...
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff: ...
def get_shadow_diff(
self, *, tenant_id: str, ledger_id: str, diff_key: str
) -> KnowledgeFSShadowAuthorizationDiff | None: ...
def set_shadow_diff_status(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool: ...
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool: ...
def add_shadow_observation(
self, observation: KnowledgeFSShadowAuthorizationObservation
) -> KnowledgeFSShadowAuthorizationObservation: ...
def get_shadow_observation(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
evidence_digest: str,
) -> KnowledgeFSShadowAuthorizationObservation | None: ...
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int: ...
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int: ...
def count_unresolved_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
) -> int: ...
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]: ...
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]: ...
def list_shadow_diffs(
self, *, tenant_id: str, ledger_id: str
) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]: ...
def list_shadow_observations(
self, *, tenant_id: str, ledger_id: str
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]: ...
__all__ = [
"KnowledgeFSCutoverCASUpdate",
"KnowledgeFSCutoverRepository",
"KnowledgeFSQuarantineCASUpdate",
"KnowledgeFSShadowDiffCASUpdate",
]
@@ -0,0 +1,95 @@
"""Persistence boundary for durable KnowledgeFS lifecycle commands."""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import Protocol
from models.knowledge_fs import KnowledgeFSLifecycleOperation, KnowledgeFSLifecycleOutbox
class KnowledgeFSLifecycleOutboxRepository(Protocol):
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox: ...
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
def find_open_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None: ...
def find_latest_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None: ...
def reactivate_dead_letter(self, *, outbox_id: str) -> bool: ...
def supersede_unattempted(
self,
*,
outbox_id: str,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def supersede_after_remote_absence(
self,
*,
outbox_id: str,
observed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def claim_next(
self,
*,
lease_owner: str,
now: datetime,
lease_duration: timedelta,
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
) -> KnowledgeFSLifecycleOutbox | None: ...
def acknowledge(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
) -> bool: ...
def schedule_retry(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
next_attempt_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def mark_dead_letter(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
__all__ = ["KnowledgeFSLifecycleOutboxRepository"]
@@ -0,0 +1,35 @@
"""Durable SQLAlchemy sink for sanitized KnowledgeFS capability issuance audits."""
from __future__ import annotations
from typing import cast, override
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import KnowledgeFSCapabilityClaimsSummary, KnowledgeFSCapabilityIssuanceAudit
from services.knowledge_fs_capability import CapabilityIssuanceAuditEvent, CapabilityIssuanceAuditor
class SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor(CapabilityIssuanceAuditor):
"""Commit the audit independently so token issuance fails closed on audit failure."""
def __init__(self, session_maker: sessionmaker[Session]):
self._session_maker = session_maker
@override
def record(self, event: CapabilityIssuanceAuditEvent) -> None:
serialized = event.model_dump(mode="json")
trace_id = cast(str, serialized.pop("trace_id"))
jti_hash = cast(str, serialized.pop("jti_hash"))
audit = KnowledgeFSCapabilityIssuanceAudit(
tenant_id=event.namespace_id,
control_space_id=event.control_space_id,
trace_id=trace_id,
jti_hash=jti_hash,
claims_summary=cast(KnowledgeFSCapabilityClaimsSummary, serialized),
)
with self._session_maker.begin() as session:
session.add(audit)
__all__ = ["SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor"]
@@ -0,0 +1,185 @@
"""Transactional persistence for pre-signing KnowledgeFS capability reservations."""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import cast
import sqlalchemy as sa
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs import (
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSCapabilityIssuanceReservationStatus,
KnowledgeFSCapabilityReservationSummary,
)
from services.knowledge_fs_capability import (
CAPABILITY_ISSUANCE_PROFILES,
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
CapabilityIssueRequest,
)
_TERMINAL_RETENTION = timedelta(days=1)
class KnowledgeFSCapabilityIssuanceReservationError(RuntimeError):
"""A retry attempted to change or lose a durable issuance binding."""
class SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository:
def __init__(self, session: Session) -> None:
self._session = session
def reserve(self, request: CapabilityIssueRequest) -> KnowledgeFSCapabilityIssuanceReservation:
summary = _request_summary(request)
reservation = self._session.scalar(
sa.select(KnowledgeFSCapabilityIssuanceReservation)
.where(
KnowledgeFSCapabilityIssuanceReservation.tenant_id == request.namespace_id,
KnowledgeFSCapabilityIssuanceReservation.grant_id == request.grant_id,
)
.with_for_update()
)
if reservation is None:
reservation = KnowledgeFSCapabilityIssuanceReservation(
tenant_id=request.namespace_id,
control_space_id=request.control_space_id,
grant_id=request.grant_id,
trace_id=request.trace_id,
subject=summary["subject"],
caller_kind=request.caller_kind,
request_summary=summary,
)
self._session.add(reservation)
self._session.flush()
return reservation
if (
reservation.control_space_id != request.control_space_id
or reservation.trace_id != request.trace_id
or reservation.subject != summary["subject"]
or reservation.caller_kind != request.caller_kind
or reservation.request_summary != summary
):
raise KnowledgeFSCapabilityIssuanceReservationError(
"Capability grant retry does not match its durable reservation"
)
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.FAILED:
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED
reservation.failed_at = None
reservation.failure_code = None
reservation.cleanup_after = None
reservation.row_version += 1
self._session.flush()
return reservation
def mark_issued(
self,
*,
tenant_id: str,
grant_id: str,
issued_at: datetime,
token_expires_at: datetime,
) -> None:
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED
reservation.issued_at = issued_at
reservation.token_expires_at = token_expires_at
reservation.failed_at = None
reservation.failure_code = None
reservation.cleanup_after = token_expires_at + _TERMINAL_RETENTION
reservation.row_version += 1
self._session.flush()
def mark_failed(
self,
*,
tenant_id: str,
grant_id: str,
failed_at: datetime,
failure_code: str,
) -> None:
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED:
return
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.FAILED
reservation.issued_at = None
reservation.token_expires_at = None
reservation.failed_at = failed_at
reservation.failure_code = failure_code[:128]
reservation.cleanup_after = failed_at + _TERMINAL_RETENTION
reservation.row_version += 1
self._session.flush()
def cleanup_terminal(self, *, before: datetime, limit: int = 1_000) -> int:
if limit <= 0:
raise ValueError("Capability reservation cleanup limit must be positive")
reservation_ids = tuple(
self._session.scalars(
sa.select(KnowledgeFSCapabilityIssuanceReservation.id)
.where(
KnowledgeFSCapabilityIssuanceReservation.status
!= KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
KnowledgeFSCapabilityIssuanceReservation.cleanup_after <= before,
)
.order_by(
KnowledgeFSCapabilityIssuanceReservation.cleanup_after,
KnowledgeFSCapabilityIssuanceReservation.id,
)
.limit(limit)
.with_for_update()
)
)
if not reservation_ids:
return 0
result = self._session.execute(
sa.delete(KnowledgeFSCapabilityIssuanceReservation).where(
KnowledgeFSCapabilityIssuanceReservation.id.in_(reservation_ids)
)
)
return cast(CursorResult[tuple[object, ...]], result).rowcount or 0
def _locked(self, *, tenant_id: str, grant_id: str) -> KnowledgeFSCapabilityIssuanceReservation:
reservation = self._session.scalar(
sa.select(KnowledgeFSCapabilityIssuanceReservation)
.where(
KnowledgeFSCapabilityIssuanceReservation.tenant_id == tenant_id,
KnowledgeFSCapabilityIssuanceReservation.grant_id == grant_id,
)
.with_for_update()
)
if reservation is None:
raise KnowledgeFSCapabilityIssuanceReservationError("Capability issuance reservation is missing")
return reservation
def _request_summary(request: CapabilityIssueRequest) -> KnowledgeFSCapabilityReservationSummary:
operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[request.operation_id]
profile = CAPABILITY_ISSUANCE_PROFILES[request.caller_kind]
subject = f"{profile.subject_prefix}:{request.principal_id}"
return cast(
KnowledgeFSCapabilityReservationSummary,
{
"action": operation.action,
"actor": request.actor,
"authz_revision": request.authz_revision.model_dump(mode="json"),
"caller_kind": request.caller_kind,
"content_policy_revision": request.content_policy_revision,
"content_scope_ids": list(request.content_scope_ids),
"control_space_id": request.control_space_id,
"grant_id": request.grant_id,
"namespace_id": request.namespace_id,
"operation_id": request.operation_id,
"resource_id": request.resource.id,
"resource_parent_id": request.resource.parent_id,
"resource_type": request.resource.type,
"subject": subject,
"trace_id": request.trace_id,
},
)
__all__ = [
"KnowledgeFSCapabilityIssuanceReservationError",
"SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository",
]
@@ -0,0 +1,87 @@
"""SQLAlchemy cleanup-authorization repository with status and version CAS."""
from __future__ import annotations
from typing import cast, override
from sqlalchemy import select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs_cleanup import KnowledgeFSCleanupAuthorization
from repositories.knowledge_fs_cleanup_repository import (
KnowledgeFSCleanupAuthorizationCASUpdate,
KnowledgeFSCleanupAuthorizationRepository,
)
class SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(KnowledgeFSCleanupAuthorizationRepository):
_session: Session
def __init__(self, session: Session):
self._session = session
@override
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization:
self._session.add(authorization)
self._session.flush()
return authorization
@override
def get(
self,
*,
tenant_id: str,
ledger_id: str,
request_id: str,
) -> KnowledgeFSCleanupAuthorization | None:
statement = (
select(KnowledgeFSCleanupAuthorization)
.where(
KnowledgeFSCleanupAuthorization.tenant_id == tenant_id,
KnowledgeFSCleanupAuthorization.ledger_id == ledger_id,
KnowledgeFSCleanupAuthorization.request_id == request_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool:
values: dict[str, object] = {
"status": update_values.new_status,
"row_version": KnowledgeFSCleanupAuthorization.row_version + 1,
}
for field_name in (
"approved_by_account_id",
"approved_at",
"approval_expires_at",
"approved_ledger_cas_version",
"started_by_account_id",
"started_at",
"started_ledger_cas_version",
"completed_by_account_id",
"completed_at",
"completion_evidence",
"completed_ledger_cas_version",
):
value = getattr(update_values, field_name)
if value is not None:
values[field_name] = value
statement = (
update(KnowledgeFSCleanupAuthorization)
.where(
KnowledgeFSCleanupAuthorization.tenant_id == update_values.tenant_id,
KnowledgeFSCleanupAuthorization.ledger_id == update_values.ledger_id,
KnowledgeFSCleanupAuthorization.request_id == update_values.request_id,
KnowledgeFSCleanupAuthorization.status == update_values.expected_status,
KnowledgeFSCleanupAuthorization.row_version == update_values.expected_row_version,
)
.values(**values)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
__all__ = ["SQLAlchemyKnowledgeFSCleanupAuthorizationRepository"]
@@ -0,0 +1,187 @@
"""SQLAlchemy implementation of the KnowledgeFS control-space repository."""
from __future__ import annotations
from datetime import datetime
from typing import cast, override
from sqlalchemy import case, select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
from repositories.knowledge_fs_control_space_repository import (
KnowledgeFSControlSpaceCASUpdate,
KnowledgeFSControlSpaceRepository,
)
class SQLAlchemyKnowledgeFSControlSpaceRepository(KnowledgeFSControlSpaceRepository):
"""Caller-transaction-owned repository with tenant and CAS guards."""
def __init__(self, session: Session):
self._session = session
@override
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace:
self._session.add(control_space)
self._session.flush()
return control_space
@override
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.provisioning_key == provisioning_key)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_by_knowledge_space_id(self, *, tenant_id: str, knowledge_space_id: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.knowledge_space_id == knowledge_space_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]:
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
.execution_options(populate_existing=True)
)
return tuple(self._session.scalars(statement).all())
@override
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]:
if limit <= 0:
return ()
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED)
.order_by(
case(
(
KnowledgeFSControlSpace.state.in_(
(
KnowledgeFSControlSpaceState.PROVISIONING,
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
)
),
0,
),
else_=1,
),
KnowledgeFSControlSpace.updated_at,
KnowledgeFSControlSpace.id,
)
.limit(limit)
.execution_options(populate_existing=True)
)
return tuple(self._session.scalars(statement).all())
@override
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool:
values: dict[str, object] = {
"state": update_values.new_state,
"resource_version": KnowledgeFSControlSpace.resource_version + 1,
"lifecycle_operation_id": update_values.lifecycle_operation_id,
"last_error_code": update_values.last_error_code,
"last_error_message": update_values.last_error_message,
}
if update_values.knowledge_space_id is not None:
values["knowledge_space_id"] = update_values.knowledge_space_id
if update_values.knowledge_space_revision is not None:
values["knowledge_space_revision"] = update_values.knowledge_space_revision
if update_values.attempted_at is not None:
values["attempt_count"] = KnowledgeFSControlSpace.attempt_count + 1
values["last_attempt_at"] = update_values.attempted_at
statement = (
update(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == update_values.tenant_id,
KnowledgeFSControlSpace.id == update_values.control_space_id,
KnowledgeFSControlSpace.resource_version == update_values.expected_resource_version,
KnowledgeFSControlSpace.state == update_values.expected_state,
)
.values(**values)
.execution_options(synchronize_session=False)
)
if update_values.knowledge_space_id is not None:
statement = statement.where(
(KnowledgeFSControlSpace.knowledge_space_id.is_(None))
| (KnowledgeFSControlSpace.knowledge_space_id == update_values.knowledge_space_id)
)
if update_values.knowledge_space_revision is not None:
statement = statement.where(
KnowledgeFSControlSpace.knowledge_space_revision <= update_values.knowledge_space_revision
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def mark_deletion_irreversible(
self,
*,
tenant_id: str,
control_space_id: str,
lifecycle_operation_id: str,
irreversible_at: datetime,
) -> bool:
statement = (
update(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
KnowledgeFSControlSpace.lifecycle_operation_id == lifecycle_operation_id,
KnowledgeFSControlSpace.state.in_(
(
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
KnowledgeFSControlSpaceState.DELETED,
)
),
(KnowledgeFSControlSpace.deletion_irreversible_at.is_(None))
| (KnowledgeFSControlSpace.deletion_irreversible_at == irreversible_at),
)
.values(deletion_irreversible_at=irreversible_at)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]:
statement = (
select(KnowledgeFSControlSpace.id)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
)
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
)
return tuple(self._session.scalars(statement).all())
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceRepository"]

Some files were not shown because too many files have changed in this diff Show More