Compare commits

...
Author SHA1 Message Date
Jyong d74499c09d update github action 2026-07-22 06:07:12 -04:00
Jyong 29858d0db0 Expand application capabilities and refactor core workflows 2026-07-22 05:09:09 -04:00
Jyong 77732c7bb1 docs: decouple KnowledgeFS from Dify datasets 2026-07-20 06:02:06 -04:00
Jyong dde79fc9bf docs: add Dify KnowledgeFS integration plan 2026-07-20 05:33:56 -04:00
Jyong 4ee43b8afc chore: migrate knowledge-fs source tree
Import the committed KnowledgeFS snapshot dc4072ee302317145612087ce7440851dc329fd0 under knowledge-fs/ without its Git history, local IDE settings, or build artifacts.
2026-07-20 04:54:20 -04:00
1838 changed files with 556246 additions and 1869 deletions
+15
View File
@@ -0,0 +1,15 @@
{
"hooks": {
"PreToolUse": [
{
"matcher": "Bash",
"hooks": [
{
"type": "command",
"command": "npx -y block-no-verify@1.1.1"
}
]
}
]
}
}
+1
View File
@@ -8,6 +8,7 @@
**/*.pyc
**/.mypy_cache
**/.ruff_cache
knowledge-fs/
.git
.github
*.md
+9
View File
@@ -1,6 +1,15 @@
version: 2
updates:
- package-ecosystem: "npm"
directory: "/knowledge-fs"
open-pull-requests-limit: 10
schedule:
interval: "weekly"
groups:
knowledge-fs-dependencies:
patterns:
- "*"
- package-ecosystem: "uv"
directory: "/api"
open-pull-requests-limit: 10
+504
View File
@@ -0,0 +1,504 @@
name: KnowledgeFS CI
on:
pull_request:
branches: ["main"]
merge_group:
branches: ["main"]
types: [checks_requested]
push:
branches: ["main"]
workflow_dispatch:
permissions:
contents: read
pull-requests: read
concurrency:
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
CI: true
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
jobs:
check-changes:
name: Check KnowledgeFS changes
runs-on: depot-ubuntu-24.04-4
outputs:
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Detect KnowledgeFS changes
id: changes
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
with:
filters: |
knowledge-fs:
- 'knowledge-fs/**'
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
- 'knowledge-fs/scripts/export-openapi.mjs'
- 'api/dev/generate_knowledge_fs_contract.py'
- 'api/dev/knowledge_fs_product_contract.py'
- 'api/knowledge-fs-contract.lock.json'
- 'api/knowledge-fs-product-operation-gaps.json'
- 'api/knowledge-fs-product-operations.json'
- 'api/**/knowledge_fs/**'
- 'api/**/*knowledge_fs*'
- 'api/**/*knowledge-fs*'
- 'api/.env.example'
- 'api/app_factory.py'
- 'api/commands/__init__.py'
- 'api/controllers/console/__init__.py'
- 'api/controllers/console/workspace/rbac.py'
- 'api/controllers/service_api/__init__.py'
- 'api/core/agent/base_agent_runner.py'
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
- 'api/core/rbac/entities.py'
- 'api/core/tools/__base/tool_runtime.py'
- 'api/core/tools/builtin_tool/_position.yaml'
- 'api/core/workflow/node_runtime.py'
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
- 'api/extensions/ext_celery.py'
- 'api/extensions/ext_commands.py'
- 'api/models/__init__.py'
- 'api/services/account_service.py'
- 'api/services/agent_tool_inner_service.py'
- 'api/services/enterprise/rbac_service.py'
- 'api/services/entities/agent_tool_inner.py'
- 'api/services/knowledge_fs/**'
- 'api/services/knowledge_fs_capability.py'
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
- 'api/tests/unit_tests/services/test_account_service.py'
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
- 'api/pyproject.toml'
- 'api/uv.lock'
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
- 'packages/contracts/generated/api/console/**'
- 'packages/contracts/generated/api/service/**'
- 'docker/.env.example'
- 'docker/README.md'
- 'docker/dify-env-sync.py'
- 'docker/dify-env-sync.sh'
- 'docker/docker-compose-template.yaml'
- 'docker/docker-compose.yaml'
- 'docker/envs/core-services/api.env.example'
- 'docker/envs/core-services/knowledge-fs.env.example'
- 'docker/generate_docker_compose'
- 'docs/design/knowledge-fs*'
- '.github/dependabot.yml'
- '.github/workflows/knowledge-fs-ci.yml'
build:
name: Build KnowledgeFS API production image
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
- name: Login to Docker Hub
if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract KnowledgeFS image metadata
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=ref,event=branch
type=sha,format=long
- name: Build KnowledgeFS API image
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
with:
context: ./knowledge-fs
file: ./knowledge-fs/apps/api/Dockerfile
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64
push: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
tags: ${{ steps.meta.outputs.tags }}
quality:
name: Run KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
defaults:
run:
shell: bash
working-directory: ./knowledge-fs
steps:
- name: Checkout code
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
with:
package_json_file: knowledge-fs/package.json
run_install: false
- name: Setup Node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: pnpm
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
- name: Install KnowledgeFS dependencies
run: pnpm install --frozen-lockfile
- name: Scan KnowledgeFS secrets
run: pnpm security:secrets
- name: Audit KnowledgeFS production dependencies
run: pnpm security:dependencies
- name: Run KnowledgeFS checks
run: pnpm check
- name: Build KnowledgeFS
run: pnpm build
- name: Lint KnowledgeFS
run: pnpm lint
- name: Setup UV and Python
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
with:
enable-cache: true
python-version: "3.12"
cache-dependency-glob: |
api/uv.lock
dify-agent/uv.lock
- name: Verify Dify dependency lock
working-directory: .
run: uv lock --project api --check
- name: Install Dify contract dependencies
working-directory: .
run: uv sync --project api --locked --dev
- name: Collect Dify KnowledgeFS gate targets
working-directory: .
run: |
set -euo pipefail
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
mkdir -p "$target_dir"
production_targets=()
add_production_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS production target is missing: $path" >&2
exit 1
fi
production_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_production_target "$path"
fi
done < <(
find api \
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
-o -type f -name '*.py' -print0
)
production_touchpoints=(
api/app_factory.py
api/commands/__init__.py
api/controllers/console/__init__.py
api/controllers/console/workspace/rbac.py
api/controllers/service_api/__init__.py
api/core/agent/base_agent_runner.py
api/core/app/apps/agent_app/runtime_request_builder.py
api/core/rbac/entities.py
api/core/tools/__base/tool_runtime.py
api/core/workflow/node_runtime.py
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
api/extensions/ext_celery.py
api/extensions/ext_commands.py
api/models/__init__.py
api/services/account_service.py
api/services/agent_tool_inner_service.py
api/services/enterprise/rbac_service.py
api/services/entities/agent_tool_inner.py
)
for path in "${production_touchpoints[@]}"; do
add_production_target "$path"
done
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
if ((${#production_targets[@]} == 0)); then
echo "Dify KnowledgeFS production target set is empty" >&2
exit 1
fi
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
test_targets=()
add_test_target() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
exit 1
fi
test_targets+=("$path")
}
while IFS= read -r -d '' path; do
if [[ "$path" == *knowledge_fs* ]]; then
add_test_target "$path"
fi
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
test_touchpoints=(
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
api/tests/unit_tests/core/agent/test_base_agent_runner.py
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
api/tests/unit_tests/core/workflow/test_node_runtime.py
api/tests/unit_tests/services/enterprise/test_rbac_service.py
api/tests/unit_tests/services/test_account_service.py
api/tests/unit_tests/services/test_agent_tool_inner_service.py
)
for path in "${test_touchpoints[@]}"; do
add_test_target "$path"
done
required_test_scopes=(
/commands/
/configs/
/controllers/
/core/agent/
/core/app/apps/agent_app/
/core/tools/builtin_tool/providers/knowledge_fs/
/core/workflow/
/dev/
/extensions/
/migrations/
/models/
/repositories/
/services/
/tasks/
)
for required_scope in "${required_test_scopes[@]}"; do
scope_found=false
for path in "${test_targets[@]}"; do
if [[ "$path" == *"$required_scope"* ]]; then
scope_found=true
break
fi
done
if [[ "$scope_found" != true ]]; then
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
exit 1
fi
done
if ((${#test_targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target set is empty" >&2
exit 1
fi
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
- name: Lint Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
uv run --project api --dev ruff format --check "${targets[@]}"
uv run --project api --dev ruff check "${targets[@]}"
- name: Type-check Dify KnowledgeFS integration
working-directory: .
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS production target manifest is empty" >&2
exit 1
fi
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
mypy_targets=()
for path in "${targets[@]}"; do
if [[ "$path" != api/migrations/* ]]; then
mypy_targets+=("${path#api/}")
fi
done
if ((${#mypy_targets[@]} == 0)); then
echo "Dify KnowledgeFS Mypy target set is empty" >&2
exit 1
fi
uv run --directory api --dev mypy \
--explicit-package-bases \
--exclude-gitignore \
--exclude '(^|/)conftest\.py$' \
--exclude 'tests/' \
--exclude 'migrations/' \
--check-untyped-defs \
--disable-error-code=import-untyped \
"${mypy_targets[@]}"
- name: Test Dify KnowledgeFS unit surface
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
run: |
set -euo pipefail
targets=()
while IFS= read -r -d '' path; do
targets+=("$path")
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
if ((${#targets[@]} == 0)); then
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
exit 1
fi
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
- name: Enforce Dify KnowledgeFS focused coverage
working-directory: .
env:
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
KNOWLEDGE_FS_COVERAGE_BASE: >-
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
run: |
set -euo pipefail
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
uv run --project api --dev coverage json --show-contexts -o "$report"
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
--coverage-json "$report" \
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
--base "$KNOWLEDGE_FS_COVERAGE_BASE" \
--minimum 90 \
--glue-minimum 90
- name: Verify Dify KnowledgeFS contract
working-directory: .
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
- name: Verify Dify Agent dependency lock
working-directory: .
run: uv lock --project dify-agent --check
- name: Install Dify Agent gate dependencies
working-directory: .
run: uv sync --project dify-agent --locked --dev
- name: Lint Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: |
uv run --project . --dev ruff format --check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
uv run --project . --dev ruff check \
src/dify_agent/layers/dify_core_tools/client.py \
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Type-check Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev basedpyright --level error
src/dify_agent/layers/dify_core_tools/client.py
tests/local/dify_agent/layers/dify_core_tools/test_client.py
- name: Test Dify Agent KnowledgeFS integration
working-directory: ./dify-agent
run: >-
uv run --project . --dev python -m pytest
tests/local/dify_agent/layers/dify_core_tools/test_client.py
-q
skip:
name: Skip KnowledgeFS quality and contract gates
needs: check-changes
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
runs-on: depot-ubuntu-24.04-4
steps:
- name: Report skipped KnowledgeFS checks
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
final:
name: KnowledgeFS CI
if: ${{ always() }}
needs:
- check-changes
- build
- quality
- skip
runs-on: depot-ubuntu-24.04-4
steps:
- name: Finalize KnowledgeFS CI status
env:
EVENT_NAME: ${{ github.event_name }}
BUILD_RESULT: ${{ needs.build.result }}
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
QUALITY_RESULT: ${{ needs.quality.result }}
SKIP_RESULT: ${{ needs.skip.result }}
run: |
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
echo "KnowledgeFS build and checks ran successfully."
exit 0
fi
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
exit 1
fi
if [[ "$SKIP_RESULT" == 'success' ]]; then
echo "KnowledgeFS checks were skipped because no related files changed."
exit 0
fi
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
exit 1
+5
View File
@@ -30,6 +30,11 @@ share/python-wheels/
*.egg
MANIFEST
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
# directory contains source files rather than Python build output.
!/knowledge-fs/apps/admin/lib/
!/knowledge-fs/apps/admin/lib/**
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
+18 -1
View File
@@ -683,11 +683,28 @@ AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
# KnowledgeFS (Dataset 2.0)
KNOWLEDGE_FS_ENABLED=false
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
KNOWLEDGE_FS_BASE_URL=
# Shared with KnowledgeFS; use at least 32 random characters.
KNOWLEDGE_FS_DIRECT_ORIGIN=
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
# Legacy rollback-only HMAC; Capability v2 deployments leave this blank.
KNOWLEDGE_FS_JWT_SECRET=
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
# Marketplace configuration
MARKETPLACE_ENABLED=true
+2
View File
@@ -154,6 +154,7 @@ def initialize_extensions(app: DifyApp):
ext_forward_refs,
ext_hosting_provider,
ext_import_modules,
ext_knowledge_fs_observability,
ext_logging,
ext_login,
ext_logstore,
@@ -204,6 +205,7 @@ def initialize_extensions(app: DifyApp):
ext_enterprise_telemetry,
ext_request_logging,
ext_session_factory,
ext_knowledge_fs_observability,
ext_oauth_bearer,
]
for ext in extensions:
+2
View File
@@ -10,6 +10,7 @@ from .data_migration import (
import_migration_data,
migration_data_wizard,
)
from .knowledge_fs import knowledge_fs_control_space
from .plugin import (
backfill_plugin_auto_upgrade,
extract_plugins,
@@ -75,6 +76,7 @@ __all__ = [
"import_migration_data",
"install_plugins",
"install_rag_pipeline_plugins",
"knowledge_fs_control_space",
"legacy_model_types",
"migrate_annotation_vector_database",
"migrate_data_for_plugin",
+604
View File
@@ -0,0 +1,604 @@
"""Operator commands for the independent KnowledgeFS control-plane."""
from __future__ import annotations
import json
from collections.abc import Callable
from datetime import datetime
from functools import partial
from pathlib import Path
import click
from pydantic import BaseModel, ValidationError
from core.db.session_factory import session_factory
from services.knowledge_fs.cleanup import (
CleanupApprovalInput,
CleanupCompletionEvidenceInput,
CleanupReadinessEvidenceInput,
CleanupStartInput,
KnowledgeFSCleanupError,
KnowledgeFSCleanupService,
)
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
from services.knowledge_fs.control_space_management import (
KnowledgeFSControlSpaceManagementService,
KnowledgeFSControlSpaceRegistration,
)
from services.knowledge_fs.cutover import (
CutoverSmokeResultsInput,
FinalDeltaInput,
KnowledgeFSCutoverError,
KnowledgeFSWorkspaceCutoverService,
LegacyDependencyInput,
QuarantineResolutionInput,
ShadowAuthorizationObservationInput,
ShadowCompletionInput,
WorkspaceInventoryInput,
)
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
@click.group("knowledge-fs-control-space")
def knowledge_fs_control_space() -> None:
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
@knowledge_fs_control_space.command("dry-run")
@click.option("--tenant-id", default=None)
def dry_run(tenant_id: str | None) -> None:
report = _management_service().dry_run(tenant_id=tenant_id)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("inventory")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
def inventory(input_path: Path, apply: bool) -> None:
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.inventory, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("register")
@click.option("--tenant-id", required=True)
@click.option("--owner-account-id", required=True)
@click.option("--provisioning-key", required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def register(
tenant_id: str,
owner_account_id: str,
provisioning_key: str,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space, replayed = _management_service().register(
KnowledgeFSControlSpaceRegistration(
tenant_id,
owner_account_id,
provisioning_key,
knowledge_space_id,
knowledge_space_revision,
)
)
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
@knowledge_fs_control_space.command("backfill")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
def backfill(input_path: Path, apply: bool) -> None:
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
service = _cutover_service()
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
report = _operator_call(partial(service.backfill, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("quarantine-resolve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
def quarantine_resolve(input_path: Path, apply: bool) -> None:
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
service = _cutover_service()
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-start")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
service = _cutover_service()
_operator_call(
lambda: service.begin_shadow(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at) if started_at is not None else None,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-report")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
def shadow_report(input_path: Path, apply: bool) -> None:
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("shadow-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
def shadow_complete(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("issue-approve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_issue_fail_closed(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("issue-resolve")
@click.option("--tenant-id", required=True)
@click.option("--issue-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_issue(
tenant_id=tenant_id,
issue_key=issue_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-approve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "approved_at", required=True)
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
_operator_call(
lambda: _cutover_service().approve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
approved_at=_parse_timestamp(approved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("shadow-resolve")
@click.option("--tenant-id", required=True)
@click.option("--diff-key", required=True)
@click.option("--account-id", required=True)
@click.option("--at", "resolved_at", required=True)
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
_operator_call(
lambda: _cutover_service().resolve_shadow_diff(
tenant_id=tenant_id,
diff_key=diff_key,
account_id=account_id,
resolved_at=_parse_timestamp(resolved_at),
)
)
_echo_json(_cutover_service().status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("legacy-dashboard")
@click.option("--tenant-id", required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--checked-at", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
def legacy_dashboard(
tenant_id: str,
input_path: Path,
checked_at: str,
expected_cas_version: int | None,
apply: bool,
) -> None:
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
report = _operator_call(
lambda: _cutover_service().legacy_dependency_dashboard(
tenant_id=tenant_id,
dependencies=dependencies,
expected_cas_version=expected_cas_version,
checked_at=_parse_timestamp(checked_at),
apply=apply,
)
)
_echo_json(report._asdict())
@knowledge_fs_control_space.command("legacy-check")
@click.option("--tenant-id", required=True)
def legacy_check(tenant_id: str) -> None:
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
passed = (
bool(status_report["legacy_dependency_ready"])
and status_report["open_issues"] == 0
and status_report["unresolved_cutover_quarantine"] == 0
)
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
if not passed:
raise click.exceptions.Exit(1)
@knowledge_fs_control_space.command("freeze")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "freeze_at", required=True)
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.freeze(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
freeze_at=_parse_timestamp(freeze_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("final-delta")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def final_delta(input_path: Path) -> None:
payload = _read_one_jsonl(input_path, FinalDeltaInput)
service = _cutover_service()
_operator_call(lambda: service.apply_final_delta(payload))
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
@knowledge_fs_control_space.command("cutover")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "cutover_at", required=True)
@click.option("--rollback-cutoff-at", required=True)
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.cutover(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
cutover_at=_parse_timestamp(cutover_at),
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("smoke")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
service = _cutover_service()
_operator_call(
lambda: service.record_smoke_results(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
results=results,
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("observe")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--started-at", default=None)
@click.option("--window-ends-at", default=None)
@click.option("--maximum-task-expires-at", default=None)
@click.option("--observed-at", default=None)
def observe(
tenant_id: str,
expected_cas_version: int,
started_at: str | None,
window_ends_at: str | None,
maximum_task_expires_at: str | None,
observed_at: str | None,
) -> None:
service = _cutover_service()
if observed_at is not None:
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
raise click.UsageError("--observed-at cannot be combined with observation start options")
_operator_call(
lambda: service.complete_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
observed_at=_parse_timestamp(observed_at),
)
)
else:
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
raise click.UsageError(
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
)
_operator_call(
lambda: service.begin_observation(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
started_at=_parse_timestamp(started_at),
window_ends_at=_parse_timestamp(window_ends_at),
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("rollback")
@click.option("--tenant-id", required=True)
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
@click.option("--at", "rolled_back_at", required=True)
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
service = _cutover_service()
_operator_call(
lambda: service.rollback(
tenant_id=tenant_id,
expected_cas_version=expected_cas_version,
rolled_back_at=_parse_timestamp(rolled_back_at),
)
)
_echo_json(service.status(tenant_id=tenant_id))
@knowledge_fs_control_space.command("status")
@click.option("--tenant-id", required=True)
def status(tenant_id: str) -> None:
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
@knowledge_fs_control_space.command("cleanup-request")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
def cleanup_request(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-approve")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
def cleanup_approve(input_path: Path, apply: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-start")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
@click.option(
"--acknowledge-irreversible",
is_flag=True,
default=False,
help="Required with --apply; confirms rollback will be permanently closed.",
)
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupStartInput)
if apply and not acknowledge_irreversible:
raise click.UsageError("--apply requires --acknowledge-irreversible")
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-complete")
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
@click.option(
"--acknowledge-executed",
is_flag=True,
default=False,
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
)
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
if apply and not acknowledge_executed:
raise click.UsageError("--apply requires --acknowledge-executed")
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
_echo_json(report._asdict())
@knowledge_fs_control_space.command("cleanup-status")
@click.option("--tenant-id", required=True)
@click.option("--request-id", required=True)
def cleanup_status(tenant_id: str, request_id: str) -> None:
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
@knowledge_fs_control_space.command("repair")
@click.option("--tenant-id", required=True)
@click.option("--control-space-id", required=True)
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
@click.option("--knowledge-space-id", required=True)
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
def repair(
tenant_id: str,
control_space_id: str,
expected_resource_version: int,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> None:
control_space = _management_service().repair_registration(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=expected_resource_version,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
@knowledge_fs_control_space.command("orphan-report")
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
def orphan_report(limit: int) -> None:
report = KnowledgeFSOrphanReconciler(
session_factory.get_session_maker(),
get_knowledge_fs_lifecycle_remote(),
).reconcile(limit=limit, apply_repairs=False)
click.echo(json.dumps(report._asdict(), sort_keys=True))
@knowledge_fs_control_space.command("workspace-delete-request")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
if not apply:
report = _management_service().dry_run(tenant_id=tenant_id)
_echo_json(
{
"apply": False,
"by_state": report.by_state,
"tenant_id": tenant_id,
"total": report.total,
}
)
return
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
_echo_json(
{
"apply": True,
"control_space_ids": [result.control_space.id for result in results],
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
"tenant_id": tenant_id,
}
)
@knowledge_fs_control_space.command("workspace-delete-finalize")
@click.option("--tenant-id", required=True)
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
@click.option(
"--acknowledge-control-plane-purge",
is_flag=True,
default=False,
help="Required with --apply after every remote Space has reached deleted.",
)
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
"""Release the Workspace FK only after all remote deletions are terminal."""
service = _lifecycle_service()
if not apply:
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
return
if not acknowledge_control_plane_purge:
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
def _read_jsonl[InputT: BaseModel](
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
) -> tuple[InputT, ...]:
records: list[InputT] = []
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
records.append(input_type.model_validate_json(line))
except ValidationError as exc:
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
if not records and not allow_empty:
raise click.ClickException("strict JSONL input must contain at least one record")
return tuple(records)
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
records = _read_jsonl(input_path, input_type)
if len(records) != 1:
raise click.ClickException("this command requires exactly one JSONL record")
return records[0]
def _parse_timestamp(value: str) -> datetime:
try:
parsed = datetime.fromisoformat(value)
except ValueError as exc:
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
if parsed.tzinfo is None:
raise click.ClickException("operator timestamps must include an explicit timezone")
return parsed
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCutoverError as exc:
raise click.ClickException(str(exc)) from exc
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSCleanupError as exc:
raise click.ClickException(str(exc)) from exc
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
try:
return operation()
except KnowledgeFSControlSpaceLifecycleError as exc:
raise click.ClickException(str(exc)) from exc
def _echo_json(payload: object) -> None:
click.echo(json.dumps(payload, default=str, sort_keys=True))
def _management_service() -> KnowledgeFSControlSpaceManagementService:
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
return KnowledgeFSWorkspaceCutoverService(
session_factory.get_session_maker(),
remote_factory=get_knowledge_fs_lifecycle_remote,
)
def _cleanup_service() -> KnowledgeFSCleanupService:
return KnowledgeFSCleanupService(session_factory.get_session_maker())
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
__all__ = ["knowledge_fs_control_space"]
+90 -20
View File
@@ -1,30 +1,68 @@
"""Configuration for the optional KnowledgeFS Console bridge."""
"""Configuration for the optional KnowledgeFS control-plane integration."""
from ipaddress import ip_address
from urllib.parse import urlsplit
from pydantic import Field, PositiveFloat, SecretStr, field_validator, model_validator
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
from pydantic_settings import BaseSettings
class KnowledgeFSConfig(BaseSettings):
"""Server-only settings for the KnowledgeFS production connection."""
"""Server-only KnowledgeFS connection and rollout settings."""
KNOWLEDGE_FS_ENABLED: bool = Field(
default=False,
description="Enable the private KnowledgeFS Console bridge.",
)
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
KNOWLEDGE_FS_JWT_SECRET: SecretStr | None = Field(
default=None,
min_length=32,
description="Shared secret used to sign short-lived KnowledgeFS service JWTs.",
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
default=False,
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
)
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS: PositiveFloat = Field(default=300.0, le=3600.0, allow_inf_nan=False)
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
default=False,
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
)
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
default=False,
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
)
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
KNOWLEDGE_FS_DIRECT_ORIGIN: str | None = Field(
default=None,
description="Public KnowledgeFS origin returned with direct upload capabilities.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
default=False,
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
)
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
default=None,
description="Identifier for the current asymmetric Capability v2 signing key.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
default=None,
description="Server-only PEM for the current Capability v2 RSA signing key.",
)
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
default=None,
description="Optional public-only JWKS JSON retained during key rotation overlap.",
)
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
@field_validator(
"KNOWLEDGE_FS_BASE_URL",
"KNOWLEDGE_FS_JWT_SECRET",
"KNOWLEDGE_FS_DIRECT_ORIGIN",
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
mode="before",
)
@classmethod
@@ -40,25 +78,57 @@ class KnowledgeFSConfig(BaseSettings):
@field_validator("KNOWLEDGE_FS_BASE_URL")
@classmethod
def validate_base_url(cls, value: str | None) -> str | None:
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
@field_validator("KNOWLEDGE_FS_DIRECT_ORIGIN")
@classmethod
def validate_direct_origin(cls, value: str | None) -> str | None:
return cls._validate_origin(value, name="KNOWLEDGE_FS_DIRECT_ORIGIN")
@classmethod
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
if value is None:
return None
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("KNOWLEDGE_FS_BASE_URL must be an absolute HTTP(S) URL")
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
try:
_ = parsed.port
except ValueError as exc:
raise ValueError("KNOWLEDGE_FS_BASE_URL must include a valid port") from exc
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("KNOWLEDGE_FS_BASE_URL must not include credentials, query, or fragment")
raise ValueError(f"{name} must include a valid port") from exc
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
return value.rstrip("/")
@model_validator(mode="after")
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
if not self.KNOWLEDGE_FS_ENABLED:
return self
if bool(self.KNOWLEDGE_FS_BASE_URL) != bool(self.KNOWLEDGE_FS_JWT_SECRET):
raise ValueError("KNOWLEDGE_FS_BASE_URL and KNOWLEDGE_FS_JWT_SECRET must be configured together")
if not self.KNOWLEDGE_FS_BASE_URL:
raise ValueError("KnowledgeFS connection settings are required when the integration is enabled")
if str(getattr(self, "DEPLOY_ENV", "")).strip().upper() == "PRODUCTION":
for name, value in (
("KNOWLEDGE_FS_BASE_URL", self.KNOWLEDGE_FS_BASE_URL),
("KNOWLEDGE_FS_DIRECT_ORIGIN", self.KNOWLEDGE_FS_DIRECT_ORIGIN),
):
if value and not self._is_secure_or_loopback_origin(value):
raise ValueError(f"{name} must use HTTPS in production unless it targets loopback")
if self.KNOWLEDGE_FS_ENABLED:
if not self.KNOWLEDGE_FS_BASE_URL:
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
):
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
return self
@staticmethod
def _is_secure_or_loopback_origin(value: str) -> bool:
parsed = urlsplit(value)
if parsed.scheme == "https":
return True
hostname = (parsed.hostname or "").rstrip(".").lower()
if hostname == "localhost":
return True
try:
return ip_address(hostname).is_loopback
except ValueError:
return False
+2 -2
View File
@@ -38,7 +38,6 @@ from . import (
feature,
human_input_form,
init_validate,
knowledge_fs_proxy,
notification,
ping,
setup,
@@ -126,6 +125,7 @@ from .explore import (
saved_message,
trial,
)
from .knowledge_fs import resources as knowledge_fs_resources
from .snippets import snippet_workflow, snippet_workflow_draft_variable
from .socketio import workflow as socketio_workflow
@@ -196,7 +196,7 @@ __all__ = [
"human_input_form",
"init_validate",
"installed_app",
"knowledge_fs_proxy",
"knowledge_fs_resources",
"load_balancing_config",
"login",
"mcp_server",
@@ -0,0 +1,5 @@
"""Typed Dify-owned KnowledgeFS Console product API."""
from . import resources
__all__ = ["resources"]
@@ -0,0 +1,56 @@
"""Stable, non-enumerating Console error contract for KnowledgeFS."""
from libs.exception import BaseHTTPException
class KnowledgeFSSpaceNotFoundHTTPError(BaseHTTPException):
error_code = "knowledge_fs_space_not_found"
description = "KnowledgeFS space was not found."
code = 404
class KnowledgeFSOperationUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_operation_unavailable"
description = "KnowledgeFS operation is not available."
code = 503
class KnowledgeFSUpstreamUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_upstream_unavailable"
description = "KnowledgeFS is unavailable."
code = 502
class KnowledgeFSInvalidRequestHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_request"
description = "KnowledgeFS request is invalid."
code = 400
class KnowledgeFSAccessDeniedHTTPError(BaseHTTPException):
error_code = "knowledge_fs_access_denied"
description = "KnowledgeFS operation is not allowed."
code = 403
class KnowledgeFSRateLimitHTTPError(BaseHTTPException):
error_code = "knowledge_fs_rate_limit_exceeded"
description = "KnowledgeFS operation rate limit exceeded."
code = 429
class KnowledgeFSQuotaExceededHTTPError(BaseHTTPException):
error_code = "knowledge_fs_quota_exceeded"
description = "KnowledgeFS operation quota exceeded."
code = 403
__all__ = [
"KnowledgeFSAccessDeniedHTTPError",
"KnowledgeFSInvalidRequestHTTPError",
"KnowledgeFSOperationUnavailableHTTPError",
"KnowledgeFSQuotaExceededHTTPError",
"KnowledgeFSRateLimitHTTPError",
"KnowledgeFSSpaceNotFoundHTTPError",
"KnowledgeFSUpstreamUnavailableHTTPError",
]
File diff suppressed because it is too large Load Diff
@@ -1,332 +0,0 @@
"""Authenticated transport adapter for the Console-to-KnowledgeFS proxy.
These raw Blueprint routes deliberately stay outside Dify's OpenAPI surface:
KnowledgeFS owns the wire contract consumed by the frontend. The catch-all path
avoids resource-specific Dify controllers, while the forwarding module consumes
only the operations explicitly enabled by Dify's product registry. The registry
can be validated explicitly against the pinned KnowledgeFS contract during development.
Console auth and contract-specific dataset RBAC run before forwarding. Request
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
SSE responses remain streaming with a bounded idle read timeout, and only safe
response headers are exposed. Upstream 401 responses become 502 so they cannot
trigger Dify browser-session recovery; resource-level 403 responses remain 403.
"""
from __future__ import annotations
import logging
from collections.abc import Callable, Iterator
from functools import wraps
from http import HTTPStatus
from typing import NoReturn, cast
import httpx
from flask import Response, request, stream_with_context
from flask.typing import ResponseReturnValue
from werkzeug.exceptions import (
BadGateway,
Forbidden,
GatewayTimeout,
NotFound,
RequestEntityTooLarge,
ServiceUnavailable,
)
from configs import dify_config
from controllers.console import api, bp
from controllers.console.wraps import (
account_initialization_required,
cloud_edition_billing_rate_limit_check,
setup_required,
)
from core.helper import ssrf_proxy
from libs.login import current_account_with_tenant, login_required
from services.knowledge_fs_proxy import (
KnowledgeFSAccessDeniedError,
KnowledgeFSConfigurationError,
KnowledgeFSMethod,
KnowledgeFSRouteNotAllowedError,
KnowledgeFSTimeoutError,
KnowledgeFSTransportError,
KnowledgeFSUpstreamResponse,
authorize_knowledge_fs_request,
get_knowledge_fs_operation,
proxy_knowledge_fs_request,
)
logger = logging.getLogger(__name__)
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
_RESPONSE_HEADER_ALLOWLIST = (
"Cache-Control",
"Content-Disposition",
"Content-Type",
"Retry-After",
"X-Trace-Id",
)
_RESPONSE_HEADER_DENYLIST = frozenset(
{
"authorization",
"connection",
"cookie",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"set-cookie",
"te",
"trailer",
"transfer-encoding",
"upgrade",
}
)
def _console_api_errors[**P](
view: Callable[P, ResponseReturnValue],
) -> Callable[P, ResponseReturnValue]:
"""Route raw Blueprint exceptions through the Console API JSON handlers."""
@wraps(view)
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
try:
return view(*args, **kwargs)
except Exception as exc:
return api.handle_error(exc)
return decorated
def _knowledge_fs_enabled[**P](
view: Callable[P, ResponseReturnValue],
) -> Callable[P, ResponseReturnValue]:
"""Hide the complete KnowledgeFS route surface while the bridge is disabled."""
@wraps(view)
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
if not dify_config.KNOWLEDGE_FS_ENABLED:
raise NotFound()
return view(*args, **kwargs)
return decorated
def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
"""Map forwarding failures to the stable Console HTTP error surface."""
if isinstance(exc, KnowledgeFSRouteNotAllowedError):
raise NotFound() from exc
if isinstance(exc, KnowledgeFSAccessDeniedError):
raise Forbidden() from exc
if isinstance(exc, KnowledgeFSConfigurationError):
logger.error("KnowledgeFS request was blocked by invalid configuration for tenant_id=%s", tenant_id)
raise ServiceUnavailable("KnowledgeFS integration is misconfigured") from exc
if isinstance(exc, KnowledgeFSTimeoutError):
raise GatewayTimeout("KnowledgeFS request timed out") from exc
if isinstance(exc, KnowledgeFSTransportError):
logger.warning("KnowledgeFS transport request failed for tenant_id=%s", tenant_id)
raise BadGateway("KnowledgeFS is unavailable") from exc
raise exc
def _knowledge_fs_operation_access_required(
view: Callable[[KnowledgeFSMethod, str], ResponseReturnValue],
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
"""Authorize one declared operation before billing and request-body work."""
@wraps(view)
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
try:
operation = get_knowledge_fs_operation(method, upstream_path)
except KnowledgeFSRouteNotAllowedError as exc:
raise NotFound() from exc
current_user, tenant_id = current_account_with_tenant()
try:
authorize_knowledge_fs_request(
account=current_user,
tenant_id=tenant_id,
operation=operation,
)
except KnowledgeFSAccessDeniedError as exc:
_translate_proxy_error(exc, tenant_id=tenant_id)
return view(method, upstream_path)
return decorated
def _request_body() -> bytes:
"""Read the raw body up to the proxy limit or raise RequestEntityTooLarge."""
body = request.stream.read(_MAX_PROXY_BODY_BYTES + 1)
if len(body) > _MAX_PROXY_BODY_BYTES:
raise RequestEntityTooLarge("KnowledgeFS proxy request body is too large")
return body
def _stream_response_body(
upstream: httpx.Response,
*,
tenant_id: str,
max_response_bytes: int,
) -> Iterator[bytes]:
"""Yield one bounded SSE response and always release its pooled connection."""
total_bytes = 0
try:
for chunk in upstream.iter_bytes():
total_bytes += len(chunk)
if total_bytes > max_response_bytes:
logger.warning("KnowledgeFS stream exceeded the proxy limit for tenant_id=%s", tenant_id)
raise ssrf_proxy.ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
yield chunk
finally:
upstream.close()
def _proxy_response(
upstream_result: KnowledgeFSUpstreamResponse,
*,
tenant_id: str,
contract_response_headers: tuple[str, ...],
max_response_bytes: int,
) -> Response:
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
Raises:
BadGateway: KnowledgeFS rejects the configured server credential.
Forbidden: KnowledgeFS denies the account access to the requested resource.
"""
upstream = upstream_result.response
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
upstream.close()
logger.error(
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
upstream.status_code,
tenant_id,
)
raise BadGateway("KnowledgeFS authentication failed")
if upstream.status_code == HTTPStatus.FORBIDDEN:
upstream.close()
raise Forbidden()
allowed_header_names = dict.fromkeys(
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
)
headers = {
name: value
for name in allowed_header_names
if name not in _RESPONSE_HEADER_DENYLIST
if (value := upstream.headers.get(name)) is not None
}
if upstream_result.response_kind == "stream":
response = Response(
stream_with_context( # pyrefly: ignore[no-matching-overload]
_stream_response_body(
upstream,
tenant_id=tenant_id,
max_response_bytes=max_response_bytes,
)
),
status=upstream.status_code,
headers=headers,
)
response.call_on_close(upstream.close)
return response
try:
content = upstream.content
finally:
upstream.close()
return Response(content, status=upstream.status_code, headers=headers)
def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
"""Forward the current raw request and return its filtered upstream response.
The call performs one outbound KnowledgeFS request. Integration failures are
converted to Console HTTP exceptions for the outer JSON error adapter.
"""
if not dify_config.KNOWLEDGE_FS_ENABLED:
raise NotFound()
current_user, tenant_id = current_account_with_tenant()
try:
proxy_result = proxy_knowledge_fs_request(
account=current_user,
method=method,
path=upstream_path,
tenant_id=tenant_id,
accept=request.headers.get("Accept"),
content_type=request.content_type,
query=request.query_string or None,
body=_request_body() if method != "GET" else None,
request_headers=request.headers,
)
except (
KnowledgeFSConfigurationError,
KnowledgeFSAccessDeniedError,
KnowledgeFSRouteNotAllowedError,
KnowledgeFSTimeoutError,
KnowledgeFSTransportError,
) as exc:
_translate_proxy_error(exc, tenant_id=tenant_id)
return _proxy_response(
proxy_result,
tenant_id=tenant_id,
contract_response_headers=proxy_result.operation.response_headers,
max_response_bytes=proxy_result.operation.max_response_bytes,
)
@_knowledge_fs_enabled
@_knowledge_fs_operation_access_required
@cloud_edition_billing_rate_limit_check("knowledge")
def _proxy_knowledge_fs_non_get(
method: KnowledgeFSMethod,
upstream_path: str,
) -> ResponseReturnValue:
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
return _proxy_request(method, upstream_path)
@bp.route(
"/knowledge-fs/<path:upstream_path>",
methods=["GET", "OPTIONS"],
provide_automatic_options=False,
)
@_console_api_errors
@_knowledge_fs_enabled
@setup_required
@login_required
@account_initialization_required
def proxy_knowledge_fs_get(upstream_path: str) -> ResponseReturnValue:
"""Forward one authenticated, dataset-readable GET request.
Args:
upstream_path: Relative KFS path captured after the Console proxy prefix.
Returns:
The filtered raw KnowledgeFS response or a Console JSON error response.
"""
if request.method != "GET":
raise NotFound()
return _proxy_request("GET", upstream_path)
@bp.route(
"/knowledge-fs/<path:upstream_path>",
methods=["DELETE", "PATCH", "POST", "PUT"],
provide_automatic_options=False,
)
@_console_api_errors
@_knowledge_fs_enabled
@setup_required
@login_required
@account_initialization_required
def proxy_knowledge_fs_write(upstream_path: str) -> ResponseReturnValue:
"""Forward one authenticated non-GET request under its contract access policy.
Args:
upstream_path: Relative KFS path captured after the Console proxy prefix.
Returns:
The filtered raw KnowledgeFS response or a Console JSON error response.
"""
method = cast(KnowledgeFSMethod, request.method)
return _proxy_knowledge_fs_non_get(method, upstream_path)
+14 -3
View File
@@ -346,7 +346,13 @@ class RBACRoleItemApi(Resource):
def put(self, role_id):
tenant_id, account_id = _current_ids()
request = _payload(_RoleUpsertRequest)
role = svc.RBACService.Roles.update(tenant_id, account_id, str(role_id), request.to_mutation())
role = svc.RBACService.KnowledgeFSRoleMutations.update_role(
tenant_id,
account_id,
str(role_id),
request.to_mutation(),
session=db.session(),
)
return _dump(role)
@login_required
@@ -356,7 +362,12 @@ class RBACRoleItemApi(Resource):
@console_ns.response(200, "Success", console_ns.models[svc.RBACRole.__name__])
def delete(self, role_id):
tenant_id, account_id = _current_ids()
svc.RBACService.Roles.delete(tenant_id, account_id, str(role_id))
svc.RBACService.KnowledgeFSRoleMutations.delete_role(
tenant_id,
account_id,
str(role_id),
session=db.session(),
)
return {"result": "success"}
@@ -915,7 +926,7 @@ class RBACMemberRolesApi(Resource):
tenant_id, account_id = _current_ids()
request = _payload(_ReplaceMemberRolesRequest)
return _dump(
svc.RBACService.MemberRoles.replace(
svc.RBACService.KnowledgeFSRoleMutations.replace_member_roles(
tenant_id,
account_id,
str(member_id),
@@ -8,6 +8,7 @@ from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
from controllers.inner_api.wraps import plugin_inner_api_only
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
from core.plugin.backwards_invocation.datasource import PluginDatasourceBackwardsInvocation
from core.plugin.backwards_invocation.encrypt import PluginEncrypter
from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
@@ -15,10 +16,12 @@ from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
from core.plugin.entities.request import (
RequestFetchAppInfo,
RequestInvokeApp,
RequestInvokeDatasource,
RequestInvokeEncrypt,
RequestInvokeLLM,
RequestInvokeLLMWithStructuredOutput,
RequestInvokeModeration,
RequestInvokeMultimodalEmbedding,
RequestInvokeParameterExtractorNode,
RequestInvokeQuestionClassifierNode,
RequestInvokeRerank,
@@ -27,6 +30,7 @@ from core.plugin.entities.request import (
RequestInvokeTextEmbedding,
RequestInvokeTool,
RequestInvokeTTS,
RequestListModels,
RequestRequestDownloadFile,
RequestRequestUploadFile,
)
@@ -118,6 +122,36 @@ class PluginInvokeTextEmbeddingApi(Resource):
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/multimodal-embedding")
class PluginInvokeMultimodalEmbeddingApi(Resource):
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestInvokeMultimodalEmbedding)
@inner_api_ns.doc("plugin_invoke_multimodal_embedding")
@inner_api_ns.doc(description="Invoke multimodal embedding models through Dify model management")
@inner_api_ns.doc(
responses={
200: "Multimodal embedding successful",
401: "Unauthorized - invalid API key",
404: "Service not available",
}
)
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeMultimodalEmbedding):
try:
return jsonable_encoder(
BaseBackwardsInvocationResponse(
data=PluginModelBackwardsInvocation.invoke_multimodal_embedding(
user_id=user_model.id,
tenant=tenant_model,
payload=payload,
)
)
)
except Exception as e:
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/rerank")
class PluginInvokeRerankApi(Resource):
@get_user_tenant
@@ -144,6 +178,70 @@ class PluginInvokeRerankApi(Resource):
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/model-catalog")
class PluginModelCatalogApi(Resource):
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestListModels)
@inner_api_ns.doc("plugin_model_catalog")
@inner_api_ns.doc(description="List tenant-active models managed by Dify")
@inner_api_ns.doc(
responses={
200: "Model catalog lookup successful",
401: "Unauthorized - invalid API key",
404: "Service not available",
}
)
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestListModels):
try:
return jsonable_encoder(
BaseBackwardsInvocationResponse(
data=PluginModelBackwardsInvocation.list_models(
tenant_id=tenant_model.id,
user_id=user_model.id,
payload=payload,
)
)
)
except Exception as e:
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
@inner_api_ns.route("/invoke/datasource")
class PluginInvokeDatasourceApi(Resource):
"""Invoke an installed datasource with credentials resolved inside Dify."""
@get_user_tenant
@setup_required
@plugin_inner_api_only
@plugin_data(payload_type=RequestInvokeDatasource)
@inner_api_ns.doc("plugin_invoke_datasource")
@inner_api_ns.doc(description="Invoke datasource plugins through Dify credential management")
@inner_api_ns.doc(
responses={
200: "Datasource invocation successful (streaming response)",
401: "Unauthorized - invalid API key",
404: "Datasource provider, datasource, or credential not found",
}
)
def post(
self,
user_model: Account | EndUser,
tenant_model: Tenant,
payload: RequestInvokeDatasource,
):
response = PluginDatasourceBackwardsInvocation.invoke(
user_id=user_model.id,
tenant=tenant_model,
payload=payload,
)
return length_prefixed_response(
0xF,
PluginDatasourceBackwardsInvocation.convert_to_event_stream(response),
)
@inner_api_ns.route("/invoke/tts")
class PluginInvokeTTSApi(Resource):
@get_user_tenant
+2
View File
@@ -38,6 +38,7 @@ from .dataset import (
)
from .dataset.rag_pipeline import rag_pipeline_workflow
from .end_user import end_user
from .knowledge_fs import resources as knowledge_fs_resources
from .workspace import models
__all__ = [
@@ -54,6 +55,7 @@ __all__ = [
"hit_testing",
"human_input_form",
"index",
"knowledge_fs_resources",
"message",
"metadata",
"models",
@@ -0,0 +1,5 @@
"""KnowledgeFS-specific Service API authenticated by resource credentials."""
from . import resources
__all__ = ["resources"]
@@ -0,0 +1,49 @@
"""Stable KnowledgeFS Service API error contract."""
from libs.exception import BaseHTTPException
class KnowledgeFSInvalidCredentialHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_credential"
description = "Invalid KnowledgeFS service credential."
code = 401
class KnowledgeFSServiceOperationUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_operation_unavailable"
description = "KnowledgeFS operation is not available."
code = 503
class KnowledgeFSServiceUpstreamUnavailableHTTPError(BaseHTTPException):
error_code = "knowledge_fs_upstream_unavailable"
description = "KnowledgeFS is unavailable."
code = 502
class KnowledgeFSServiceInvalidRequestHTTPError(BaseHTTPException):
error_code = "knowledge_fs_invalid_request"
description = "KnowledgeFS request is invalid."
code = 400
class KnowledgeFSServiceRateLimitHTTPError(BaseHTTPException):
error_code = "knowledge_fs_rate_limit_exceeded"
description = "KnowledgeFS operation rate limit exceeded."
code = 429
class KnowledgeFSServiceQuotaExceededHTTPError(BaseHTTPException):
error_code = "knowledge_fs_quota_exceeded"
description = "KnowledgeFS operation quota exceeded."
code = 403
__all__ = [
"KnowledgeFSInvalidCredentialHTTPError",
"KnowledgeFSServiceInvalidRequestHTTPError",
"KnowledgeFSServiceOperationUnavailableHTTPError",
"KnowledgeFSServiceQuotaExceededHTTPError",
"KnowledgeFSServiceRateLimitHTTPError",
"KnowledgeFSServiceUpstreamUnavailableHTTPError",
]
File diff suppressed because it is too large Load Diff
+12
View File
@@ -14,7 +14,10 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
from core.app.apps.base_app_runner import AppRunner
from core.app.entities.app_invoke_entities import (
AgentChatAppGenerateEntity,
DifyRunContext,
InvokeFrom,
ModelConfigWithCredentialsEntity,
UserFrom,
)
from core.app.file_access import DatabaseFileAccessController
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
@@ -148,6 +151,15 @@ class BaseAgentRunner(AppRunner):
user_id=self.user_id,
invoke_from=self.application_generate_entity.invoke_from,
)
invoke_from = self.application_generate_entity.invoke_from
if isinstance(invoke_from, InvokeFrom):
tool_entity.runtime.dify_run_context = DifyRunContext(
tenant_id=self.tenant_id,
app_id=self.app_config.app_id,
user_id=self.user_id,
user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER),
invoke_from=invoke_from,
)
assert tool_entity.entity.description
message_tool = PromptMessageTool(
name=tool.tool_name,
@@ -159,6 +159,7 @@ class AgentAppRuntimeRequestBuilder:
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
agent_mode="agent_app",
trace_id=context.dify_context.trace_session_id,
),
# ENG-616: expand slash-menu mention tokens to canonical names so
# no frontend-internal {{#…#}} marker ever reaches the model.
@@ -38,6 +38,11 @@ class DatasourcePluginProviderController(ABC):
):
raise ToolProviderCredentialValidationError("Invalid credentials")
def validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
"""Validate credential shape and value against this installed provider declaration."""
self.validate_credentials_format(credentials)
self._validate_credentials(user_id, credentials)
@property
def provider_type(self) -> DatasourceProviderType:
"""
@@ -0,0 +1,110 @@
"""Datasource backward invocation through Dify's tenant-bound runtime.
This module is the internal service boundary used by trusted callers such as
KnowledgeFS. It resolves installed provider declarations and Dify-owned
credential references before reaching ``PluginDatasourceManager``; callers
must never provide raw datasource credentials or a plugin-daemon API key.
"""
from collections.abc import Generator
from typing import Any, cast
from pydantic import BaseModel
from core.datasource.datasource_manager import DatasourceManager
from core.datasource.entities.datasource_entities import (
OnlineDriveBrowseFilesRequest,
OnlineDriveDownloadFileRequest,
)
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
from core.plugin.entities.request import RequestInvokeDatasource
from models.account import Tenant
from models.provider_ids import DatasourceProviderID
from services.datasource_provider_service import DatasourceProviderService
class PluginDatasourceBackwardsInvocation(BaseBackwardsInvocation):
"""Resolve and invoke a datasource without exposing credential material to the caller."""
@classmethod
def invoke(
cls,
*,
user_id: str,
tenant: Tenant,
payload: RequestInvokeDatasource,
) -> Generator[BaseModel | dict[str, Any], None, None]:
"""Yield datasource messages for one validated inner-runtime request."""
provider_id = DatasourceProviderID(payload.provider)
canonical_provider_id = str(provider_id)
controller = DatasourceManager.get_datasource_plugin_provider(
provider_id=canonical_provider_id,
tenant_id=tenant.id,
datasource_type=payload.datasource_type,
)
if controller.entity.provider_type != payload.datasource_type:
raise ValueError("Datasource provider type mismatch")
# Resolving the datasource from the installed declaration prevents a caller
# from dispatching an arbitrary datasource name under a valid plugin ID.
runtime = controller.get_datasource(payload.datasource)
credentials = DatasourceProviderService().get_datasource_credentials(
tenant_id=tenant.id,
provider=provider_id.provider_name,
plugin_id=provider_id.plugin_id,
credential_id=payload.credential_id,
)
if controller.need_credentials and not credentials:
raise ValueError("Datasource credential not found")
if payload.operation == "validate_credentials":
controller.validate_credentials(user_id=user_id, credentials=credentials)
yield {"result": True}
return
runtime.runtime.credentials = credentials
provider_type = runtime.datasource_provider_type()
match payload.operation:
case "get_website_crawl":
website = cast(WebsiteCrawlDatasourcePlugin, runtime)
yield from website.get_website_crawl(
user_id=user_id,
datasource_parameters=payload.datasource_parameters,
provider_type=provider_type,
)
case "get_online_document_pages":
document = cast(OnlineDocumentDatasourcePlugin, runtime)
yield from document.get_online_document_pages(
user_id=user_id,
datasource_parameters=payload.datasource_parameters,
provider_type=provider_type,
)
case "get_online_document_page_content":
if payload.page is None:
raise ValueError("Online-document page input is required")
document = cast(OnlineDocumentDatasourcePlugin, runtime)
yield from document.get_online_document_page_content(
user_id=user_id,
datasource_parameters=payload.page,
provider_type=provider_type,
)
case "online_drive_browse_files":
drive = cast(OnlineDriveDatasourcePlugin, runtime)
yield from drive.online_drive_browse_files(
user_id=user_id,
request=OnlineDriveBrowseFilesRequest.model_validate(payload.request),
provider_type=provider_type,
)
case "online_drive_download_file":
drive = cast(OnlineDriveDatasourcePlugin, runtime)
yield from drive.online_drive_download_file(
user_id=user_id,
request=OnlineDriveDownloadFileRequest.model_validate(payload.request),
provider_type=provider_type,
)
case _:
raise ValueError(f"Unsupported datasource operation: {payload.operation}")
+109 -2
View File
@@ -1,22 +1,31 @@
import tempfile
from binascii import hexlify, unhexlify
from collections.abc import Generator
from collections.abc import Generator, Mapping
from enum import Enum
from typing import Any
from pydantic import BaseModel
from core.app.llm import deduct_llm_quota
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
from core.model_manager import ModelManager
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
from core.plugin.entities.request import (
InvokableModelCatalogItem,
InvokableModelCatalogPage,
RequestInvokeLLM,
RequestInvokeLLMWithStructuredOutput,
RequestInvokeModeration,
RequestInvokeMultimodalEmbedding,
RequestInvokeRerank,
RequestInvokeSpeech2Text,
RequestInvokeSummary,
RequestInvokeTextEmbedding,
RequestInvokeTTS,
RequestListModels,
)
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
from core.plugin.plugin_service import PluginService
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.utils.model_invocation_utils import ModelInvocationUtils
from graphon.model_runtime.entities.llm_entities import (
@@ -33,6 +42,20 @@ from graphon.model_runtime.entities.message_entities import (
)
from graphon.model_runtime.entities.model_entities import ModelType
from models.account import Tenant
from models.provider_ids import ModelProviderID
def _json_compatible(value: Any) -> Any:
"""Convert model-runtime metadata into stable JSON-compatible values."""
if isinstance(value, BaseModel):
return value.model_dump(mode="json")
if isinstance(value, Enum):
return value.value
if isinstance(value, Mapping):
return {str(_json_compatible(key)): _json_compatible(child) for key, child in value.items()}
if isinstance(value, list | tuple | set):
return [_json_compatible(child) for child in value]
return value
class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
@@ -183,7 +206,30 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
)
# invoke model
response = model_instance.invoke_text_embedding(texts=payload.texts)
response = model_instance.invoke_text_embedding(texts=payload.texts, input_type=payload.input_type)
return response
@classmethod
def invoke_multimodal_embedding(
cls,
user_id: str,
tenant: Tenant,
payload: RequestInvokeMultimodalEmbedding,
):
"""Invoke multimodal embedding through the tenant-bound model instance."""
model_instance = cls._get_bound_model_instance(
tenant_id=tenant.id,
user_id=user_id,
provider=payload.provider,
model_type=payload.model_type,
model=payload.model,
)
response = model_instance.invoke_multimodal_embedding(
multimodel_documents=[document.model_dump(exclude_none=True) for document in payload.documents],
input_type=payload.input_type,
)
return response
@@ -210,6 +256,67 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
return response
@classmethod
def list_models(
cls,
tenant_id: str,
user_id: str,
payload: RequestListModels,
) -> InvokableModelCatalogPage:
"""List only models that are active for the tenant's Dify configuration."""
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id, user_id=user_id)
active_models = provider_manager.get_configurations(tenant_id).get_models(
model_type=payload.model_type,
only_active=True,
)
installed_identities: dict[str, str] = {}
for plugin in PluginService.list(tenant_id):
existing = installed_identities.get(plugin.plugin_id)
if existing is not None and existing != plugin.plugin_unique_identifier:
raise ValueError(f"Ambiguous installed identity for model plugin {plugin.plugin_id}")
installed_identities[plugin.plugin_id] = plugin.plugin_unique_identifier
requested_provider = str(ModelProviderID(payload.provider)) if payload.provider else None
matched_models = [
model
for model in active_models
if (requested_provider is None or model.provider.provider == requested_provider)
and (payload.model is None or model.model == payload.model)
]
matched_models.sort(key=lambda model: (model.provider.provider, model.model))
page_models = matched_models[payload.offset : payload.offset + payload.limit]
items: list[InvokableModelCatalogItem] = []
for model in page_models:
provider_id = ModelProviderID(model.provider.provider)
unique_identifier = installed_identities.get(provider_id.plugin_id)
if unique_identifier is None:
raise ValueError(f"Installed identity not found for active model plugin {provider_id.plugin_id}")
items.append(
InvokableModelCatalogItem(
plugin_id=provider_id.plugin_id,
plugin_unique_identifier=unique_identifier,
provider=provider_id.provider_name,
model=model.model,
model_type=model.model_type,
capabilities={
"deprecated": model.deprecated,
"features": _json_compatible(model.features or []),
"fetchFrom": _json_compatible(model.fetch_from),
"modelProperties": _json_compatible(model.model_properties),
"modelType": model.model_type.value,
"status": _json_compatible(model.status),
},
)
)
next_offset = payload.offset + len(page_models)
return InvokableModelCatalogPage(
items=items,
next_offset=next_offset if next_offset < len(matched_models) else None,
)
@classmethod
def invoke_tts(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTTS):
"""
+113 -2
View File
@@ -6,6 +6,11 @@ from typing import Any, Literal
from flask import Response
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from core.datasource.entities.datasource_entities import (
DatasourceProviderType,
GetOnlineDocumentPageContentRequest,
)
from core.entities.embedding_type import EmbeddingInputType
from core.entities.provider_entities import BasicProviderConfig
from core.plugin.utils.http_parser import deserialize_response
from core.workflow.file_reference import is_canonical_file_reference
@@ -54,6 +59,61 @@ class RequestInvokeTool(BaseModel):
credential_id: str | None = None
DatasourceInvocationOperation = Literal[
"get_online_document_page_content",
"get_online_document_pages",
"get_website_crawl",
"online_drive_browse_files",
"online_drive_download_file",
"validate_credentials",
]
class RequestInvokeDatasource(BaseModel):
"""Invoke one installed datasource using a Dify-owned credential reference.
Raw credentials are intentionally not part of this contract. ``tenant_id`` and
``user_id`` are consumed by the inner-API request context, while the remaining
fields select an installed provider declaration and an operation-specific input.
"""
tenant_id: str = Field(min_length=1, max_length=512)
user_id: str = Field(min_length=1, max_length=512)
provider: str = Field(min_length=1, max_length=768)
datasource: str = Field(min_length=1, max_length=256)
datasource_type: DatasourceProviderType
credential_id: str = Field(min_length=1, max_length=512)
operation: DatasourceInvocationOperation
datasource_parameters: dict[str, Any] = Field(default_factory=dict)
page: GetOnlineDocumentPageContentRequest | None = None
request: dict[str, Any] | None = None
model_config = ConfigDict(extra="forbid")
@model_validator(mode="after")
def validate_operation_payload(self) -> "RequestInvokeDatasource":
expected_type = {
"get_online_document_page_content": DatasourceProviderType.ONLINE_DOCUMENT,
"get_online_document_pages": DatasourceProviderType.ONLINE_DOCUMENT,
"get_website_crawl": DatasourceProviderType.WEBSITE_CRAWL,
"online_drive_browse_files": DatasourceProviderType.ONLINE_DRIVE,
"online_drive_download_file": DatasourceProviderType.ONLINE_DRIVE,
"validate_credentials": self.datasource_type,
}[self.operation]
if self.datasource_type != expected_type:
raise ValueError(f"{self.operation} requires datasource_type {expected_type.value}")
page_required = self.operation == "get_online_document_page_content"
if page_required != (self.page is not None):
raise ValueError("page is required only for get_online_document_page_content")
request_required = self.operation in {"online_drive_browse_files", "online_drive_download_file"}
if request_required != (self.request is not None):
raise ValueError("request is required only for online-drive operations")
return self
class BaseRequestInvokeModel(BaseModel):
provider: str
model: str
@@ -115,6 +175,25 @@ class RequestInvokeTextEmbedding(BaseRequestInvokeModel):
model_type: ModelType = ModelType.TEXT_EMBEDDING
texts: list[str]
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
class MultimodalEmbeddingDocument(BaseModel):
"""A document accepted by a multimodal text-embedding model."""
content: str
content_type: str
file_id: str | None = None
model_config = ConfigDict(extra="forbid")
class RequestInvokeMultimodalEmbedding(BaseRequestInvokeModel):
"""Request to invoke a multimodal text-embedding model."""
model_type: ModelType = ModelType.TEXT_EMBEDDING
documents: list[MultimodalEmbeddingDocument] = Field(min_length=1)
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
class RequestInvokeRerank(BaseRequestInvokeModel):
@@ -125,8 +204,40 @@ class RequestInvokeRerank(BaseRequestInvokeModel):
model_type: ModelType = ModelType.RERANK
query: str
docs: list[str]
score_threshold: float
top_n: int
score_threshold: float | None = None
top_n: int | None = None
class RequestListModels(BaseModel):
"""Tenant-scoped query for models that Dify can invoke."""
model_type: Literal[ModelType.LLM, ModelType.TEXT_EMBEDDING, ModelType.RERANK]
provider: str | None = None
model: str | None = None
offset: int = Field(default=0, ge=0)
limit: int = Field(default=50, ge=1, le=100)
model_config = ConfigDict(protected_namespaces=())
class InvokableModelCatalogItem(BaseModel):
"""Installed identity and active Dify capability metadata for one model."""
plugin_id: str
plugin_unique_identifier: str
provider: str
model: str
model_type: ModelType
capabilities: dict[str, Any] = Field(default_factory=dict)
model_config = ConfigDict(protected_namespaces=())
class InvokableModelCatalogPage(BaseModel):
"""Offset page returned by the internal model catalog endpoint."""
items: list[InvokableModelCatalogItem] = Field(default_factory=list)
next_offset: int | None = None
class RequestInvokeTTS(BaseRequestInvokeModel):
+10
View File
@@ -10,6 +10,7 @@ class RBACResourceScope(StrEnum):
APP = "app"
DATASET = "dataset"
KNOWLEDGE_FS = "knowledge_space"
WORKSPACE = "workspace"
@@ -57,6 +58,15 @@ class RBACPermission(StrEnum):
DATASET_EXTERNAL_CONNECT = "dataset_external_connect"
DATASET_IMPORT_EXPORT_DSL = "dataset_import_export_dsl"
KNOWLEDGE_FS_READ = "knowledge_space_read"
KNOWLEDGE_FS_CREATE = "knowledge_space_create"
KNOWLEDGE_FS_EDIT = "knowledge_space_edit"
KNOWLEDGE_FS_DELETE = "knowledge_space_delete"
KNOWLEDGE_FS_ACCESS_CONFIG = "knowledge_space_access_config"
KNOWLEDGE_FS_API_KEY_MANAGE = "knowledge_space_api_key_manage"
KNOWLEDGE_FS_DOCUMENT_WRITE = "knowledge_space_document_write"
KNOWLEDGE_FS_QUERY = "knowledge_space_query"
WORKSPACE_MEMBER_MANAGE = "workspace_member_manage"
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
API_EXTENSION_MANAGE = "api_extension_manage"
+2 -1
View File
@@ -2,7 +2,7 @@ from typing import Any
from pydantic import BaseModel, Field
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
from core.plugin.entities.plugin_daemon import CredentialType
from core.tools.entities.tool_entities import ToolInvokeFrom
@@ -20,6 +20,7 @@ class ToolRuntime(BaseModel):
tool_id: str | None = None
invoke_from: InvokeFrom | None = None
tool_invoke_from: ToolInvokeFrom | None = None
dify_run_context: DifyRunContext | None = Field(default=None, exclude=True, repr=False)
credentials: dict[str, Any] = Field(default_factory=dict)
credential_type: CredentialType = Field(default=CredentialType.API_KEY)
runtime_parameters: dict[str, Any] = Field(default_factory=dict)
@@ -1,4 +1,5 @@
- audio
- code
- knowledge_fs
- time
- webscraper
@@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="none">
<rect width="64" height="64" rx="14" fill="#155EEF"/>
<path d="M17 17h20c5.5 0 10 4.5 10 10v20H27c-5.5 0-10-4.5-10-10V17Z" fill="white" fill-opacity=".96"/>
<path d="M26 27h12M26 34h12M26 41h7" stroke="#155EEF" stroke-width="4" stroke-linecap="round"/>
</svg>

After

Width:  |  Height:  |  Size: 340 B

@@ -0,0 +1,9 @@
from typing import Any, override
from core.tools.builtin_tool.provider import BuiltinToolProviderController
class KnowledgeFSProvider(BuiltinToolProviderController):
@override
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
_ = (user_id, credentials)
@@ -0,0 +1,13 @@
identity:
author: Dify
name: knowledge_fs
label:
en_US: KnowledgeFS
zh_Hans: KnowledgeFS
description:
en_US: Run explicitly bound KnowledgeFS operations from an Agent or Workflow.
zh_Hans: 从 Agent 或 Workflow 执行显式绑定的 KnowledgeFS 操作。
icon: icon.svg
tags:
- rag
@@ -0,0 +1,60 @@
from collections.abc import Generator
from typing import Any, override
from pydantic import ValidationError
from sqlalchemy.orm import Session, sessionmaker
from core.tools.builtin_tool.tool import BuiltinTool
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage
from core.tools.errors import ToolInvokeError
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
from services.knowledge_fs.app_execution_capability import KnowledgeResourceRef
from services.knowledge_fs.product_dto import KnowledgeFSResearchTaskCreatePayload
from services.knowledge_fs.runtime import create_knowledge_fs_runtime
class KnowledgeFSCreateResearchTaskTool(BuiltinTool):
@override
def _invoke(
self,
session: Session,
user_id: str,
tool_parameters: dict[str, Any],
conversation_id: str | None = None,
app_id: str | None = None,
message_id: str | None = None,
) -> Generator[ToolInvokeMessage, None, None]:
_ = (user_id, conversation_id, app_id, message_id)
run_context = self.runtime.dify_run_context
if run_context is None or self.runtime.tenant_id != run_context.tenant_id:
raise ToolInvokeError("KnowledgeFS requires a trusted Dify run context")
match self.runtime.tool_invoke_from:
case ToolInvokeFrom.AGENT:
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
case ToolInvokeFrom.WORKFLOW:
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
case _:
raise ToolInvokeError("KnowledgeFS is only available to Agent and Workflow callers")
try:
resource = KnowledgeResourceRef.model_validate(tool_parameters.get("resource"))
payload_data: dict[str, object] = {
"query": tool_parameters.get("query"),
}
mode = tool_parameters.get("mode")
if mode:
payload_data["mode"] = mode
payload = KnowledgeFSResearchTaskCreatePayload.model_validate(payload_data)
runtime = create_knowledge_fs_runtime(sessionmaker(bind=session.get_bind(), expire_on_commit=False))
response = runtime.app_capabilities.create_research_task(
run_context=run_context,
caller_kind=caller_kind,
resource=resource,
payload=payload,
)
except ToolInvokeError:
raise
except (ValidationError, RuntimeError, ValueError) as exc:
raise ToolInvokeError(str(exc)) from exc
yield self.create_json_message(response.model_dump(mode="json", by_alias=True))
@@ -0,0 +1,75 @@
identity:
name: create_research_task
author: Dify
label:
en_US: Create KnowledgeFS Research Task
zh_Hans: 创建 KnowledgeFS Research 任务
description:
human:
en_US: Create a Research task in an explicitly bound KnowledgeFS space.
zh_Hans: 在显式绑定的 KnowledgeFS 空间中创建 Research 任务。
llm: Create a durable research task using an explicitly configured KnowledgeFS resource.
parameters:
- name: resource
type: object
required: true
label:
en_US: KnowledgeFS resource
zh_Hans: KnowledgeFS 资源
human_description:
en_US: A typed KnowledgeFS control-space reference configured by the app author.
zh_Hans: 由应用作者配置的类型化 KnowledgeFS control-space 引用。
form: form
input_schema:
type: object
additionalProperties: false
properties:
kind:
type: string
const: knowledge_fs
control_space_id:
type: string
minLength: 1
required:
- kind
- control_space_id
- name: query
type: string
required: true
label:
en_US: Research query
zh_Hans: Research 查询
human_description:
en_US: The question the Research task should investigate.
zh_Hans: Research 任务需要调查的问题。
llm_description: The question to investigate with KnowledgeFS.
form: llm
- name: mode
type: select
required: false
label:
en_US: Mode
zh_Hans: 模式
human_description:
en_US: Optional KnowledgeFS retrieval mode.
zh_Hans: 可选的 KnowledgeFS 检索模式。
llm_description: Optional retrieval mode. Use auto unless the task needs a specific mode.
form: llm
options:
- value: auto
label:
en_US: Auto
zh_Hans: 自动
- value: fast
label:
en_US: Fast
zh_Hans: 快速
- value: deep
label:
en_US: Deep
zh_Hans: 深度
- value: research
label:
en_US: Research
zh_Hans: Research
+1
View File
@@ -493,6 +493,7 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):
self._run_context.invoke_from,
variable_pool,
)
tool_runtime.runtime.dify_run_context = self._run_context
except ToolNodeError:
raise
except Exception as exc:
@@ -250,6 +250,7 @@ class WorkflowAgentRuntimeRequestBuilder:
agent_config_version_kind="snapshot",
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
trace_id=context.dify_context.trace_session_id,
),
agent_soul_prompt=soul_prompt or None,
workflow_node_job_prompt=workflow_job_prompt,
+346
View File
@@ -0,0 +1,346 @@
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
The primary threshold aggregates statement and branch coverage for every Dify
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
that only contain narrow integration hooks are checked with changed-line
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import re
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import TypedDict, cast
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
{
"api/dev/check_knowledge_fs_coverage.py",
"api/dev/generate_knowledge_fs_contract.py",
"api/dev/knowledge_fs_product_contract.py",
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
}
)
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
logger = logging.getLogger(__name__)
class CoverageSummary(TypedDict):
"""Coverage.py counts required by the aggregate gate."""
covered_lines: int
num_statements: int
covered_branches: int
num_branches: int
class CoverageFile(TypedDict):
"""Per-file coverage data emitted by ``coverage json``."""
executed_lines: list[int]
missing_lines: list[int]
summary: CoverageSummary
class CoverageReport(TypedDict):
"""Relevant top-level shape of a coverage.py JSON report."""
files: dict[str, CoverageFile]
@dataclass(frozen=True, slots=True)
class CoverageTotals:
"""Covered and measurable units for one gate surface."""
covered: int
total: int
@property
def percent(self) -> float:
return 100.0 if self.total == 0 else self.covered * 100 / self.total
class CoverageGateError(RuntimeError):
"""Raised when coverage input is incomplete or below its threshold."""
def main() -> None:
"""Validate focused module coverage and changed integration glue."""
logging.basicConfig(level=logging.INFO, format="%(message)s")
parser = argparse.ArgumentParser()
parser.add_argument("--coverage-json", type=Path, required=True)
parser.add_argument("--glue-manifest", type=Path, required=True)
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
parser.add_argument("--base", default="")
parser.add_argument("--minimum", type=float, default=90.0)
parser.add_argument("--glue-minimum", type=float, default=90.0)
args = parser.parse_args()
workspace_root = args.workspace_root.resolve()
report = load_coverage_report(args.coverage_json)
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
base = resolve_diff_base(workspace_root, args.base)
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
glue_totals = validate_changed_glue_coverage(
report,
changed_lines=changed_lines,
minimum=args.glue_minimum,
)
logger.info(
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
core_totals.percent,
core_totals.covered,
core_totals.total,
glue_totals.percent,
glue_totals.covered,
glue_totals.total,
)
def load_coverage_report(path: Path) -> CoverageReport:
"""Load the detailed JSON report used by both coverage checks."""
if not path.is_file():
raise CoverageGateError(f"coverage JSON does not exist: {path}")
document = json.loads(path.read_text())
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
raise CoverageGateError(f"coverage JSON has no files object: {path}")
return cast(CoverageReport, document)
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
if not path.is_file():
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
try:
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
except UnicodeDecodeError as error:
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
if not paths:
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
if len(paths) != len(set(paths)):
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
invalid_paths = [
candidate
for candidate in paths
if not candidate.startswith("api/")
or not candidate.endswith(".py")
or not (workspace_root / candidate).is_file()
]
if invalid_paths:
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
return paths
def is_core_coverage_path(path: str) -> bool:
"""Return whether a repository-relative path belongs to the focused aggregate."""
if not path.endswith(".py"):
return False
if path in {
"api/commands/knowledge_fs.py",
"api/configs/extra/knowledge_fs_config.py",
"api/extensions/ext_knowledge_fs_observability.py",
"api/services/knowledge_fs_capability.py",
}:
return True
if path.startswith(
(
"api/controllers/console/knowledge_fs/",
"api/controllers/service_api/knowledge_fs/",
"api/core/tools/builtin_tool/providers/knowledge_fs/",
"api/services/knowledge_fs/",
)
):
return True
filename = path.rsplit("/", maxsplit=1)[-1]
return (
path.startswith("api/models/")
and filename.startswith("knowledge_fs")
or path.startswith("api/repositories/")
and "knowledge_fs" in filename
or path.startswith("api/tasks/")
and "knowledge_fs" in filename
)
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
"""Classify every KnowledgeFS-named production file or fail closed."""
named_paths = discover_knowledge_fs_production_paths(workspace_root)
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
if unclassified_paths:
raise CoverageGateError(
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
+ ", ".join(sorted(unclassified_paths))
)
if not core_paths:
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
return tuple(sorted(core_paths))
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
api_root = workspace_root / "api"
if not api_root.is_dir():
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
paths: set[str] = set()
for directory, child_directories, filenames in os.walk(api_root):
current_directory = Path(directory)
if current_directory == api_root:
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
for filename in filenames:
path = (current_directory / filename).relative_to(workspace_root).as_posix()
if filename.endswith(".py") and "knowledge_fs" in path:
paths.add(path)
if not paths:
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
return tuple(sorted(paths))
def validate_core_coverage(
report: CoverageReport,
*,
workspace_root: Path,
minimum: float,
) -> CoverageTotals:
"""Require the exact combined line-and-branch percentage for all core files."""
paths = discover_core_coverage_paths(workspace_root)
missing_paths = [path for path in paths if path not in report["files"]]
if missing_paths:
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
covered = 0
total = 0
for path in paths:
summary = report["files"][path]["summary"]
covered += summary["covered_lines"] + summary["covered_branches"]
total += summary["num_statements"] + summary["num_branches"]
if total == 0:
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
return totals
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
base = preferred.strip()
if not base or set(base) == {"0"}:
base = "HEAD^"
result = subprocess.run(
["git", "cat-file", "-e", f"{base}^{{commit}}"],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "commit is unavailable"
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
return base
def collect_changed_glue_lines(
workspace_root: Path,
base: str,
*,
glue_paths: tuple[str, ...],
) -> dict[str, set[int]]:
"""Return added line numbers in the narrow Dify modules touched by this integration."""
result = subprocess.run(
[
"git",
"diff",
"--no-ext-diff",
"--no-color",
"--unified=0",
base,
"--",
*glue_paths,
],
cwd=workspace_root,
check=False,
capture_output=True,
text=True,
)
if result.returncode != 0:
detail = result.stderr.strip() or "git diff failed"
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
return parse_added_lines(result.stdout)
def parse_added_lines(diff: str) -> dict[str, set[int]]:
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
changed_lines: dict[str, set[int]] = {}
current_path: str | None = None
current_line: int | None = None
for raw_line in diff.splitlines():
if raw_line.startswith("diff --git "):
current_line = None
continue
if raw_line.startswith("+++ "):
candidate = raw_line[4:]
current_path = candidate[2:] if candidate.startswith("b/") else None
if current_path is not None:
changed_lines.setdefault(current_path, set())
current_line = None
continue
if raw_line.startswith("@@ "):
match = HUNK_HEADER.match(raw_line)
current_line = int(match.group(1)) if match is not None else None
continue
if current_path is None or current_line is None:
continue
if raw_line.startswith("+"):
changed_lines[current_path].add(current_line)
current_line += 1
elif raw_line.startswith("-") or raw_line.startswith("\\"):
continue
else:
current_line += 1
return changed_lines
def validate_changed_glue_coverage(
report: CoverageReport,
*,
changed_lines: dict[str, set[int]],
minimum: float,
) -> CoverageTotals:
"""Require added executable glue lines to be exercised by the focused unit suite."""
covered = 0
total = 0
for path, lines in sorted(changed_lines.items()):
if not lines:
continue
file_coverage = report["files"].get(path)
if file_coverage is None:
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
executed_lines = set(file_coverage["executed_lines"])
executable_lines = executed_lines | set(file_coverage["missing_lines"])
changed_executable_lines = lines & executable_lines
covered += len(changed_executable_lines & executed_lines)
total += len(changed_executable_lines)
totals = CoverageTotals(covered=covered, total=total)
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
return totals
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
if totals.percent + 1e-12 < minimum:
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
if __name__ == "__main__":
main()
+534 -57
View File
@@ -1,7 +1,14 @@
"""Validate Dify Console KnowledgeFS declarations against a pinned OpenAPI document.
"""Pin the in-repository KnowledgeFS contract and validate every Dify product operation.
The OpenAPI document is exported only during explicit development validation. Runtime declarations live with Dify
product policy; this module validates their transport metadata without generating a complete operation catalog.
The lock is intentionally independent of the enclosing Dify commit: it records the staged ``knowledge-fs/`` tree,
the complete generated OpenAPI document, both explicit product-operation manifests, and the active Capability v2
profile and deterministic public-key vector. The full OpenAPI hash
covers request/response schemas, status codes, security, deprecation, and stream metadata. Field-level validation
cross-checks the Dify product registry, Python Capability issuer, TypeScript request guard, and exported OpenAPI;
each product operation must be ready or an explicit gap, and KFS-only activation remains explicitly internal.
Contract export reads the working tree only after proving it matches the staged KnowledgeFS index. This keeps the
OpenAPI bytes and auth manifest aligned with the exact subtree tree ID that will be reviewed and committed.
"""
from __future__ import annotations
@@ -13,17 +20,36 @@ import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Literal, TypedDict
from typing import Any, Literal, TypedDict, cast
import jwt
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
from jwt.algorithms import RSAAlgorithm
API_ROOT = Path(__file__).resolve().parents[1]
if str(API_ROOT) not in sys.path:
sys.path.insert(0, str(API_ROOT))
from dev.knowledge_fs_product_contract import (
capability_operation_runtime_contracts,
parse_capability_operation_policy,
parse_product_operation_gap_manifest,
parse_product_operation_manifest,
product_operation_runtime_contracts,
validate_product_operation_contracts,
)
WORKSPACE_ROOT = API_ROOT.parent
LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
KNOWLEDGE_FS_DIRECTORY = "knowledge-fs"
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
UPSTREAM_PROVENANCE_RELATIVE_PATH = Path("upstream-provenance.json")
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
LOCK_SCHEMA_VERSION = 5
class ContractDeclaration(TypedDict):
@@ -40,6 +66,18 @@ class ContractDeclaration(TypedDict):
response_media_types: tuple[str, ...]
class ContractLock(TypedDict):
"""Content-addressed contract inputs that must move together."""
schemaVersion: int
subtreeTree: str
openapiSha256: str
capabilityV2AuthManifestSha256: str
capabilityV2AuthTestVectorSha256: str
productOperationManifestSha256: str
productOperationGapManifestSha256: str
type DeclarationField = Literal[
"method",
"path",
@@ -64,59 +102,507 @@ DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
def main() -> None:
"""Update or verify the pin and validate Console declarations against its OpenAPI document."""
"""Update or verify the monorepo pin and validate Dify product declarations."""
parser = argparse.ArgumentParser()
mode = parser.add_mutually_exclusive_group()
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--check", action="store_true")
mode.add_argument("--update-lock", action="store_true")
parser.add_argument("--repository", type=Path, default=DEFAULT_REPOSITORY)
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
args = parser.parse_args()
repository = args.repository.resolve()
lock = json.loads(LOCK_PATH.read_text())
tracked_changes = run("git", "status", "--porcelain", "--untracked-files=no", cwd=repository).strip()
if tracked_changes:
raise RuntimeError("KnowledgeFS checkout must not contain tracked changes during contract export")
workspace_root = args.workspace_root.resolve()
knowledge_fs_root = workspace_root / KNOWLEDGE_FS_DIRECTORY
lock_path = workspace_root / LOCK_RELATIVE_PATH
capability_v2_auth_manifest_path = knowledge_fs_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH
capability_v2_auth_test_vector_path = knowledge_fs_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH
product_operations_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
product_operation_gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
upstream_provenance_path = knowledge_fs_root / UPSTREAM_PROVENANCE_RELATIVE_PATH
ensure_clean_knowledge_fs_worktree(workspace_root)
ensure_contract_inputs_exist(
knowledge_fs_root=knowledge_fs_root,
capability_v2_auth_manifest_path=capability_v2_auth_manifest_path,
capability_v2_auth_test_vector_path=capability_v2_auth_test_vector_path,
product_operations_path=product_operations_path,
product_operation_gaps_path=product_operation_gaps_path,
upstream_provenance_path=upstream_provenance_path,
)
commit = run("git", "rev-parse", "HEAD", cwd=repository).strip()
if not args.update_lock and commit != lock["commit"]:
raise RuntimeError(
f"KnowledgeFS checkout mismatch: expected {lock['commit']}, received {commit}. "
"Use the pinned commit or pass --update-lock intentionally."
)
subtree_tree = staged_subtree_tree(workspace_root)
capability_v2_auth_manifest_content = capability_v2_auth_manifest_path.read_bytes()
capability_v2_auth_test_vector_content = capability_v2_auth_test_vector_path.read_bytes()
product_operation_manifest_content = product_operations_path.read_bytes()
product_operation_gap_manifest_content = product_operation_gaps_path.read_bytes()
capability_v2_auth_manifest = load_json_object(capability_v2_auth_manifest_path)
validate_capability_v2_auth_manifest(capability_v2_auth_manifest)
validate_capability_v2_auth_test_vector(
load_json_object(capability_v2_auth_test_vector_path),
capability_v2_auth_manifest,
)
validate_upstream_provenance(load_json_object(upstream_provenance_path))
product_manifest = parse_product_operation_manifest(load_json_object(product_operations_path))
product_gap_manifest = parse_product_operation_gap_manifest(load_json_object(product_operation_gaps_path))
product_runtime_operations = product_operation_runtime_contracts()
capability_runtime_operations = capability_operation_runtime_contracts()
declarations = console_contract_declarations()
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
capability_policy_path = Path(directory) / "dify-capability-v2-operations.json"
subprocess.run(
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
cwd=repository,
cwd=knowledge_fs_root,
check=True,
)
subprocess.run(
["pnpm", "capability:export", "--", "--output", str(capability_policy_path)],
cwd=knowledge_fs_root,
check=True,
)
openapi_content = openapi_path.read_bytes()
openapi_sha256 = sha256(openapi_content)
if not args.update_lock and openapi_sha256 != lock["openapiSha256"]:
raise RuntimeError(
f"KnowledgeFS OpenAPI hash mismatch: expected {lock['openapiSha256']}, received {openapi_sha256}"
)
capability_policy = parse_capability_operation_policy(load_json_object(capability_policy_path))
document: dict[str, Any] = json.loads(openapi_content)
validate_declarations(document, console_contract_declarations())
validate_product_operation_contracts(
capability_operations=capability_runtime_operations,
capability_policy=capability_policy,
document=document,
gap_manifest=product_gap_manifest,
manifest=product_manifest,
product_operations=product_runtime_operations,
)
validate_declarations(document, declarations)
expected_lock: ContractLock = {
"schemaVersion": LOCK_SCHEMA_VERSION,
"subtreeTree": subtree_tree,
"openapiSha256": sha256(openapi_content),
"capabilityV2AuthManifestSha256": sha256(capability_v2_auth_manifest_content),
"capabilityV2AuthTestVectorSha256": sha256(capability_v2_auth_test_vector_content),
"productOperationManifestSha256": sha256(product_operation_manifest_content),
"productOperationGapManifestSha256": sha256(product_operation_gap_manifest_content),
}
if args.update_lock:
LOCK_PATH.write_text(
json.dumps(
{
"commit": commit,
"openapiSha256": openapi_sha256,
"repository": lock["repository"],
},
indent=2,
lock_path.write_text(json.dumps(expected_lock, indent=2) + "\n")
return
received_lock = parse_contract_lock(load_json_object(lock_path))
lock_fields = (
(
"capabilityV2AuthManifestSha256",
received_lock["capabilityV2AuthManifestSha256"],
expected_lock["capabilityV2AuthManifestSha256"],
),
(
"capabilityV2AuthTestVectorSha256",
received_lock["capabilityV2AuthTestVectorSha256"],
expected_lock["capabilityV2AuthTestVectorSha256"],
),
(
"productOperationManifestSha256",
received_lock["productOperationManifestSha256"],
expected_lock["productOperationManifestSha256"],
),
(
"productOperationGapManifestSha256",
received_lock["productOperationGapManifestSha256"],
expected_lock["productOperationGapManifestSha256"],
),
("schemaVersion", received_lock["schemaVersion"], expected_lock["schemaVersion"]),
("subtreeTree", received_lock["subtreeTree"], expected_lock["subtreeTree"]),
("openapiSha256", received_lock["openapiSha256"], expected_lock["openapiSha256"]),
)
for field, received_value, expected_value in lock_fields:
if received_value != expected_value:
raise RuntimeError(
f"KnowledgeFS contract lock field {field} drifted: "
f"expected {expected_value!r}, received {received_value!r}. "
"Run --update-lock intentionally after reviewing the staged subtree and contract changes."
)
+ "\n"
def ensure_contract_inputs_exist(
*,
knowledge_fs_root: Path,
capability_v2_auth_manifest_path: Path,
capability_v2_auth_test_vector_path: Path,
product_operations_path: Path,
product_operation_gaps_path: Path,
upstream_provenance_path: Path,
) -> None:
"""Fail with a stable error before invoking package tooling when a contract input is absent."""
required_paths = (
knowledge_fs_root / "package.json",
capability_v2_auth_manifest_path,
capability_v2_auth_test_vector_path,
product_operations_path,
product_operation_gaps_path,
upstream_provenance_path,
)
missing_paths = [path for path in required_paths if not path.is_file()]
if missing_paths:
missing = ", ".join(str(path) for path in missing_paths)
raise RuntimeError(f"KnowledgeFS contract input is missing: {missing}")
def ensure_clean_knowledge_fs_worktree(workspace_root: Path) -> None:
"""Require exported KnowledgeFS files to exactly match the staged index tree.
Staged changes are expected during intentional lock updates. Unstaged tracked changes and untracked files are
rejected because the export process reads the working tree while the tree ID is calculated from the index.
"""
subtree_path = f"{KNOWLEDGE_FS_DIRECTORY}/"
unstaged = subprocess.run(
["git", "diff", "--quiet", "--", subtree_path],
cwd=workspace_root,
check=False,
)
if unstaged.returncode > 1:
raise RuntimeError("git diff failed while validating the staged KnowledgeFS subtree")
untracked = run(
"git",
"ls-files",
"--others",
"--exclude-standard",
"--",
subtree_path,
cwd=workspace_root,
).strip()
if unstaged.returncode != 0 or untracked:
raise RuntimeError(
"knowledge-fs/ contains unstaged or untracked changes; stage or remove them before contract export"
)
def staged_subtree_tree(workspace_root: Path) -> str:
"""Return the Git tree object for the staged ``knowledge-fs/`` subtree."""
return run("git", "write-tree", f"--prefix={KNOWLEDGE_FS_DIRECTORY}/", cwd=workspace_root).strip()
def load_json_object(path: Path) -> dict[str, Any]:
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
return cast(dict[str, Any], value)
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
"""Validate the compact, non-self-referential contract lock schema."""
expected_fields = {
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"schemaVersion",
"subtreeTree",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
for field in (
"capabilityV2AuthManifestSha256",
"capabilityV2AuthTestVectorSha256",
"openapiSha256",
"productOperationGapManifestSha256",
"productOperationManifestSha256",
"subtreeTree",
):
field_value = value.get(field)
expected_length = 40 if field == "subtreeTree" else 64
if (
not isinstance(field_value, str)
or len(field_value) != expected_length
or any(character not in "0123456789abcdef" for character in field_value)
):
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
return cast(ContractLock, value)
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
available_operation_ids = [
operation_id
for path_item in document.get("paths", {}).values()
for method in OPENAPI_METHODS
if isinstance(path_item, dict)
for operation in (path_item.get(method),)
if isinstance(operation, dict)
for operation_id in (operation.get("operationId"),)
if isinstance(operation_id, str) and operation_id
]
for operation_id in required_operation_ids:
count = available_operation_ids.count(operation_id)
if count != 1:
raise ValueError(
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
)
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
expected_fields = {
"active",
"audience",
"callerProfiles",
"claimBindings",
"issuer",
"lifecycle",
"maxTtlSeconds",
"productionReady",
"profileId",
"protectedHeader",
"requiredClaims",
"resourceContract",
"runtimeAssembly",
"schemaVersion",
"signatureAlgorithms",
"tokenKind",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
fixed_values = {
"active": True,
"audience": "knowledge-fs",
"issuer": "dify-control-plane",
"lifecycle": "active",
"maxTtlSeconds": 60,
"productionReady": True,
"profileId": "dify-capability-v2",
"schemaVersion": 3,
"signatureAlgorithms": ["RS256"],
"tokenKind": "jwt",
}
for field, expected in fixed_values.items():
if value.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
if value.get("protectedHeader") != {
"algorithm": "RS256",
"keyIdClaim": "kid",
"keyIdRequired": True,
"type": "JWT",
}:
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
required_claims = [
"action",
"actor",
"aud",
"authz_revision",
"azp",
"caller_kind",
"cap_ver",
"content_policy_revision",
"content_scope_ids",
"control_space_id",
"exp",
"grant_id",
"iat",
"iss",
"jti",
"namespace_id",
"nbf",
"resource",
"sub",
"trace_id",
]
if value.get("requiredClaims") != required_claims:
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
if value.get("claimBindings") != {
"action": "action",
"callerKind": "caller_kind",
"controlSpace": "control_space_id",
"namespace": "namespace_id",
"resource": "resource",
"resourceParent": "resource.parent_id",
"subject": "sub",
}:
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
if value.get("resourceContract") != {
"fields": ["id", "parent_id", "type"],
"parentForbiddenFor": ["namespace", "knowledge_space"],
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
}:
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
if value.get("callerProfiles") != {
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-kfs-credential:"},
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
}:
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
if value.get("runtimeAssembly") != {
"failClosed": True,
"keySelection": "kid",
"maximumPublishedKeys": 3,
"verificationKeySource": "jwks",
}:
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
def validate_capability_v2_auth_test_vector(
value: dict[str, Any],
manifest: dict[str, Any],
) -> None:
"""Verify the deterministic public-key vector and every security-sensitive binding."""
expected_fields = {
"algorithm",
"audience",
"expectedClaims",
"expectedPrincipal",
"issuer",
"operation",
"profileId",
"protectedHeader",
"publicJwk",
"schemaVersion",
"testOnly",
"token",
"ttlSeconds",
}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
if (
value.get("schemaVersion") != 2
or value.get("profileId") != manifest.get("profileId")
or value.get("testOnly") is not True
or value.get("algorithm") != "RS256"
or value.get("issuer") != manifest.get("issuer")
or value.get("audience") != manifest.get("audience")
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
):
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
protected_header = value.get("protectedHeader")
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
kid = protected_header.get("kid")
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
public_jwk = value.get("publicJwk")
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
if (
public_jwk.get("alg") != "RS256"
or public_jwk.get("kid") != kid
or public_jwk.get("kty") != "RSA"
or public_jwk.get("use") != "sig"
or not _is_non_blank(public_jwk.get("e"))
or not _is_non_blank(public_jwk.get("n"))
):
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
claims = value.get("expectedClaims")
required_claims = manifest.get("requiredClaims")
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
operation = value.get("operation")
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
resource = claims.get("resource")
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
expected_operation = {
"action": "documents.read",
"method": "GET",
"operationId": "getDocument",
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
}
if operation != expected_operation:
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
exact_claims = {
"action": operation["action"],
"aud": value["audience"],
"caller_kind": "interactive",
"cap_ver": 2,
"control_space_id": "control-space-contract-vector",
"iss": value["issuer"],
"namespace_id": "workspace-contract-vector",
"resource": {
"id": "document-contract-vector",
"parent_id": "space-contract-vector",
"type": "document",
},
"sub": "dify-account:account-contract-vector",
}
for field, expected in exact_claims.items():
if claims.get(field) != expected:
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
issued_at = claims.get("iat")
not_before = claims.get("nbf")
expires_at = claims.get("exp")
if (
not isinstance(issued_at, int)
or isinstance(issued_at, bool)
or not_before != issued_at
or not isinstance(expires_at, int)
or isinstance(expires_at, bool)
or expires_at - issued_at != value["ttlSeconds"]
):
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
expected_principal = {
"callerKind": claims["caller_kind"],
"subject": {
"scopes": ["knowledge-spaces:read"],
"subjectId": claims["sub"],
"tenantId": claims["namespace_id"],
},
}
if value.get("expectedPrincipal") != expected_principal:
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
token = value.get("token")
if not isinstance(token, str) or not _is_non_blank(token):
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
try:
verification_key = RSAAlgorithm.from_jwk(public_jwk)
if not isinstance(verification_key, RSAPublicKey):
raise ValueError("Capability vector verification key is not RSA public material")
header = jwt.get_unverified_header(token)
decoded_claims = jwt.decode(
token,
verification_key,
algorithms=["RS256"],
audience=cast(str, value["audience"]),
issuer=cast(str, value["issuer"]),
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
)
except (jwt.PyJWTError, TypeError, ValueError) as exc:
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
if header != protected_header or decoded_claims != claims:
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
def _is_non_blank(value: object) -> bool:
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
def validate_upstream_provenance(value: dict[str, Any]) -> None:
"""Validate the imported-source provenance that is itself covered by the subtree tree ID."""
expected_fields = {"commit", "release", "repository", "schemaVersion"}
if set(value) != expected_fields:
raise ValueError(f"KnowledgeFS upstream provenance fields must be exactly {sorted(expected_fields)}")
if value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS upstream provenance must use schemaVersion 1")
repository = value.get("repository")
commit = value.get("commit")
if not isinstance(repository, str) or not repository.startswith("https://"):
raise ValueError("KnowledgeFS upstream provenance repository must be an HTTPS URL")
if (
not isinstance(commit, str)
or len(commit) != 40
or any(character not in "0123456789abcdef" for character in commit)
):
raise ValueError("KnowledgeFS upstream provenance commit must be a lowercase full Git SHA")
if value.get("release") is not None and not isinstance(value["release"], str):
raise ValueError("KnowledgeFS upstream provenance release must be null or a string")
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
@@ -151,7 +637,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
"operation_id": operation_id,
"method": method.upper(),
"path": path[1:],
"required_scope": required_scope(operation),
"required_scope": required_scope(document, operation),
"response_kind": response_kind(operation),
"max_response_bytes": required_max_response_bytes(operation),
"request_headers": request_header_names(path_item, operation),
@@ -169,23 +655,9 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
"""Return transport declarations from the runtime Console operation registry."""
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
return tuple(
{
"operation_id": operation.operation_id,
"method": operation.method,
"path": operation.path,
"required_scope": operation.required_scope,
"response_kind": operation.response_kind,
"max_response_bytes": operation.max_response_bytes,
"request_headers": operation.request_headers,
"response_headers": operation.response_headers,
"response_media_types": operation.response_media_types,
}
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
)
return ()
def response_kind(operation: dict[str, Any]) -> str:
@@ -205,12 +677,17 @@ def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
return tuple(sorted(media_types))
def required_scope(operation: dict[str, Any]) -> str | None:
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
scope = operation.get("x-knowledge-fs-required-scope")
security = operation["security"] if "security" in operation else document.get("security")
if security == []:
if scope is not None:
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
return None
if security != [{"bearerAuth": []}]:
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
return scope
if operation.get("security") == []:
return None
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
+494
View File
@@ -0,0 +1,494 @@
"""Cross-service KnowledgeFS product-operation contract validation.
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
policy remain executable sources of truth. A product operation must appear exactly once as ready
or as an explicit gap; internal KFS-only operations require a named exclusion.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any, Literal, NamedTuple, TypedDict, cast
class ProductOperationLimits(TypedDict):
kfsMaxResponseBytes: int
productMaxRequestBytes: int
productMaxResponseBytes: int
class ProductOperationStream(TypedDict):
kfsResponseKind: str
productKind: str
class ProductOperationManifestEntry(TypedDict):
action: str
kfsOperationId: str
limits: ProductOperationLimits
method: str
path: str
productOperationId: str
resource: str
stream: ProductOperationStream
transport: str
type ProductOperationManifestField = Literal[
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
]
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
)
class ProductOperationManifest(TypedDict):
operations: list[ProductOperationManifestEntry]
schemaVersion: int
class ProductOperationGapEntry(ProductOperationManifestEntry):
reason: str
reasonCode: str
replacementProductOperationIds: list[str]
class InternalKfsOperationExclusion(TypedDict):
kfsOperationId: str
reason: str
reasonCode: str
class ProductOperationGapManifest(TypedDict):
gaps: list[ProductOperationGapEntry]
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
schemaVersion: int
class ProductOperationRuntimeContract(NamedTuple):
action: str
kfs_operation_id: str
max_request_bytes: int
max_response_bytes: int
method: str
path: str
product_operation_id: str
ready: bool
resource: str
stream_kind: str
transport: str
class CapabilityOperationRuntimeContract(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
path: str
resource: str
class CapabilityOperationPolicy(NamedTuple):
action: str
allowed_caller_kinds: tuple[str, ...]
method: str
operation_id: str
parent_resource_binding: dict[str, object] | None
path: str
resource: str
resource_binding: dict[str, object]
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
"""Read the complete typed Dify product registry without normalizing away gaps."""
from services.knowledge_fs.product_operations import (
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
is_product_operation_ready,
)
contracts: list[ProductOperationRuntimeContract] = []
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
contracts.append(
ProductOperationRuntimeContract(
action=operation.action,
kfs_operation_id=operation.capability_operation_id,
max_request_bytes=operation.max_request_bytes,
max_response_bytes=operation.max_response_bytes,
method=operation.method,
path=operation.kfs_path,
product_operation_id=product_operation_id,
ready=is_product_operation_ready(product_operation_id),
resource=operation.resource_resolver,
stream_kind=operation.stream_kind,
transport=operation.transport,
)
)
return tuple(contracts)
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
"""Read the Python issuer policy used to mint each operation Capability."""
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
return tuple(
CapabilityOperationRuntimeContract(
action=operation.action,
allowed_caller_kinds=operation.allowed_caller_kinds,
method=operation.method,
operation_id=operation_id,
path=operation.path,
resource=operation.resource_type,
)
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
)
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
"""Parse the ready product-operation inventory with a closed schema."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
return {
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
"schemaVersion": 2,
}
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
if set(value) != expected_fields or value.get("schemaVersion") != 2:
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
gaps = value.get("gaps")
exclusions = value.get("internalKfsOperationExclusions")
if not isinstance(gaps, list) or not gaps or not isinstance(exclusions, list):
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
parsed_exclusions: list[InternalKfsOperationExclusion] = []
for raw in exclusions:
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
for field in ("kfsOperationId", "reason", "reasonCode"):
_required_string(raw.get(field), f"internal exclusion {field}")
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
return {
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
"internalKfsOperationExclusions": parsed_exclusions,
"schemaVersion": 2,
}
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
"""Parse the runtime-exported TypeScript request-guard registry."""
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
operations = value.get("operations")
if not isinstance(operations, list) or not operations:
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
parsed: list[CapabilityOperationPolicy] = []
for raw in operations:
expected_fields = {
"action",
"allowedCallerKinds",
"method",
"operationId",
"parentResourceBinding",
"path",
"resourceBinding",
"resourceType",
}
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
callers = raw.get("allowedCallerKinds")
resource_binding = raw.get("resourceBinding")
parent_binding = raw.get("parentResourceBinding")
if (
not isinstance(callers, list)
or not callers
or not all(isinstance(caller, str) and caller for caller in callers)
or not isinstance(resource_binding, dict)
or not resource_binding
or (parent_binding is not None and not isinstance(parent_binding, dict))
):
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
for field in ("action", "method", "operationId", "path", "resourceType"):
_required_string(raw.get(field), f"Capability operation {field}")
parsed.append(
CapabilityOperationPolicy(
action=cast(str, raw["action"]),
allowed_caller_kinds=tuple(callers),
method=cast(str, raw["method"]),
operation_id=cast(str, raw["operationId"]),
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
path=cast(str, raw["path"]),
resource=cast(str, raw["resourceType"]),
resource_binding=cast(dict[str, object], resource_binding),
)
)
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
return tuple(parsed)
def validate_product_operation_contracts(
*,
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
capability_policy: tuple[CapabilityOperationPolicy, ...],
document: dict[str, Any],
gap_manifest: ProductOperationGapManifest,
manifest: ProductOperationManifest,
product_operations: tuple[ProductOperationRuntimeContract, ...],
) -> None:
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
if len(ready_entries) != len(manifest["operations"]):
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
if len(gap_entries) != len(gap_manifest["gaps"]):
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
if set(ready_entries) & set(gap_entries):
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
openapi_operations = _openapi_operations(document)
for product_operation_id, runtime in runtime_by_product.items():
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
if runtime.ready != (product_operation_id in ready_entries):
state = "ready" if runtime.ready else "gap"
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
if product_operation_id in gap_entries:
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
if not replacements or any(replacement not in ready_entries for replacement in replacements):
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
exclusions = gap_manifest["internalKfsOperationExclusions"]
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
if len(exclusion_by_id) != len(exclusions):
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
lifecycle_operation_ids = {
"activateDifyWorkspaceIntegration",
"freezeDifyWorkspaceIntegration",
}
if set(exclusion_by_id) != lifecycle_operation_ids:
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
for operation_id in exclusion_by_id:
if operation_id in product_capability_ids:
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
_validate_capability_alignment(
operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
python_operation = python_capabilities[operation_id]
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
base_fields = {
"action",
"kfsOperationId",
"limits",
"method",
"path",
"productOperationId",
"resource",
"stream",
"transport",
}
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
if not isinstance(raw, dict) or set(raw) != expected_fields:
raise ValueError("KnowledgeFS product operation entry fields are invalid")
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
_required_string(raw.get(field), f"product operation {field}")
limits = raw.get("limits")
stream = raw.get("stream")
if not isinstance(limits, dict) or set(limits) != {
"kfsMaxResponseBytes",
"productMaxRequestBytes",
"productMaxResponseBytes",
}:
raise ValueError("KnowledgeFS product operation limits are invalid")
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
raise ValueError("KnowledgeFS product operation stream contract is invalid")
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
_required_string(stream.get("productKind"), "product stream kind")
if gap:
_required_string(raw.get("reason"), "product gap reason")
_required_string(raw.get("reasonCode"), "product gap reasonCode")
replacements = raw.get("replacementProductOperationIds")
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
return cast(ProductOperationManifestEntry, raw)
def _validate_product_entry(
entry: ProductOperationManifestEntry,
runtime: ProductOperationRuntimeContract,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
_validate_capability_alignment(
runtime.kfs_operation_id,
openapi_operations,
python_capabilities,
guard_capabilities,
)
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
expected: ProductOperationManifestEntry = {
"action": runtime.action,
"kfsOperationId": runtime.kfs_operation_id,
"limits": {
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
"productMaxRequestBytes": runtime.max_request_bytes,
"productMaxResponseBytes": runtime.max_response_bytes,
},
"method": runtime.method,
"path": runtime.path,
"productOperationId": runtime.product_operation_id,
"resource": runtime.resource,
"stream": {
"kfsResponseKind": _openapi_response_kind(openapi_operation),
"productKind": runtime.stream_kind,
},
"transport": runtime.transport,
}
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
expected_value = expected[field]
if entry[field] != expected_value:
raise ValueError(
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
f"expected {expected_value!r}, received {entry[field]!r}"
)
def _validate_capability_alignment(
operation_id: str,
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
) -> None:
python_operation = python_capabilities.get(operation_id)
guard_operation = guard_capabilities.get(operation_id)
openapi_operation = openapi_operations.get(operation_id)
if python_operation is None or guard_operation is None or openapi_operation is None:
raise ValueError(
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
)
method, path, _ = openapi_operation
expected = (
python_operation.action,
python_operation.method,
python_operation.path,
python_operation.resource,
)
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
if received_guard != expected:
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
if (method, path) != (python_operation.method, python_operation.path):
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
duplicates: set[str] = set()
for path, path_item in document.get("paths", {}).items():
if not isinstance(path, str) or not isinstance(path_item, dict):
continue
for method in ("delete", "get", "patch", "post", "put"):
operation = path_item.get(method)
if not isinstance(operation, dict):
continue
operation_id = operation.get("operationId")
if not isinstance(operation_id, str) or not operation_id:
continue
if operation_id in operations:
duplicates.add(operation_id)
operations[operation_id] = (method.upper(), path, operation)
if duplicates:
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
return operations
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
media_types = {
media_type
for status, response in operation.get("responses", {}).items()
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
for media_type in response.get("content", {})
}
if "text/event-stream" in media_types:
return "stream"
if "application/octet-stream" in media_types:
return "binary"
return "buffered"
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
value = operation.get("x-knowledge-fs-max-response-bytes")
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
return value
def _required_string(value: object, field: str) -> str:
if not isinstance(value, str) or not value.strip() or value != value.strip():
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
return value
def _unique(values: Iterable[str], label: str) -> None:
sequence = tuple(values)
if len(sequence) != len(set(sequence)):
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
__all__ = [
"CapabilityOperationPolicy",
"CapabilityOperationRuntimeContract",
"ProductOperationGapManifest",
"ProductOperationManifest",
"ProductOperationRuntimeContract",
"capability_operation_runtime_contracts",
"parse_capability_operation_policy",
"parse_product_operation_gap_manifest",
"parse_product_operation_manifest",
"product_operation_runtime_contracts",
"validate_product_operation_contracts",
]
+8
View File
@@ -164,6 +164,14 @@ def init_app(app: DifyApp) -> Celery:
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
if get_configured_knowledge_fs_lifecycle_worker_readiness().ready:
imports.append("tasks.knowledge_fs_lifecycle_tasks")
beat_schedule["knowledge_fs_lifecycle_worker"] = {
"task": "tasks.knowledge_fs_lifecycle_tasks.run_knowledge_fs_lifecycle_worker",
"schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
}
if dify_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK:
imports.append("schedule.clean_embedding_cache_task")
beat_schedule["clean_embedding_cache_task"] = {
+2
View File
@@ -27,6 +27,7 @@ def init_app(app: DifyApp):
import_migration_data,
install_plugins,
install_rag_pipeline_plugins,
knowledge_fs_control_space,
migrate_data_for_plugin,
migrate_dataset_permissions_to_rbac,
migrate_member_roles_to_rbac,
@@ -88,6 +89,7 @@ def init_app(app: DifyApp):
export_migration_data_template,
import_migration_data,
migration_data_wizard,
knowledge_fs_control_space,
]
for cmd in cmds_to_register:
app.cli.add_command(cmd)
@@ -0,0 +1,24 @@
"""Production assembly for KnowledgeFS database-backed metrics."""
from __future__ import annotations
import logging
from flask import Flask
from core.db.session_factory import session_factory
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
)
from services.knowledge_fs.observability import get_knowledge_fs_operational_metrics
logger = logging.getLogger(__name__)
def init_app(app: Flask) -> None:
del app
try:
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_factory.get_session_maker())
get_knowledge_fs_operational_metrics().register_control_space_state_gauge(reader)
except Exception:
logger.exception("KnowledgeFS operational metric registration failed; continuing application startup")
+7 -3
View File
@@ -1,5 +1,9 @@
{
"commit": "4310e2d582d25e7de58183f27720afab01e123cf",
"openapiSha256": "5827ca930ce38462bfd1b2bef387efbf37eb7ffcaedde4558af2fbaeccbfbc4b",
"repository": "https://github.com/langgenius/knowledge-fs"
"schemaVersion": 5,
"subtreeTree": "04f9a6189c215f4fcccdccbb3ae7fc810b927463",
"openapiSha256": "df702b15769f79c561e34d72472a2ec8dddcdefe4bd4c82858f06a98cd6eae8c",
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
"productOperationManifestSha256": "5ceef85419dfba9c803732f78f31c803a0836e883972bb67f26ae43b0c78d341",
"productOperationGapManifestSha256": "ccbae37fe658177a77529822211a2cf02e72b5815cbc9b7e7e65e6817ba5e0a9"
}
@@ -0,0 +1,44 @@
{
"schemaVersion": 2,
"gaps": [
{
"productOperationId": "createDocument",
"kfsOperationId": "uploadDocument",
"method": "POST",
"path": "/knowledge-spaces/{id}/documents",
"action": "documents.create",
"resource": "knowledge_space",
"transport": "multipart",
"stream": {
"productKind": "buffered-multipart",
"kfsResponseKind": "buffered"
},
"limits": {
"productMaxRequestBytes": 0,
"productMaxResponseBytes": 0,
"kfsMaxResponseBytes": 1048576
},
"reasonCode": "NON_HOMOMORPHIC_MULTIPART_TRANSPORT",
"reason": "Legacy buffered multipart document creation has no homomorphic Dify product transport; use Capability v2 direct-upload operations.",
"replacementProductOperationIds": [
"createUploadSession",
"presignUploadSessionPart",
"uploadSmallFile",
"completeUploadSession",
"abortUploadSession"
]
}
],
"internalKfsOperationExclusions": [
{
"kfsOperationId": "activateDifyWorkspaceIntegration",
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
"reason": "Workspace activation is an internal cutover control-plane operation and is never exposed as a product operation."
},
{
"kfsOperationId": "freezeDifyWorkspaceIntegration",
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
"reason": "Workspace maintenance freeze is an internal cutover control-plane operation and is never exposed as a product operation."
}
]
}
+53
View File
@@ -0,0 +1,53 @@
{
"schemaVersion": 2,
"operations": [
{"productOperationId":"batchSpaceSummaries","kfsOperationId":"batchKnowledgeSpaceProductSummaries","method":"POST","path":"/internal/knowledge-spaces/product-summaries/batch","action":"knowledge_spaces.status.batch","resource":"namespace","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSpace","kfsOperationId":"getKnowledgeSpace","method":"GET","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSpace","kfsOperationId":"updateKnowledgeSpace","method":"PATCH","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSettings","kfsOperationId":"getKnowledgeSpaceProductSettings","method":"GET","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocument","kfsOperationId":"getDocument","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocumentOutline","kfsOperationId":"getDocumentOutline","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/outline","action":"documents.outline.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocumentRevisions","kfsOperationId":"listDocumentRevisions","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions","action":"documents.revisions.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateDocumentMetadata","kfsOperationId":"patchDocumentMetadata","method":"PATCH","path":"/knowledge-spaces/{id}/documents/{documentId}/metadata","action":"documents.metadata.update","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listDocumentChunks","kfsOperationId":"listDocumentChunks","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks","action":"documents.chunks.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getDocumentChunk","kfsOperationId":"getDocumentChunk","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}","action":"documents.chunks.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"deleteDocument","kfsOperationId":"requestDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.delete","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"bulkDeleteDocuments","kfsOperationId":"requestBulkDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/bulk","action":"documents.bulk.delete","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"reindexDocuments","kfsOperationId":"bulkReindexDocuments","method":"POST","path":"/knowledge-spaces/{id}/documents/bulk/reindex","action":"documents.bulk.reindex","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getCompilationJob","kfsOperationId":"getDocumentCompilationJob","method":"GET","path":"/jobs/{id}","action":"document_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"cancelCompilationJob","kfsOperationId":"cancelDocumentCompilationJob","method":"DELETE","path":"/jobs/{id}","action":"document_jobs.cancel","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"retryCompilationJob","kfsOperationId":"retryDocumentCompilationJob","method":"POST","path":"/jobs/{id}/retry","action":"document_jobs.retry","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getBulkJob","kfsOperationId":"getBulkOperation","method":"GET","path":"/bulk-jobs/{id}","action":"bulk_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSources","kfsOperationId":"listKnowledgeSpaceSources","method":"GET","path":"/knowledge-spaces/{id}/sources","action":"sources.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createSource","kfsOperationId":"createKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources","action":"sources.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getSource","kfsOperationId":"getKnowledgeSpaceSource","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.read","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"updateSource","kfsOperationId":"updateKnowledgeSpaceSource","method":"PATCH","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"deleteSource","kfsOperationId":"requestSourceDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.delete","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"testSource","kfsOperationId":"testKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/test","action":"sources.test","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"crawlSource","kfsOperationId":"crawlKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl","action":"sources.crawl","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourcePages","kfsOperationId":"listKnowledgeSpaceSourcePages","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/pages","action":"sources.pages.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"importSourcePages","kfsOperationId":"importKnowledgeSpaceSourcePages","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import","action":"sources.pages.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listSourceFiles","kfsOperationId":"listKnowledgeSpaceSourceFiles","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/files","action":"sources.files.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"importSourceFiles","kfsOperationId":"importKnowledgeSpaceSourceFiles","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import-files","action":"sources.files.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createQuery","kfsOperationId":"createQuery","method":"POST","path":"/queries","action":"queries.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}},
{"productOperationId":"listResearchTasks","kfsOperationId":"listKnowledgeSpaceResearchTasks","method":"GET","path":"/knowledge-spaces/{id}/research-tasks","action":"research_tasks.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createResearchTask","kfsOperationId":"createResearchTask","method":"POST","path":"/research-tasks","action":"research_tasks.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"planResearchTask","kfsOperationId":"planResearchTask","method":"POST","path":"/research-tasks/plan","action":"research_tasks.plan","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getResearchTask","kfsOperationId":"getResearchTask","method":"GET","path":"/research-tasks/{id}","action":"research_tasks.read","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listResearchTaskPartials","kfsOperationId":"listResearchTaskPartials","method":"GET","path":"/research-tasks/{id}/partials","action":"research_tasks.partials.list","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"cancelResearchTask","kfsOperationId":"cancelResearchTask","method":"DELETE","path":"/research-tasks/{id}","action":"research_tasks.cancel","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraces","kfsOperationId":"listKnowledgeSpaceQualityTraces","method":"GET","path":"/knowledge-spaces/{id}/quality/traces","action":"quality.traces.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"getTrace","kfsOperationId":"getAnswerTrace","method":"GET","path":"/queries/{traceId}","action":"queries.read","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceEvidence","kfsOperationId":"listQueryEvidence","method":"GET","path":"/queries/{traceId}/evidence","action":"queries.evidence.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceConflicts","kfsOperationId":"listQueryConflicts","method":"GET","path":"/queries/{traceId}/conflicts","action":"queries.conflicts.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"listTraceMissing","kfsOperationId":"listQueryMissing","method":"GET","path":"/queries/{traceId}/missing","action":"queries.missing.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"createUploadSession","kfsOperationId":"createUploadSession","method":"POST","path":"/knowledge-spaces/{id}/upload-sessions","action":"upload_sessions.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"presignUploadSessionPart","kfsOperationId":"presignUploadSessionPart","method":"POST","path":"/upload-sessions/{id}/parts/{partNumber}/presign","action":"upload_sessions.write","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"uploadSmallFile","kfsOperationId":"uploadSmallFile","method":"POST","path":"/upload-sessions/{id}/small-file","action":"upload_sessions.write","resource":"upload_session","transport":"binary","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":8388608,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"completeUploadSession","kfsOperationId":"completeUploadSession","method":"POST","path":"/upload-sessions/{id}/complete","action":"upload_sessions.complete","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"abortUploadSession","kfsOperationId":"abortUploadSession","method":"POST","path":"/upload-sessions/{id}/abort","action":"upload_sessions.abort","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
{"productOperationId":"streamResearchTask","kfsOperationId":"streamResearchTaskProgress","method":"GET","path":"/research-tasks/{id}/events","action":"research_tasks.stream","resource":"research_task","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}}
]
}
@@ -0,0 +1,387 @@
"""add independent KnowledgeFS control-plane tables
Revision ID: a4e7c2f91b30
Revises: 3c9f8e2a1d7b
Create Date: 2026-07-21 12:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "a4e7c2f91b30"
down_revision = "3c9f8e2a1d7b"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def _timestamps() -> tuple[sa.Column, sa.Column]:
return (
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
)
def upgrade() -> None:
op.create_table(
"knowledge_fs_control_spaces",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("knowledge_space_id", models.types.StringUUID(), nullable=True),
sa.Column("knowledge_space_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("owner_account_id", models.types.StringUUID(), nullable=False),
sa.Column("visibility", sa.String(length=32), server_default=sa.text("'only_me'"), nullable=False),
sa.Column("provisioning_key", sa.String(length=255), nullable=False),
sa.Column("lifecycle_operation_id", models.types.StringUUID(), nullable=True),
sa.Column("state", sa.String(length=32), server_default=sa.text("'provisioning'"), nullable=False),
sa.Column("resource_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
sa.Column("last_error_code", sa.String(length=255), nullable=True),
sa.Column("last_error_message", models.types.LongText(), nullable=True),
sa.Column("last_synced_at", sa.DateTime(), nullable=True),
sa.Column("deletion_irreversible_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
sa.UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
sa.UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id"],
["tenants.id"],
name="kfs_control_space_workspace_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("resource_version >= 0", name="kfs_control_space_resource_version_ck"),
sa.CheckConstraint("attempt_count >= 0", name="kfs_control_space_attempt_count_ck"),
sa.CheckConstraint(
"state != 'active' OR knowledge_space_id IS NOT NULL",
name="kfs_control_space_active_registration_ck",
),
sa.CheckConstraint(
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
name="kfs_control_space_irreversible_state_ck",
),
sa.CheckConstraint(
"knowledge_space_revision >= 0",
name="kfs_control_space_remote_revision_ck",
),
)
if op.get_bind().dialect.name == "postgresql":
op.create_index(
"kfs_control_space_tenant_space_uq",
"knowledge_fs_control_spaces",
["tenant_id", "knowledge_space_id"],
unique=True,
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
)
else:
op.create_index(
"kfs_control_space_tenant_space_uq",
"knowledge_fs_control_spaces",
["tenant_id", "knowledge_space_id"],
unique=True,
)
op.create_index(
"kfs_control_space_state_updated_idx",
"knowledge_fs_control_spaces",
["state", "updated_at"],
)
op.create_index(
"kfs_control_space_tenant_state_updated_idx",
"knowledge_fs_control_spaces",
["tenant_id", "state", "updated_at"],
)
op.create_index(
"kfs_control_space_tenant_owner_state_idx",
"knowledge_fs_control_spaces",
["tenant_id", "owner_account_id", "state"],
)
op.create_table(
"knowledge_fs_control_space_permissions",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("account_id", models.types.StringUUID(), nullable=False),
sa.Column("role", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("granted_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
sa.UniqueConstraint(
"tenant_id",
"control_space_id",
"account_id",
name="kfs_control_space_permission_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_control_space_permission_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_control_space_permission_revision_ck"),
)
op.create_index(
"kfs_control_space_permission_account_idx",
"knowledge_fs_control_space_permissions",
["tenant_id", "account_id", "status"],
)
op.create_table(
"knowledge_fs_external_access_policies",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("service_api_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("agent_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("workflow_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("mcp_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("updated_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_external_access_policy_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_external_access_policy_revision_ck"),
)
op.create_table(
"knowledge_fs_api_credentials",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("credential_hash", sa.String(length=255), nullable=False),
sa.Column("credential_prefix", sa.String(length=32), nullable=False),
sa.Column("credential_last4", sa.String(length=4), nullable=False),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("allowed_actions", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("expires_at", sa.DateTime(), nullable=True),
sa.Column("last_used_at", sa.DateTime(), nullable=True),
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoke_reason", sa.String(length=255), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
sa.UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_api_credential_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="kfs_api_credential_revision_ck"),
)
op.create_index(
"kfs_api_credential_tenant_space_status_idx",
"knowledge_fs_api_credentials",
["tenant_id", "control_space_id", "status"],
)
op.create_index(
"kfs_api_credential_tenant_prefix_idx",
"knowledge_fs_api_credentials",
["tenant_id", "credential_prefix"],
)
op.create_table(
"app_knowledge_fs_space_joins",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("app_id", models.types.StringUUID(), nullable=False),
sa.Column("join_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("revoked_at", sa.DateTime(), nullable=True),
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
sa.UniqueConstraint(
"tenant_id",
"app_id",
"control_space_id",
"join_type",
name="app_kfs_space_join_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="app_kfs_space_join_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name="app_kfs_space_join_revision_ck"),
)
op.create_index(
"app_kfs_space_join_app_status_idx",
"app_knowledge_fs_space_joins",
["tenant_id", "app_id", "status"],
)
op.create_table(
"knowledge_fs_authorization_revisions",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("membership_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("space_acl_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("external_access_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("content_policy_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("revoke_sequence", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_authorization_revision_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("membership_epoch >= 0", name="kfs_authorization_membership_epoch_ck"),
sa.CheckConstraint("space_acl_epoch >= 0", name="kfs_authorization_space_acl_epoch_ck"),
sa.CheckConstraint("external_access_epoch >= 0", name="kfs_authorization_external_access_epoch_ck"),
sa.CheckConstraint("content_policy_revision >= 0", name="kfs_authorization_content_policy_revision_ck"),
sa.CheckConstraint("revoke_sequence >= 0", name="kfs_authorization_revoke_sequence_ck"),
)
op.create_table(
"knowledge_fs_capability_issuance_audits",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("trace_id", sa.String(length=128), nullable=False),
sa.Column("jti_hash", sa.String(length=80), nullable=False),
sa.Column("claims_summary", sa.JSON(), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
sa.UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_audit_space_fk",
ondelete="RESTRICT",
),
)
op.create_index(
"kfs_capability_issuance_audit_space_created_idx",
"knowledge_fs_capability_issuance_audits",
["tenant_id", "control_space_id", "created_at"],
)
op.create_index(
"kfs_capability_issuance_audit_trace_idx",
"knowledge_fs_capability_issuance_audits",
["tenant_id", "trace_id"],
)
op.create_table(
"knowledge_fs_lifecycle_outbox",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("operation_id", models.types.StringUUID(), nullable=False),
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
sa.Column("operation", sa.String(length=32), nullable=False),
sa.Column("command_payload", sa.JSON(), nullable=False),
sa.Column("expected_control_space_version", sa.BigInteger(), nullable=False),
sa.Column("expected_knowledge_space_revision", sa.BigInteger(), nullable=True),
sa.Column("command_schema_version", sa.Integer(), server_default=sa.text("1"), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'pending'"), nullable=False),
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
sa.Column("next_attempt_at", sa.DateTime(), nullable=True),
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
sa.Column("lease_owner", sa.String(length=255), nullable=True),
sa.Column("lease_expires_at", sa.DateTime(), nullable=True),
sa.Column("last_error_code", sa.String(length=255), nullable=True),
sa.Column("last_error_message", models.types.LongText(), nullable=True),
sa.Column("completed_at", sa.DateTime(), nullable=True),
sa.Column("retain_until", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
sa.UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
sa.UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_lifecycle_outbox_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("command_schema_version >= 1", name="kfs_lifecycle_outbox_schema_version_ck"),
sa.CheckConstraint("expected_control_space_version >= 0", name="kfs_lifecycle_outbox_expected_version_ck"),
sa.CheckConstraint("attempt_count >= 0", name="kfs_lifecycle_outbox_attempt_count_ck"),
sa.CheckConstraint(
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
name="kfs_lifecycle_outbox_status_ck",
),
sa.CheckConstraint(
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
name="kfs_lifecycle_outbox_lease_state_ck",
),
sa.CheckConstraint(
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
name="kfs_lifecycle_outbox_terminal_state_ck",
),
)
op.create_index(
"kfs_lifecycle_outbox_dispatch_idx",
"knowledge_fs_lifecycle_outbox",
["status", "next_attempt_at", "id"],
)
op.create_index(
"kfs_lifecycle_outbox_space_created_idx",
"knowledge_fs_lifecycle_outbox",
["tenant_id", "control_space_id", "created_at"],
)
def downgrade() -> None:
op.drop_index("kfs_lifecycle_outbox_space_created_idx", table_name="knowledge_fs_lifecycle_outbox")
op.drop_index("kfs_lifecycle_outbox_dispatch_idx", table_name="knowledge_fs_lifecycle_outbox")
op.drop_table("knowledge_fs_lifecycle_outbox")
op.drop_index(
"kfs_capability_issuance_audit_trace_idx",
table_name="knowledge_fs_capability_issuance_audits",
)
op.drop_index(
"kfs_capability_issuance_audit_space_created_idx",
table_name="knowledge_fs_capability_issuance_audits",
)
op.drop_table("knowledge_fs_capability_issuance_audits")
op.drop_table("knowledge_fs_authorization_revisions")
op.drop_index("app_kfs_space_join_app_status_idx", table_name="app_knowledge_fs_space_joins")
op.drop_table("app_knowledge_fs_space_joins")
op.drop_index("kfs_api_credential_tenant_prefix_idx", table_name="knowledge_fs_api_credentials")
op.drop_index("kfs_api_credential_tenant_space_status_idx", table_name="knowledge_fs_api_credentials")
op.drop_table("knowledge_fs_api_credentials")
op.drop_table("knowledge_fs_external_access_policies")
op.drop_index(
"kfs_control_space_permission_account_idx",
table_name="knowledge_fs_control_space_permissions",
)
op.drop_table("knowledge_fs_control_space_permissions")
op.drop_index("kfs_control_space_tenant_owner_state_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_tenant_state_updated_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_state_updated_idx", table_name="knowledge_fs_control_spaces")
op.drop_index("kfs_control_space_tenant_space_uq", table_name="knowledge_fs_control_spaces")
op.drop_table("knowledge_fs_control_spaces")
@@ -0,0 +1,205 @@
"""add KnowledgeFS workspace cutover ledger and migration evidence
Revision ID: b7f2a9d41c60
Revises: a4e7c2f91b30
Create Date: 2026-07-21 13:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "b7f2a9d41c60"
down_revision = "a4e7c2f91b30"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def _timestamps() -> tuple[sa.Column, sa.Column]:
return (
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
)
def upgrade() -> None:
op.create_table(
"knowledge_fs_workspace_cutover_ledgers",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("phase", sa.String(length=32), server_default=sa.text("'inventory'"), nullable=False),
sa.Column("source_revision_watermark", sa.JSON(), nullable=False),
sa.Column("final_revision_watermark", sa.JSON(), nullable=True),
sa.Column("applied_revision_watermark", sa.JSON(), nullable=False),
sa.Column("source_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("final_task_watermark", sa.BigInteger(), nullable=True),
sa.Column("applied_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("freeze_at", sa.DateTime(), nullable=True),
sa.Column("cutover_at", sa.DateTime(), nullable=True),
sa.Column("rolled_back_at", sa.DateTime(), nullable=True),
sa.Column("rollback_cutoff_at", sa.DateTime(), nullable=True),
sa.Column("observation_started_at", sa.DateTime(), nullable=True),
sa.Column("observation_window_ends_at", sa.DateTime(), nullable=True),
sa.Column("observation_completed_at", sa.DateTime(), nullable=True),
sa.Column("maximum_task_expires_at", sa.DateTime(), nullable=True),
sa.Column("irreversible_cleanup_at", sa.DateTime(), nullable=True),
sa.Column("product_routes_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("capability_v2_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("integrated_mode_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("legacy_acl_read_only", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("smoke_results", sa.JSON(), nullable=True),
sa.Column("legacy_dependency_report", sa.JSON(), nullable=True),
sa.Column("legacy_dependency_checked_at", sa.DateTime(), nullable=True),
sa.Column("legacy_dependency_ready", sa.Boolean(), server_default=sa.text("false"), nullable=False),
sa.Column("cas_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
sa.UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
sa.UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
sa.CheckConstraint("cas_version >= 0", name="kfs_workspace_cutover_cas_version_ck"),
sa.CheckConstraint(
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
name="kfs_workspace_cutover_task_watermark_ck",
),
sa.CheckConstraint(
"cutover_at IS NULL OR freeze_at IS NOT NULL",
name="kfs_workspace_cutover_freeze_before_cutover_ck",
),
sa.CheckConstraint(
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
name="kfs_workspace_cutover_rollback_cutoff_ck",
),
)
op.create_index(
"kfs_workspace_cutover_ledger_phase_updated_idx",
"knowledge_fs_workspace_cutover_ledgers",
["phase", "updated_at"],
)
op.create_table(
"knowledge_fs_migration_issues",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("issue_key", sa.String(length=255), nullable=False),
sa.Column("kind", sa.String(length=48), nullable=False),
sa.Column("status", sa.String(length=32), server_default=sa.text("'open'"), nullable=False),
sa.Column("resource_type", sa.String(length=64), nullable=True),
sa.Column("resource_id", sa.String(length=255), nullable=True),
sa.Column("details", sa.JSON(), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
sa.UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_issue_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_migration_issue_gate_idx",
"knowledge_fs_migration_issues",
["tenant_id", "ledger_id", "status", "kind"],
)
op.create_table(
"knowledge_fs_migration_quarantine",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("source_kind", sa.String(length=32), nullable=False),
sa.Column("source_id", sa.String(length=255), nullable=False),
sa.Column("reason_code", sa.String(length=64), nullable=False),
sa.Column("disposition", sa.String(length=32), nullable=False),
sa.Column("details", sa.JSON(), nullable=False),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"source_kind",
"source_id",
name="kfs_migration_quarantine_source_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_quarantine_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_migration_quarantine_disposition_idx",
"knowledge_fs_migration_quarantine",
["tenant_id", "ledger_id", "disposition"],
)
op.create_table(
"knowledge_fs_shadow_authorization_diffs",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("diff_key", sa.String(length=255), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
sa.Column("decision", sa.String(length=32), nullable=False),
sa.Column("reason", models.types.LongText(), nullable=False),
sa.Column("observed_revision", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
*_timestamps(),
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
sa.UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_diff_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_shadow_authorization_diff_gate_idx",
"knowledge_fs_shadow_authorization_diffs",
["tenant_id", "ledger_id", "status", "decision"],
)
def downgrade() -> None:
op.drop_index(
"kfs_shadow_authorization_diff_gate_idx",
table_name="knowledge_fs_shadow_authorization_diffs",
)
op.drop_table("knowledge_fs_shadow_authorization_diffs")
op.drop_index(
"kfs_migration_quarantine_disposition_idx",
table_name="knowledge_fs_migration_quarantine",
)
op.drop_table("knowledge_fs_migration_quarantine")
op.drop_index("kfs_migration_issue_gate_idx", table_name="knowledge_fs_migration_issues")
op.drop_table("knowledge_fs_migration_issues")
op.drop_index(
"kfs_workspace_cutover_ledger_phase_updated_idx",
table_name="knowledge_fs_workspace_cutover_ledgers",
)
op.drop_table("knowledge_fs_workspace_cutover_ledgers")
@@ -0,0 +1,365 @@
"""add guarded cleanup, cutover evidence, and capability issuance reservations
Revision ID: c8e31b7d52a4
Revises: b7f2a9d41c60
Create Date: 2026-07-21 14:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "c8e31b7d52a4"
down_revision = "b7f2a9d41c60"
branch_labels = None
depends_on = None
def _uuid_column(name: str, **kwargs):
if op.get_bind().dialect.name == "postgresql":
kwargs.setdefault("server_default", sa.text("uuidv7()"))
return sa.Column(name, models.types.StringUUID(), **kwargs)
def upgrade() -> None:
op.create_table(
"knowledge_fs_cleanup_authorizations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("request_id", models.types.StringUUID(), nullable=False),
sa.Column("plan_schema_version", sa.String(length=32), nullable=False),
sa.Column("plan_digest", sa.String(length=71), nullable=False),
sa.Column("targets", sa.JSON(), nullable=False),
sa.Column("readiness_evidence", sa.JSON(), nullable=False),
sa.Column("requested_by_account_id", models.types.StringUUID(), nullable=False),
sa.Column("requested_at", sa.DateTime(), nullable=False),
sa.Column("readiness_ledger_cas_version", sa.BigInteger(), nullable=False),
sa.Column("status", sa.String(length=16), server_default=sa.text("'requested'"), nullable=False),
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("approved_at", sa.DateTime(), nullable=True),
sa.Column("approval_expires_at", sa.DateTime(), nullable=True),
sa.Column("approved_ledger_cas_version", sa.BigInteger(), nullable=True),
sa.Column("started_by_account_id", models.types.StringUUID(), nullable=True),
sa.Column("started_at", sa.DateTime(), nullable=True),
sa.Column("started_ledger_cas_version", sa.BigInteger(), nullable=True),
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"request_id",
name="kfs_cleanup_authorization_request_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_cleanup_authorization_ledger_fk",
),
sa.CheckConstraint("row_version >= 0", name="kfs_cleanup_authorization_version_ck"),
sa.CheckConstraint(
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
name="kfs_cleanup_authorization_ledger_versions_ck",
),
sa.CheckConstraint(
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL)",
name="kfs_cleanup_authorization_status_fields_ck",
),
sa.CheckConstraint(
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
name="kfs_cleanup_authorization_approval_window_ck",
),
)
op.create_index(
"kfs_cleanup_authorization_status_idx",
"knowledge_fs_cleanup_authorizations",
["tenant_id", "status", "updated_at"],
)
op.create_table(
"knowledge_fs_capability_issuance_reservations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
sa.Column("grant_id", models.types.StringUUID(), nullable=False),
sa.Column("trace_id", sa.String(length=128), nullable=False),
sa.Column("subject", sa.String(length=255), nullable=False),
sa.Column("caller_kind", sa.String(length=32), nullable=False),
sa.Column("request_summary", sa.JSON(), nullable=False),
sa.Column("status", sa.String(length=16), server_default=sa.text("'reserved'"), nullable=False),
sa.Column("issued_at", sa.DateTime(), nullable=True),
sa.Column("token_expires_at", sa.DateTime(), nullable=True),
sa.Column("failed_at", sa.DateTime(), nullable=True),
sa.Column("failure_code", sa.String(length=128), nullable=True),
sa.Column("cleanup_after", sa.DateTime(), nullable=True),
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
sa.UniqueConstraint(
"tenant_id",
"grant_id",
name="kfs_capability_issuance_reservation_grant_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_reservation_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint(
"row_version >= 0",
name="kfs_capability_issuance_reservation_version_ck",
),
sa.CheckConstraint(
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
name="kfs_capability_issuance_reservation_status_fields_ck",
),
sa.CheckConstraint(
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
name="kfs_capability_issuance_reservation_cleanup_window_ck",
),
)
op.create_index(
"kfs_capability_issuance_reservation_subject_idx",
"knowledge_fs_capability_issuance_reservations",
["tenant_id", "control_space_id", "subject", "caller_kind"],
)
op.create_index(
"kfs_capability_issuance_reservation_trace_idx",
"knowledge_fs_capability_issuance_reservations",
["tenant_id", "trace_id"],
)
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
batch_op.add_column(sa.Column("resolved_by_operator", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
batch_op.create_check_constraint(
"kfs_migration_quarantine_version_ck",
"row_version >= 0",
)
batch_op.create_check_constraint(
"kfs_migration_quarantine_resolution_fields_ck",
"(disposition = 'resolved' "
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
"(disposition <> 'resolved' "
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
"AND evidence IS NULL AND resolved_at IS NULL)",
)
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.add_column(sa.Column("shadow_started_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_evidence_digest", sa.String(length=71), nullable=True))
batch_op.add_column(
sa.Column("shadow_observation_count", sa.BigInteger(), server_default=sa.text("0"), nullable=False)
)
batch_op.add_column(sa.Column("shadow_window_started_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("shadow_window_ended_at", sa.DateTime(), nullable=True))
batch_op.add_column(
sa.Column("shadow_traffic_zero", sa.Boolean(), server_default=sa.text("false"), nullable=False)
)
batch_op.add_column(sa.Column("shadow_traffic_zero_evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("shadow_latest_observed_revision", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("shadow_producer", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_by_operator", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("shadow_completed_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_id", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("remote_activation_revision", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("remote_activation_control_space_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_activated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_updated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_acknowledged_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_applied", sa.Boolean(), nullable=True))
batch_op.add_column(sa.Column("remote_activation_replayed", sa.Boolean(), nullable=True))
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_count_ck",
"shadow_observation_count >= 0",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_window_ck",
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_shadow_completion_fields_ck",
"(shadow_completed_at IS NULL "
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
"AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_activation_fields_ck",
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
"AND remote_activation_replayed IS NULL) OR "
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
"AND remote_activation_acknowledged_at IS NOT NULL "
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_activation_time_ck",
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
)
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.add_column(sa.Column("current_evidence_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("last_observed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
batch_op.create_check_constraint(
"kfs_shadow_authorization_diff_version_ck",
"row_version >= 0",
)
op.execute(
sa.text(
"UPDATE knowledge_fs_shadow_authorization_diffs "
"SET current_evidence_digest = 'sha256:legacy-unversioned', last_observed_at = created_at "
"WHERE current_evidence_digest IS NULL OR last_observed_at IS NULL"
)
)
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.alter_column("current_evidence_digest", existing_type=sa.String(length=71), nullable=False)
batch_op.alter_column("last_observed_at", existing_type=sa.DateTime(), nullable=False)
op.create_table(
"knowledge_fs_shadow_authorization_observations",
_uuid_column("id", nullable=False),
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
sa.Column("diff_key", sa.String(length=255), nullable=False),
sa.Column("producer", sa.String(length=255), nullable=False),
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
sa.Column("principal", sa.String(length=255), nullable=False),
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
sa.Column("decision", sa.String(length=32), nullable=False),
sa.Column("reason", models.types.LongText(), nullable=False),
sa.Column("observed_revision", sa.JSON(), nullable=False),
sa.Column("observed_at", sa.DateTime(), nullable=False),
sa.Column("evidence_digest", sa.String(length=71), nullable=False),
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
sa.UniqueConstraint(
"tenant_id",
"ledger_id",
"diff_key",
"evidence_digest",
name="kfs_shadow_authorization_observation_evidence_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_observation_ledger_fk",
ondelete="CASCADE",
),
)
op.create_index(
"kfs_shadow_authorization_observation_window_idx",
"knowledge_fs_shadow_authorization_observations",
["tenant_id", "ledger_id", "observed_at"],
)
def downgrade() -> None:
op.drop_index(
"kfs_shadow_authorization_observation_window_idx",
table_name="knowledge_fs_shadow_authorization_observations",
)
op.drop_table("knowledge_fs_shadow_authorization_observations")
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
batch_op.drop_constraint("kfs_shadow_authorization_diff_version_ck", type_="check")
batch_op.drop_column("row_version")
batch_op.drop_column("last_observed_at")
batch_op.drop_column("current_evidence_digest")
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_time_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_fields_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_completion_fields_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_window_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_shadow_count_ck", type_="check")
batch_op.drop_column("remote_activation_replayed")
batch_op.drop_column("remote_activation_applied")
batch_op.drop_column("remote_activation_acknowledged_at")
batch_op.drop_column("remote_activation_updated_at")
batch_op.drop_column("remote_activation_activated_at")
batch_op.drop_column("remote_activation_control_space_id")
batch_op.drop_column("remote_activation_digest")
batch_op.drop_column("remote_activation_revision")
batch_op.drop_column("remote_activation_id")
batch_op.drop_column("shadow_completed_by_account_id")
batch_op.drop_column("shadow_completed_by_operator")
batch_op.drop_column("shadow_producer")
batch_op.drop_column("shadow_latest_observed_revision")
batch_op.drop_column("shadow_traffic_zero_evidence")
batch_op.drop_column("shadow_traffic_zero")
batch_op.drop_column("shadow_window_ended_at")
batch_op.drop_column("shadow_window_started_at")
batch_op.drop_column("shadow_observation_count")
batch_op.drop_column("shadow_evidence_digest")
batch_op.drop_column("shadow_completed_at")
batch_op.drop_column("shadow_started_at")
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
batch_op.drop_constraint(
"kfs_migration_quarantine_resolution_fields_ck",
type_="check",
)
batch_op.drop_constraint("kfs_migration_quarantine_version_ck", type_="check")
batch_op.drop_column("row_version")
batch_op.drop_column("evidence")
batch_op.drop_column("resolved_by_account_id")
batch_op.drop_column("resolved_by_operator")
op.drop_index(
"kfs_capability_issuance_reservation_trace_idx",
table_name="knowledge_fs_capability_issuance_reservations",
)
op.drop_index(
"kfs_capability_issuance_reservation_subject_idx",
table_name="knowledge_fs_capability_issuance_reservations",
)
op.drop_table("knowledge_fs_capability_issuance_reservations")
op.drop_index(
"kfs_cleanup_authorization_status_idx",
table_name="knowledge_fs_cleanup_authorizations",
)
op.drop_table("knowledge_fs_cleanup_authorizations")
@@ -0,0 +1,66 @@
"""add durable KnowledgeFS remote freeze evidence
Revision ID: d4f6e8a1c305
Revises: c8e31b7d52a4
Create Date: 2026-07-21 15:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "d4f6e8a1c305"
down_revision = "c8e31b7d52a4"
branch_labels = None
depends_on = None
def upgrade() -> None:
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.add_column(sa.Column("remote_freeze_id", sa.String(length=255), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_revision", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_digest", sa.String(length=71), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_task_watermark", sa.BigInteger(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_control_space_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_frozen_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_updated_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_acknowledged_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_applied", sa.Boolean(), nullable=True))
batch_op.add_column(sa.Column("remote_freeze_replayed", sa.Boolean(), nullable=True))
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_freeze_fields_ck",
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
)
batch_op.create_check_constraint(
"kfs_workspace_cutover_remote_freeze_time_ck",
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
)
def downgrade() -> None:
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_time_ck", type_="check")
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_fields_ck", type_="check")
batch_op.drop_column("remote_freeze_replayed")
batch_op.drop_column("remote_freeze_applied")
batch_op.drop_column("remote_freeze_acknowledged_at")
batch_op.drop_column("remote_freeze_updated_at")
batch_op.drop_column("remote_freeze_frozen_at")
batch_op.drop_column("remote_freeze_control_space_id")
batch_op.drop_column("remote_freeze_task_watermark")
batch_op.drop_column("remote_freeze_digest")
batch_op.drop_column("remote_freeze_revision")
batch_op.drop_column("remote_freeze_id")
@@ -0,0 +1,113 @@
"""add global KnowledgeFS cleanup completion evidence
Revision ID: e5a7c9b2d416
Revises: d4f6e8a1c305
Create Date: 2026-07-21 16:00:00.000000
"""
import sqlalchemy as sa
from alembic import op
import models
# revision identifiers, used by Alembic.
revision = "e5a7c9b2d416"
down_revision = "d4f6e8a1c305"
branch_labels = None
depends_on = None
_LEDGER_VERSIONS_CHECK = (
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)"
)
_STATUS_FIELDS_CHECK = (
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'completed' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)"
)
def upgrade() -> None:
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
batch_op.add_column(sa.Column("completed_by_account_id", models.types.StringUUID(), nullable=True))
batch_op.add_column(sa.Column("completed_at", sa.DateTime(), nullable=True))
batch_op.add_column(sa.Column("completion_evidence", sa.JSON(none_as_null=True), nullable=True))
batch_op.add_column(sa.Column("completed_ledger_cas_version", sa.BigInteger(), nullable=True))
batch_op.create_check_constraint(
"kfs_cleanup_authorization_ledger_versions_ck",
_LEDGER_VERSIONS_CHECK,
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_status_fields_ck",
_STATUS_FIELDS_CHECK,
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_completion_time_ck",
"completed_at IS NULL OR completed_at >= started_at",
)
def downgrade() -> None:
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
batch_op.drop_constraint("kfs_cleanup_authorization_completion_time_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
batch_op.create_check_constraint(
"kfs_cleanup_authorization_ledger_versions_ck",
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
)
batch_op.create_check_constraint(
"kfs_cleanup_authorization_status_fields_ck",
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL)",
)
batch_op.drop_column("completed_ledger_cas_version")
batch_op.drop_column("completion_evidence")
batch_op.drop_column("completed_at")
batch_op.drop_column("completed_by_account_id")
+72
View File
@@ -66,6 +66,45 @@ from .enums import (
)
from .execution_extra_content import ExecutionExtraContent, HumanInputContent
from .human_input import HumanInputForm, HumanInputFormUploadFile, HumanInputFormUploadToken
from .knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSApiCredential,
KnowledgeFSApiCredentialStatus,
KnowledgeFSAppSpaceJoinStatus,
KnowledgeFSAppSpaceJoinType,
KnowledgeFSAuthorizationRevision,
KnowledgeFSCapabilityIssuanceAudit,
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSCapabilityIssuanceReservationStatus,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpacePermissionStatus,
KnowledgeFSControlSpaceState,
KnowledgeFSControlSpaceVisibility,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
)
from .knowledge_fs_cleanup import (
KnowledgeFSCleanupAuthorization,
KnowledgeFSCleanupAuthorizationStatus,
KnowledgeFSCleanupTarget,
)
from .knowledge_fs_cutover import (
KnowledgeFSMigrationIssue,
KnowledgeFSMigrationIssueKind,
KnowledgeFSMigrationIssueStatus,
KnowledgeFSMigrationQuarantine,
KnowledgeFSMigrationQuarantineDisposition,
KnowledgeFSMigrationQuarantineKind,
KnowledgeFSShadowAuthorizationDecision,
KnowledgeFSShadowAuthorizationDiff,
KnowledgeFSShadowAuthorizationObservation,
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
)
from .model import (
AccountTrialAppRecord,
ApiRequest,
@@ -180,6 +219,7 @@ __all__ = [
"AppAnnotationHitHistory",
"AppAnnotationSetting",
"AppDatasetJoin",
"AppKnowledgeFSSpaceJoin",
"AppMCPServer",
"AppMode",
"AppModelConfig",
@@ -224,6 +264,38 @@ __all__ = [
"IconType",
"InstalledApp",
"InvitationCode",
"KnowledgeFSApiCredential",
"KnowledgeFSApiCredentialStatus",
"KnowledgeFSAppSpaceJoinStatus",
"KnowledgeFSAppSpaceJoinType",
"KnowledgeFSAuthorizationRevision",
"KnowledgeFSCapabilityIssuanceAudit",
"KnowledgeFSCapabilityIssuanceReservation",
"KnowledgeFSCapabilityIssuanceReservationStatus",
"KnowledgeFSCleanupAuthorization",
"KnowledgeFSCleanupAuthorizationStatus",
"KnowledgeFSCleanupTarget",
"KnowledgeFSControlSpace",
"KnowledgeFSControlSpacePermission",
"KnowledgeFSControlSpacePermissionRole",
"KnowledgeFSControlSpacePermissionStatus",
"KnowledgeFSControlSpaceState",
"KnowledgeFSControlSpaceVisibility",
"KnowledgeFSExternalAccessPolicy",
"KnowledgeFSLifecycleOperation",
"KnowledgeFSLifecycleOutbox",
"KnowledgeFSLifecycleOutboxStatus",
"KnowledgeFSMigrationIssue",
"KnowledgeFSMigrationIssueKind",
"KnowledgeFSMigrationIssueStatus",
"KnowledgeFSMigrationQuarantine",
"KnowledgeFSMigrationQuarantineDisposition",
"KnowledgeFSMigrationQuarantineKind",
"KnowledgeFSShadowAuthorizationDecision",
"KnowledgeFSShadowAuthorizationDiff",
"KnowledgeFSShadowAuthorizationObservation",
"KnowledgeFSWorkspaceCutoverLedger",
"KnowledgeFSWorkspaceCutoverPhase",
"LoadBalancingModelConfig",
"Message",
"MessageAgentThought",
+713
View File
@@ -0,0 +1,713 @@
"""Dify-owned control-plane persistence for the KnowledgeFS product.
These models intentionally have no dependency on Dify Dataset or Document
models. KnowledgeFS data-plane objects remain owned by KnowledgeFS; these rows
only describe Dify authorization, registration, and durable lifecycle intent.
"""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import NotRequired, TypedDict
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, LongText, StringUUID
KnowledgeFSAllowedActions = list[str]
class KnowledgeFSCommandEnvelope(TypedDict):
schema_version: int
idempotency_key: str
expected_revision: int
class KnowledgeFSModelSelectionIntentPayload(TypedDict):
pluginId: str
provider: str
model: str
class KnowledgeFSRerankIntentPayload(TypedDict):
enabled: bool
model: NotRequired[KnowledgeFSModelSelectionIntentPayload]
class KnowledgeFSScoreThresholdIntentPayload(TypedDict):
enabled: bool
stage: str
value: NotRequired[float]
class KnowledgeFSRetrievalProfileIntentPayload(TypedDict):
defaultMode: str
reasoningModel: KnowledgeFSModelSelectionIntentPayload
rerank: KnowledgeFSRerankIntentPayload
scoreThreshold: KnowledgeFSScoreThresholdIntentPayload
topK: int
class KnowledgeFSProvisionCommandPayload(KnowledgeFSCommandEnvelope):
provisioning_key: str
name: str
icon: str | None
description: str | None
slug: str
model_intent: KnowledgeFSModelSelectionIntentPayload
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
class KnowledgeFSMetadataUpdateCommandPayload(KnowledgeFSCommandEnvelope):
metadata: dict[str, object]
class KnowledgeFSDeleteCommandPayload(KnowledgeFSCommandEnvelope):
knowledge_space_id: str | None
provisioning_key: str
class KnowledgeFSRevokeCommandPayload(KnowledgeFSCommandEnvelope):
event_id: str
grant_id: str
knowledge_space_id: str
principal: str
reason_code: str
revoke_sequence: int
class KnowledgeFSRepairCommandPayload(KnowledgeFSCommandEnvelope):
repair_reason: str
KnowledgeFSCommandPayload = (
KnowledgeFSProvisionCommandPayload
| KnowledgeFSMetadataUpdateCommandPayload
| KnowledgeFSDeleteCommandPayload
| KnowledgeFSRevokeCommandPayload
| KnowledgeFSRepairCommandPayload
)
class KnowledgeFSCapabilityAuthzRevisionSummary(TypedDict):
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
credential_revision: int | None
class KnowledgeFSCapabilityClaimsSummary(TypedDict):
action: str
actor: str
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
caller_kind: str
content_policy_revision: int
content_scope_ids: list[str]
control_space_id: str
expires_at: str
grant_id: str
issued_at: str
namespace_id: str
operation_id: str
resource_id: str
resource_parent_id: str | None
resource_type: str
subject: str
class KnowledgeFSCapabilityReservationSummary(TypedDict):
"""Sanitized, exact input bound before a capability can be signed."""
action: str
actor: str
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
caller_kind: str
content_policy_revision: int
content_scope_ids: list[str]
control_space_id: str
grant_id: str
namespace_id: str
operation_id: str
resource_id: str
resource_parent_id: str | None
resource_type: str
subject: str
trace_id: str
class KnowledgeFSControlSpaceVisibility(StrEnum):
ONLY_ME = "only_me"
ALL_TEAM_MEMBERS = "all_team_members"
PARTIAL_MEMBERS = "partial_members"
class KnowledgeFSControlSpaceState(StrEnum):
PROVISIONING = "provisioning"
ACTIVE = "active"
DELETING = "deleting"
DELETED = "deleted"
ERROR = "error"
class KnowledgeFSControlSpacePermissionRole(StrEnum):
OWNER = "owner"
EDITOR = "editor"
VIEWER = "viewer"
class KnowledgeFSControlSpacePermissionStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
class KnowledgeFSApiCredentialStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
EXPIRED = "expired"
class KnowledgeFSAppSpaceJoinType(StrEnum):
AGENT = "agent"
WORKFLOW = "workflow"
class KnowledgeFSAppSpaceJoinStatus(StrEnum):
ACTIVE = "active"
REVOKED = "revoked"
class KnowledgeFSCapabilityIssuanceReservationStatus(StrEnum):
RESERVED = "reserved"
ISSUED = "issued"
FAILED = "failed"
class KnowledgeFSLifecycleOperation(StrEnum):
PROVISION = "provision"
METADATA_UPDATE = "metadata_update"
DELETE = "delete"
REVOKE = "revoke"
REPAIR = "repair"
class KnowledgeFSLifecycleOutboxStatus(StrEnum):
PENDING = "pending"
PROCESSING = "processing"
SUCCEEDED = "succeeded"
RETRY = "retry"
DEAD_LETTER = "dead_letter"
class KnowledgeFSControlSpace(DefaultFieldsDCMixin, TypeBase):
"""Dify product resource registered to at most one KnowledgeFS Space."""
__tablename__ = "knowledge_fs_control_spaces"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id"],
["tenants.id"],
name="kfs_control_space_workspace_fk",
ondelete="RESTRICT",
),
Index(
"kfs_control_space_tenant_space_uq",
"tenant_id",
"knowledge_space_id",
unique=True,
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
),
Index("kfs_control_space_state_updated_idx", "state", "updated_at"),
Index("kfs_control_space_tenant_state_updated_idx", "tenant_id", "state", "updated_at"),
Index("kfs_control_space_tenant_owner_state_idx", "tenant_id", "owner_account_id", "state"),
sa.CheckConstraint(
"resource_version >= 0",
name=sa.schema.conv("kfs_control_space_resource_version_ck"),
),
sa.CheckConstraint("attempt_count >= 0", name=sa.schema.conv("kfs_control_space_attempt_count_ck")),
sa.CheckConstraint(
"state != 'active' OR knowledge_space_id IS NOT NULL",
name=sa.schema.conv("kfs_control_space_active_registration_ck"),
),
sa.CheckConstraint(
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
name=sa.schema.conv("kfs_control_space_irreversible_state_ck"),
),
sa.CheckConstraint(
"knowledge_space_revision >= 0",
name=sa.schema.conv("kfs_control_space_remote_revision_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
owner_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
provisioning_key: Mapped[str] = mapped_column(String(255), nullable=False)
knowledge_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
knowledge_space_revision: Mapped[int] = mapped_column(
sa.BigInteger,
nullable=False,
server_default=sa.text("0"),
default=0,
)
visibility: Mapped[KnowledgeFSControlSpaceVisibility] = mapped_column(
EnumText(KnowledgeFSControlSpaceVisibility, length=32),
nullable=False,
server_default=sa.text("'only_me'"),
default=KnowledgeFSControlSpaceVisibility.ONLY_ME,
)
lifecycle_operation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
state: Mapped[KnowledgeFSControlSpaceState] = mapped_column(
EnumText(KnowledgeFSControlSpaceState, length=32),
nullable=False,
server_default=sa.text("'provisioning'"),
default=KnowledgeFSControlSpaceState.PROVISIONING,
)
resource_version: Mapped[int] = mapped_column(
sa.BigInteger,
nullable=False,
server_default=sa.text("0"),
default=0,
)
attempt_count: Mapped[int] = mapped_column(
sa.Integer,
nullable=False,
server_default=sa.text("0"),
default=0,
)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
last_synced_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
deletion_irreversible_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
class KnowledgeFSControlSpacePermission(DefaultFieldsDCMixin, TypeBase):
"""Account-level product authorization for one control-space."""
__tablename__ = "knowledge_fs_control_space_permissions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
UniqueConstraint(
"tenant_id",
"control_space_id",
"account_id",
name="kfs_control_space_permission_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_control_space_permission_space_fk",
ondelete="RESTRICT",
),
Index("kfs_control_space_permission_account_idx", "tenant_id", "account_id", "status"),
sa.CheckConstraint(
"revision >= 0",
name=sa.schema.conv("kfs_control_space_permission_revision_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
role: Mapped[KnowledgeFSControlSpacePermissionRole] = mapped_column(
EnumText(KnowledgeFSControlSpacePermissionRole, length=32), nullable=False
)
status: Mapped[KnowledgeFSControlSpacePermissionStatus] = mapped_column(
EnumText(KnowledgeFSControlSpacePermissionStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSControlSpacePermissionStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
granted_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSExternalAccessPolicy(DefaultFieldsDCMixin, TypeBase):
"""Fail-closed caller-channel policy for a control-space."""
__tablename__ = "knowledge_fs_external_access_policies"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_external_access_policy_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_external_access_policy_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
service_api_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
agent_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
workflow_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
mcp_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
updated_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSApiCredential(DefaultFieldsDCMixin, TypeBase):
"""Hashed, resource-bound KnowledgeFS API credential metadata."""
__tablename__ = "knowledge_fs_api_credentials"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_api_credential_space_fk",
ondelete="RESTRICT",
),
Index("kfs_api_credential_tenant_space_status_idx", "tenant_id", "control_space_id", "status"),
Index("kfs_api_credential_tenant_prefix_idx", "tenant_id", "credential_prefix"),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_api_credential_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
credential_hash: Mapped[str] = mapped_column(String(255), nullable=False)
credential_prefix: Mapped[str] = mapped_column(String(32), nullable=False)
credential_last4: Mapped[str] = mapped_column(String(4), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
allowed_actions: Mapped[KnowledgeFSAllowedActions] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSApiCredentialStatus] = mapped_column(
EnumText(KnowledgeFSApiCredentialStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSApiCredentialStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoke_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
class AppKnowledgeFSSpaceJoin(DefaultFieldsDCMixin, TypeBase):
"""Explicit Agent or Workflow app authorization for a control-space."""
__tablename__ = "app_knowledge_fs_space_joins"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
UniqueConstraint(
"tenant_id",
"app_id",
"control_space_id",
"join_type",
name="app_kfs_space_join_identity_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="app_kfs_space_join_space_fk",
ondelete="RESTRICT",
),
Index("app_kfs_space_join_app_status_idx", "tenant_id", "app_id", "status"),
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("app_kfs_space_join_revision_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
join_type: Mapped[KnowledgeFSAppSpaceJoinType] = mapped_column(
EnumText(KnowledgeFSAppSpaceJoinType, length=32), nullable=False
)
status: Mapped[KnowledgeFSAppSpaceJoinStatus] = mapped_column(
EnumText(KnowledgeFSAppSpaceJoinStatus, length=32),
nullable=False,
server_default=sa.text("'active'"),
default=KnowledgeFSAppSpaceJoinStatus.ACTIVE,
)
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
class KnowledgeFSAuthorizationRevision(DefaultFieldsDCMixin, TypeBase):
"""Monotonic authorization epochs included in future Capability claims."""
__tablename__ = "knowledge_fs_authorization_revisions"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_authorization_revision_space_fk",
ondelete="RESTRICT",
),
sa.CheckConstraint(
"membership_epoch >= 0",
name=sa.schema.conv("kfs_authorization_membership_epoch_ck"),
),
sa.CheckConstraint(
"space_acl_epoch >= 0",
name=sa.schema.conv("kfs_authorization_space_acl_epoch_ck"),
),
sa.CheckConstraint(
"external_access_epoch >= 0",
name=sa.schema.conv("kfs_authorization_external_access_epoch_ck"),
),
sa.CheckConstraint(
"content_policy_revision >= 0",
name=sa.schema.conv("kfs_authorization_content_policy_revision_ck"),
),
sa.CheckConstraint(
"revoke_sequence >= 0",
name=sa.schema.conv("kfs_authorization_revoke_sequence_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
membership_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
space_acl_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
external_access_epoch: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
content_policy_revision: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
revoke_sequence: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSCapabilityIssuanceAudit(DefaultFieldsDCMixin, TypeBase):
"""Sanitized durable evidence for a Capability v2 issuance."""
__tablename__ = "knowledge_fs_capability_issuance_audits"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_audit_space_fk",
ondelete="RESTRICT",
),
Index(
"kfs_capability_issuance_audit_space_created_idx",
"tenant_id",
"control_space_id",
"created_at",
),
Index("kfs_capability_issuance_audit_trace_idx", "tenant_id", "trace_id"),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
jti_hash: Mapped[str] = mapped_column(String(80), nullable=False)
claims_summary: Mapped[KnowledgeFSCapabilityClaimsSummary] = mapped_column(sa.JSON, nullable=False)
class KnowledgeFSCapabilityIssuanceReservation(DefaultFieldsDCMixin, TypeBase):
"""Durable authorization fence written before signing starts.
The summary intentionally excludes bearer tokens, signing material, and raw JTI values.
Revocation producers scan these rows as well as completed issuance audits, closing the
authorization-to-signing race without holding a database transaction across signing.
"""
__tablename__ = "knowledge_fs_capability_issuance_reservations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
UniqueConstraint(
"tenant_id",
"grant_id",
name="kfs_capability_issuance_reservation_grant_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_capability_issuance_reservation_space_fk",
ondelete="RESTRICT",
),
Index(
"kfs_capability_issuance_reservation_subject_idx",
"tenant_id",
"control_space_id",
"subject",
"caller_kind",
),
Index(
"kfs_capability_issuance_reservation_trace_idx",
"tenant_id",
"trace_id",
),
sa.CheckConstraint(
"row_version >= 0",
name=sa.schema.conv("kfs_capability_issuance_reservation_version_ck"),
),
sa.CheckConstraint(
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
name=sa.schema.conv("kfs_capability_issuance_reservation_status_fields_ck"),
),
sa.CheckConstraint(
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
name=sa.schema.conv("kfs_capability_issuance_reservation_cleanup_window_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
grant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
subject: Mapped[str] = mapped_column(String(255), nullable=False)
caller_kind: Mapped[str] = mapped_column(String(32), nullable=False)
request_summary: Mapped[KnowledgeFSCapabilityReservationSummary] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSCapabilityIssuanceReservationStatus] = mapped_column(
EnumText(KnowledgeFSCapabilityIssuanceReservationStatus, length=16),
nullable=False,
server_default=sa.text("'reserved'"),
default=KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
)
issued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
token_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
failed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
failure_code: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
cleanup_after: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSLifecycleOutbox(DefaultFieldsDCMixin, TypeBase):
"""Durable command snapshot; it is never a product read-model source.
Provision payload writers must snapshot name, icon, description, slug,
model/profile intent, schema version, idempotency key, and expected
revision before enabling the later P1B dispatcher.
"""
__tablename__ = "knowledge_fs_lifecycle_outbox"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "control_space_id"],
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
name="kfs_lifecycle_outbox_space_fk",
ondelete="RESTRICT",
),
Index("kfs_lifecycle_outbox_dispatch_idx", "status", "next_attempt_at", "id"),
Index("kfs_lifecycle_outbox_space_created_idx", "tenant_id", "control_space_id", "created_at"),
sa.CheckConstraint(
"command_schema_version >= 1",
name=sa.schema.conv("kfs_lifecycle_outbox_schema_version_ck"),
),
sa.CheckConstraint(
"expected_control_space_version >= 0",
name=sa.schema.conv("kfs_lifecycle_outbox_expected_version_ck"),
),
sa.CheckConstraint(
"attempt_count >= 0",
name=sa.schema.conv("kfs_lifecycle_outbox_attempt_count_ck"),
),
sa.CheckConstraint(
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
name=sa.schema.conv("kfs_lifecycle_outbox_status_ck"),
),
sa.CheckConstraint(
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
name=sa.schema.conv("kfs_lifecycle_outbox_lease_state_ck"),
),
sa.CheckConstraint(
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
name=sa.schema.conv("kfs_lifecycle_outbox_terminal_state_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
operation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
operation: Mapped[KnowledgeFSLifecycleOperation] = mapped_column(
EnumText(KnowledgeFSLifecycleOperation, length=32), nullable=False
)
command_payload: Mapped[KnowledgeFSCommandPayload] = mapped_column(sa.JSON, nullable=False)
expected_control_space_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
expected_knowledge_space_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
command_schema_version: Mapped[int] = mapped_column(
sa.Integer, nullable=False, server_default=sa.text("1"), default=1
)
status: Mapped[KnowledgeFSLifecycleOutboxStatus] = mapped_column(
EnumText(KnowledgeFSLifecycleOutboxStatus, length=32),
nullable=False,
server_default=sa.text("'pending'"),
default=KnowledgeFSLifecycleOutboxStatus.PENDING,
)
attempt_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"), default=0)
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
lease_owner: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
retain_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
__all__ = [
"AppKnowledgeFSSpaceJoin",
"KnowledgeFSAllowedActions",
"KnowledgeFSApiCredential",
"KnowledgeFSApiCredentialStatus",
"KnowledgeFSAppSpaceJoinStatus",
"KnowledgeFSAppSpaceJoinType",
"KnowledgeFSAuthorizationRevision",
"KnowledgeFSCapabilityAuthzRevisionSummary",
"KnowledgeFSCapabilityClaimsSummary",
"KnowledgeFSCapabilityIssuanceAudit",
"KnowledgeFSCapabilityIssuanceReservation",
"KnowledgeFSCapabilityIssuanceReservationStatus",
"KnowledgeFSCapabilityReservationSummary",
"KnowledgeFSCommandEnvelope",
"KnowledgeFSCommandPayload",
"KnowledgeFSControlSpace",
"KnowledgeFSControlSpacePermission",
"KnowledgeFSControlSpacePermissionRole",
"KnowledgeFSControlSpacePermissionStatus",
"KnowledgeFSControlSpaceState",
"KnowledgeFSControlSpaceVisibility",
"KnowledgeFSDeleteCommandPayload",
"KnowledgeFSExternalAccessPolicy",
"KnowledgeFSLifecycleOperation",
"KnowledgeFSLifecycleOutbox",
"KnowledgeFSLifecycleOutboxStatus",
"KnowledgeFSMetadataUpdateCommandPayload",
"KnowledgeFSModelSelectionIntentPayload",
"KnowledgeFSProvisionCommandPayload",
"KnowledgeFSRepairCommandPayload",
"KnowledgeFSRerankIntentPayload",
"KnowledgeFSRetrievalProfileIntentPayload",
"KnowledgeFSRevokeCommandPayload",
"KnowledgeFSScoreThresholdIntentPayload",
]
+130
View File
@@ -0,0 +1,130 @@
"""Persistent authorization for the irreversible P9 KnowledgeFS cleanup window."""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, StringUUID
class KnowledgeFSCleanupAuthorizationStatus(StrEnum):
REQUESTED = "requested"
APPROVED = "approved"
STARTED = "started"
COMPLETED = "completed"
class KnowledgeFSCleanupTarget(StrEnum):
LEGACY_SNAPSHOT_FOREIGN_KEYS = "legacy_snapshot_foreign_keys"
LEGACY_ACL_ROUTES = "legacy_acl_routes"
LEGACY_ACL_SCHEMA = "legacy_acl_schema"
LEGACY_API_KEY_SCHEMA = "legacy_api_key_schema"
LEGACY_V1_AUTH = "legacy_v1_auth"
RAW_LIST_CREATE_PROXY = "raw_list_create_proxy"
class KnowledgeFSCleanupAuthorization(DefaultFieldsDCMixin, TypeBase):
"""Four-eyes approval and immutable evidence for one cleanup start fence."""
__tablename__ = "knowledge_fs_cleanup_authorizations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "request_id", name="kfs_cleanup_authorization_request_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_cleanup_authorization_ledger_fk",
),
Index("kfs_cleanup_authorization_status_idx", "tenant_id", "status", "updated_at"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_cleanup_authorization_version_ck")),
sa.CheckConstraint(
"readiness_ledger_cas_version >= 0 "
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)",
name=sa.schema.conv("kfs_cleanup_authorization_ledger_versions_ck"),
),
sa.CheckConstraint(
"(status = 'requested' "
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'approved' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NULL AND started_at IS NULL "
"AND started_ledger_cas_version IS NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'started' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
"(status = 'completed' "
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
"AND started_ledger_cas_version IS NOT NULL "
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)",
name=sa.schema.conv("kfs_cleanup_authorization_status_fields_ck"),
),
sa.CheckConstraint(
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
name=sa.schema.conv("kfs_cleanup_authorization_approval_window_ck"),
),
sa.CheckConstraint(
"completed_at IS NULL OR completed_at >= started_at",
name=sa.schema.conv("kfs_cleanup_authorization_completion_time_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
request_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
plan_schema_version: Mapped[str] = mapped_column(String(32), nullable=False)
plan_digest: Mapped[str] = mapped_column(String(71), nullable=False)
targets: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False)
readiness_evidence: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False)
requested_by_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
requested_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
readiness_ledger_cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
status: Mapped[KnowledgeFSCleanupAuthorizationStatus] = mapped_column(
EnumText(KnowledgeFSCleanupAuthorizationStatus, length=16),
nullable=False,
server_default=sa.text("'requested'"),
default=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
approval_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
approved_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
started_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
started_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
completion_evidence: Mapped[dict[str, object] | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
completed_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
__all__ = [
"KnowledgeFSCleanupAuthorization",
"KnowledgeFSCleanupAuthorizationStatus",
"KnowledgeFSCleanupTarget",
]
+507
View File
@@ -0,0 +1,507 @@
"""Workspace-scoped KnowledgeFS migration and cutover persistence.
The ledger is Dify control-plane state only. It deliberately stores opaque
KnowledgeFS identifiers and authorization watermarks without foreign keys or
queries to legacy product tables. Cutover feature switches live on one
row so the repository can change them with a single CAS statement. Quarantine
remediation is separately CAS-versioned and retains immutable operator evidence.
"""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import TypedDict
import sqlalchemy as sa
from sqlalchemy import DateTime, Index, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from .base import DefaultFieldsDCMixin, TypeBase
from .types import EnumText, LongText, StringUUID
class KnowledgeFSCutoverRevisionWatermark(TypedDict):
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
content_policy_revision: int
class KnowledgeFSCutoverSmokeChecks(TypedDict):
authorization: bool
list_spaces: bool
create_space: bool
query: bool
upload: bool
stream: bool
deletion: bool
class KnowledgeFSCutoverSmokeEvidenceReferences(TypedDict):
authorization: str
list_spaces: str
create_space: str
query: str
upload: str
stream: str
deletion: str
class KnowledgeFSCutoverSmokeResults(TypedDict):
schema_version: str
tenant_id: str
environment: str
operator: str
operator_account_id: str
observed_at: str
checks: KnowledgeFSCutoverSmokeChecks
evidence_references: KnowledgeFSCutoverSmokeEvidenceReferences
def knowledge_fs_cutover_smoke_results_passed(value: KnowledgeFSCutoverSmokeResults | None) -> bool:
"""Reject incomplete or pre-production JSON evidence instead of treating metadata as booleans."""
if not isinstance(value, dict):
return False
checks = value.get("checks")
references = value.get("evidence_references")
expected = {
"authorization",
"list_spaces",
"create_space",
"query",
"upload",
"stream",
"deletion",
}
return bool(
value.get("schema_version") == "knowledge-fs-p8-cutover-smoke/v1"
and value.get("environment") == "production"
and isinstance(checks, dict)
and set(checks) == expected
and all(checks.get(name) is True for name in expected)
and isinstance(references, dict)
and set(references) == expected
and all(isinstance(reference := references.get(name), str) and bool(reference) for name in expected)
)
class KnowledgeFSWorkspaceCutoverPhase(StrEnum):
INVENTORY = "inventory"
BACKFILL = "backfill"
SHADOW = "shadow"
FROZEN = "frozen"
CUTOVER = "cutover"
OBSERVING = "observing"
READY_FOR_CLEANUP = "ready_for_cleanup"
class KnowledgeFSMigrationIssueStatus(StrEnum):
OPEN = "open"
APPROVED_FAIL_CLOSED = "approved_fail_closed"
RESOLVED = "resolved"
class KnowledgeFSMigrationIssueKind(StrEnum):
REGISTRATION_CONFLICT = "registration_conflict"
UNRESOLVED_SUBJECT = "unresolved_subject"
UNKNOWN_EXTERNAL_ACCESS = "unknown_external_access"
REVISION_DRIFT = "revision_drift"
TASK_QUARANTINE = "task_quarantine"
LEGACY_SNAPSHOT_DEPENDENCY = "legacy_snapshot_dependency"
LEGACY_FOREIGN_KEY_DEPENDENCY = "legacy_foreign_key_dependency"
class KnowledgeFSMigrationQuarantineKind(StrEnum):
CONTROL_SPACE = "control_space"
SUBJECT = "subject"
TASK = "task"
LEGACY_API_KEY = "legacy_api_key"
ORPHAN_RESOURCE = "orphan_resource"
class KnowledgeFSMigrationQuarantineDisposition(StrEnum):
PENDING = "pending"
MIGRATABLE = "migratable"
WAIT_FOR_COMPLETION = "wait_for_completion"
CANCEL = "cancel"
ISOLATE = "isolate"
ROTATE_CREDENTIAL = "rotate_credential"
RESOLVED = "resolved"
class KnowledgeFSShadowAuthorizationDecision(StrEnum):
MATCH = "match"
TIGHTENED = "tightened"
EXPANDED = "expanded"
UNKNOWN = "unknown"
class KnowledgeFSWorkspaceCutoverLedger(DefaultFieldsDCMixin, TypeBase):
"""Single-Workspace migration state, watermarks, switches, and time fences."""
__tablename__ = "knowledge_fs_workspace_cutover_ledgers"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
Index("kfs_workspace_cutover_ledger_phase_updated_idx", "phase", "updated_at"),
sa.CheckConstraint("cas_version >= 0", name=sa.schema.conv("kfs_workspace_cutover_cas_version_ck")),
sa.CheckConstraint(
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
name=sa.schema.conv("kfs_workspace_cutover_task_watermark_ck"),
),
sa.CheckConstraint(
"cutover_at IS NULL OR freeze_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_freeze_before_cutover_ck"),
),
sa.CheckConstraint(
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_rollback_cutoff_ck"),
),
sa.CheckConstraint(
"shadow_observation_count >= 0",
name=sa.schema.conv("kfs_workspace_cutover_shadow_count_ck"),
),
sa.CheckConstraint(
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
name=sa.schema.conv("kfs_workspace_cutover_shadow_window_ck"),
),
sa.CheckConstraint(
"(shadow_completed_at IS NULL "
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
"AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
name=sa.schema.conv("kfs_workspace_cutover_shadow_completion_fields_ck"),
),
sa.CheckConstraint(
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_fields_ck"),
),
sa.CheckConstraint(
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_time_ck"),
),
sa.CheckConstraint(
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
"AND remote_activation_replayed IS NULL) OR "
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
"AND remote_activation_acknowledged_at IS NOT NULL "
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_fields_ck"),
),
sa.CheckConstraint(
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_time_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
source_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
applied_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
phase: Mapped[KnowledgeFSWorkspaceCutoverPhase] = mapped_column(
EnumText(KnowledgeFSWorkspaceCutoverPhase, length=32),
nullable=False,
server_default=sa.text("'inventory'"),
default=KnowledgeFSWorkspaceCutoverPhase.INVENTORY,
)
final_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
sa.JSON, nullable=True, default=None
)
source_task_watermark: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
final_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
applied_task_watermark: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
shadow_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_evidence_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
shadow_observation_count: Mapped[int] = mapped_column(
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
)
shadow_window_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_window_ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
shadow_traffic_zero: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
shadow_traffic_zero_evidence: Mapped[dict[str, object] | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
shadow_latest_observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
sa.JSON(none_as_null=True), nullable=True, default=None
)
shadow_producer: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
shadow_completed_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
shadow_completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_freeze_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
remote_freeze_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_freeze_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
remote_freeze_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_freeze_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_freeze_frozen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_freeze_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_freeze_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_activation_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
remote_activation_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
remote_activation_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
remote_activation_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
remote_activation_activated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
remote_activation_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
remote_activation_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
freeze_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
cutover_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
rollback_cutoff_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_window_ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
observation_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
maximum_task_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
irreversible_cleanup_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
product_routes_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
capability_v2_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
integrated_mode_enabled: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
legacy_acl_read_only: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
smoke_results: Mapped[KnowledgeFSCutoverSmokeResults | None] = mapped_column(sa.JSON, nullable=True, default=None)
legacy_dependency_report: Mapped[list[dict[str, object]] | None] = mapped_column(
sa.JSON, nullable=True, default=None
)
legacy_dependency_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
legacy_dependency_ready: Mapped[bool] = mapped_column(
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
)
cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSMigrationIssue(DefaultFieldsDCMixin, TypeBase):
"""Auditable blocker; unknown access can only be approved as fail-closed."""
__tablename__ = "knowledge_fs_migration_issues"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_issue_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_migration_issue_gate_idx", "tenant_id", "ledger_id", "status", "kind"),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
issue_key: Mapped[str] = mapped_column(String(255), nullable=False)
kind: Mapped[KnowledgeFSMigrationIssueKind] = mapped_column(
EnumText(KnowledgeFSMigrationIssueKind, length=48), nullable=False
)
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
EnumText(KnowledgeFSMigrationIssueStatus, length=32),
nullable=False,
server_default=sa.text("'open'"),
default=KnowledgeFSMigrationIssueStatus.OPEN,
)
resource_type: Mapped[str | None] = mapped_column(String(64), nullable=True, default=None)
resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
class KnowledgeFSMigrationQuarantine(DefaultFieldsDCMixin, TypeBase):
"""Fail-closed inventory item whose operator resolution is immutable and CAS-versioned."""
__tablename__ = "knowledge_fs_migration_quarantine"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
UniqueConstraint(
"tenant_id",
"ledger_id",
"source_kind",
"source_id",
name="kfs_migration_quarantine_source_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_migration_quarantine_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_migration_quarantine_disposition_idx", "tenant_id", "ledger_id", "disposition"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_migration_quarantine_version_ck")),
sa.CheckConstraint(
"(disposition = 'resolved' "
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
"(disposition <> 'resolved' "
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
"AND evidence IS NULL AND resolved_at IS NULL)",
name=sa.schema.conv("kfs_migration_quarantine_resolution_fields_ck"),
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
source_kind: Mapped[KnowledgeFSMigrationQuarantineKind] = mapped_column(
EnumText(KnowledgeFSMigrationQuarantineKind, length=32), nullable=False
)
source_id: Mapped[str] = mapped_column(String(255), nullable=False)
reason_code: Mapped[str] = mapped_column(String(64), nullable=False)
disposition: Mapped[KnowledgeFSMigrationQuarantineDisposition] = mapped_column(
EnumText(KnowledgeFSMigrationQuarantineDisposition, length=32), nullable=False
)
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
resolved_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
evidence: Mapped[dict[str, object] | None] = mapped_column(sa.JSON(none_as_null=True), nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSShadowAuthorizationDiff(DefaultFieldsDCMixin, TypeBase):
"""Current disposition for a diff key; append-only observations retain its history."""
__tablename__ = "knowledge_fs_shadow_authorization_diffs"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_diff_ledger_fk",
ondelete="CASCADE",
),
Index("kfs_shadow_authorization_diff_gate_idx", "tenant_id", "ledger_id", "status", "decision"),
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_shadow_authorization_diff_version_ck")),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
)
reason: Mapped[str] = mapped_column(LongText, nullable=False)
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
EnumText(KnowledgeFSMigrationIssueStatus, length=32), nullable=False
)
current_evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
last_observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
class KnowledgeFSShadowAuthorizationObservation(DefaultFieldsDCMixin, TypeBase):
"""Append-only producer evidence used to derive a shadow diff disposition."""
__tablename__ = "knowledge_fs_shadow_authorization_observations"
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
UniqueConstraint(
"tenant_id",
"ledger_id",
"diff_key",
"evidence_digest",
name="kfs_shadow_authorization_observation_evidence_uq",
),
sa.ForeignKeyConstraint(
["tenant_id", "ledger_id"],
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
name="kfs_shadow_authorization_observation_ledger_fk",
ondelete="CASCADE",
),
Index(
"kfs_shadow_authorization_observation_window_idx",
"tenant_id",
"ledger_id",
"observed_at",
),
)
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
producer: Mapped[str] = mapped_column(String(255), nullable=False)
principal: Mapped[str] = mapped_column(String(255), nullable=False)
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
)
reason: Mapped[str] = mapped_column(LongText, nullable=False)
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
__all__ = [
"KnowledgeFSCutoverRevisionWatermark",
"KnowledgeFSCutoverSmokeChecks",
"KnowledgeFSCutoverSmokeEvidenceReferences",
"KnowledgeFSCutoverSmokeResults",
"KnowledgeFSMigrationIssue",
"KnowledgeFSMigrationIssueKind",
"KnowledgeFSMigrationIssueStatus",
"KnowledgeFSMigrationQuarantine",
"KnowledgeFSMigrationQuarantineDisposition",
"KnowledgeFSMigrationQuarantineKind",
"KnowledgeFSShadowAuthorizationDecision",
"KnowledgeFSShadowAuthorizationDiff",
"KnowledgeFSShadowAuthorizationObservation",
"KnowledgeFSWorkspaceCutoverLedger",
"KnowledgeFSWorkspaceCutoverPhase",
"knowledge_fs_cutover_smoke_results_passed",
]
@@ -0,0 +1,53 @@
"""Persistence contract for irreversible KnowledgeFS cleanup authorization."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs_cleanup import (
KnowledgeFSCleanupAuthorization,
KnowledgeFSCleanupAuthorizationStatus,
)
@dataclass(frozen=True, slots=True)
class KnowledgeFSCleanupAuthorizationCASUpdate:
tenant_id: str
ledger_id: str
request_id: str
expected_status: KnowledgeFSCleanupAuthorizationStatus
expected_row_version: int
new_status: KnowledgeFSCleanupAuthorizationStatus
approved_by_account_id: str | None = None
approved_at: datetime | None = None
approval_expires_at: datetime | None = None
approved_ledger_cas_version: int | None = None
started_by_account_id: str | None = None
started_at: datetime | None = None
started_ledger_cas_version: int | None = None
completed_by_account_id: str | None = None
completed_at: datetime | None = None
completion_evidence: dict[str, object] | None = None
completed_ledger_cas_version: int | None = None
class KnowledgeFSCleanupAuthorizationRepository(Protocol):
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization: ...
def get(
self,
*,
tenant_id: str,
ledger_id: str,
request_id: str,
) -> KnowledgeFSCleanupAuthorization | None: ...
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool: ...
__all__ = [
"KnowledgeFSCleanupAuthorizationCASUpdate",
"KnowledgeFSCleanupAuthorizationRepository",
]
@@ -0,0 +1,60 @@
"""Persistence boundary for tenant-scoped KnowledgeFS lifecycle operations."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
@dataclass(frozen=True, slots=True)
class KnowledgeFSControlSpaceCASUpdate:
"""Values applied only when the stored state and resource version match."""
tenant_id: str
control_space_id: str
expected_resource_version: int
expected_state: KnowledgeFSControlSpaceState
new_state: KnowledgeFSControlSpaceState
lifecycle_operation_id: str
knowledge_space_id: str | None = None
knowledge_space_revision: int | None = None
attempted_at: datetime | None = None
last_error_code: str | None = None
last_error_message: str | None = None
class KnowledgeFSControlSpaceRepository(Protocol):
"""Repository operations that always include the owning Dify tenant."""
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace: ...
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None: ...
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None: ...
def find_by_knowledge_space_id(
self, *, tenant_id: str, knowledge_space_id: str
) -> KnowledgeFSControlSpace | None: ...
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]: ...
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]: ...
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool: ...
def mark_deletion_irreversible(
self,
*,
tenant_id: str,
control_space_id: str,
lifecycle_operation_id: str,
irreversible_at: datetime,
) -> bool: ...
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]: ...
__all__ = ["KnowledgeFSControlSpaceCASUpdate", "KnowledgeFSControlSpaceRepository"]
@@ -0,0 +1,229 @@
"""Persistence contract for KnowledgeFS workspace migration and cutover state."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Protocol
from models.knowledge_fs_cutover import (
KnowledgeFSCutoverRevisionWatermark,
KnowledgeFSCutoverSmokeResults,
KnowledgeFSMigrationIssue,
KnowledgeFSMigrationIssueStatus,
KnowledgeFSMigrationQuarantine,
KnowledgeFSMigrationQuarantineDisposition,
KnowledgeFSMigrationQuarantineKind,
KnowledgeFSShadowAuthorizationDecision,
KnowledgeFSShadowAuthorizationDiff,
KnowledgeFSShadowAuthorizationObservation,
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
)
@dataclass(frozen=True, slots=True)
class KnowledgeFSCutoverCASUpdate:
"""Optional values changed with the phase and CAS version in one SQL statement."""
tenant_id: str
expected_phase: KnowledgeFSWorkspaceCutoverPhase
expected_cas_version: int
new_phase: KnowledgeFSWorkspaceCutoverPhase
source_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
final_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
applied_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
source_task_watermark: int | None = None
final_task_watermark: int | None = None
applied_task_watermark: int | None = None
shadow_started_at: datetime | None = None
shadow_completed_at: datetime | None = None
shadow_evidence_digest: str | None = None
shadow_observation_count: int | None = None
shadow_window_started_at: datetime | None = None
shadow_window_ended_at: datetime | None = None
shadow_traffic_zero: bool | None = None
shadow_traffic_zero_evidence: dict[str, object] | None = None
shadow_latest_observed_revision: KnowledgeFSCutoverRevisionWatermark | None = None
shadow_producer: str | None = None
shadow_completed_by_operator: str | None = None
shadow_completed_by_account_id: str | None = None
remote_freeze_id: str | None = None
remote_freeze_revision: int | None = None
remote_freeze_digest: str | None = None
remote_freeze_task_watermark: int | None = None
remote_freeze_control_space_id: str | None = None
remote_freeze_frozen_at: datetime | None = None
remote_freeze_updated_at: datetime | None = None
remote_freeze_acknowledged_at: datetime | None = None
remote_freeze_applied: bool | None = None
remote_freeze_replayed: bool | None = None
remote_activation_id: str | None = None
remote_activation_revision: int | None = None
remote_activation_digest: str | None = None
remote_activation_control_space_id: str | None = None
remote_activation_activated_at: datetime | None = None
remote_activation_updated_at: datetime | None = None
remote_activation_acknowledged_at: datetime | None = None
remote_activation_applied: bool | None = None
remote_activation_replayed: bool | None = None
clear_smoke_results: bool = False
freeze_at: datetime | None = None
cutover_at: datetime | None = None
rolled_back_at: datetime | None = None
rollback_cutoff_at: datetime | None = None
observation_started_at: datetime | None = None
observation_window_ends_at: datetime | None = None
observation_completed_at: datetime | None = None
maximum_task_expires_at: datetime | None = None
irreversible_cleanup_at: datetime | None = None
smoke_results: KnowledgeFSCutoverSmokeResults | None = None
legacy_dependency_report: list[dict[str, object]] | None = None
legacy_dependency_checked_at: datetime | None = None
legacy_dependency_ready: bool | None = None
product_routes_enabled: bool | None = None
capability_v2_enabled: bool | None = None
integrated_mode_enabled: bool | None = None
legacy_acl_read_only: bool | None = None
@dataclass(frozen=True, slots=True)
class KnowledgeFSQuarantineCASUpdate:
"""Tenant/item-scoped transition from an operator disposition to resolved."""
tenant_id: str
ledger_id: str
source_kind: KnowledgeFSMigrationQuarantineKind
source_id: str
expected_disposition: KnowledgeFSMigrationQuarantineDisposition
expected_row_version: int
resolved_by_operator: str
resolved_by_account_id: str
evidence: dict[str, object]
resolved_at: datetime
@dataclass(frozen=True, slots=True)
class KnowledgeFSShadowDiffCASUpdate:
"""Replace one current diff disposition after append-only safe re-evaluation."""
tenant_id: str
ledger_id: str
diff_key: str
expected_row_version: int
control_space_id: str | None
principal: str
legacy_allowed: bool | None
dify_allowed: bool
decision: KnowledgeFSShadowAuthorizationDecision
reason: str
observed_revision: KnowledgeFSCutoverRevisionWatermark
status: KnowledgeFSMigrationIssueStatus
current_evidence_digest: str
last_observed_at: datetime
class KnowledgeFSCutoverRepository(Protocol):
"""Tenant-scoped storage; callers own the surrounding transaction."""
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger: ...
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None: ...
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]: ...
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool: ...
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue: ...
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None: ...
def set_issue_status(
self,
*,
tenant_id: str,
ledger_id: str,
issue_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool: ...
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine: ...
def get_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kind: KnowledgeFSMigrationQuarantineKind,
source_id: str,
) -> KnowledgeFSMigrationQuarantine | None: ...
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool: ...
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff: ...
def get_shadow_diff(
self, *, tenant_id: str, ledger_id: str, diff_key: str
) -> KnowledgeFSShadowAuthorizationDiff | None: ...
def set_shadow_diff_status(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool: ...
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool: ...
def add_shadow_observation(
self, observation: KnowledgeFSShadowAuthorizationObservation
) -> KnowledgeFSShadowAuthorizationObservation: ...
def get_shadow_observation(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
evidence_digest: str,
) -> KnowledgeFSShadowAuthorizationObservation | None: ...
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int: ...
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int: ...
def count_unresolved_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
) -> int: ...
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]: ...
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]: ...
def list_shadow_diffs(
self, *, tenant_id: str, ledger_id: str
) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]: ...
def list_shadow_observations(
self, *, tenant_id: str, ledger_id: str
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]: ...
__all__ = [
"KnowledgeFSCutoverCASUpdate",
"KnowledgeFSCutoverRepository",
"KnowledgeFSQuarantineCASUpdate",
"KnowledgeFSShadowDiffCASUpdate",
]
@@ -0,0 +1,95 @@
"""Persistence boundary for durable KnowledgeFS lifecycle commands."""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import Protocol
from models.knowledge_fs import KnowledgeFSLifecycleOperation, KnowledgeFSLifecycleOutbox
class KnowledgeFSLifecycleOutboxRepository(Protocol):
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox: ...
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
def find_open_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None: ...
def find_latest_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None: ...
def reactivate_dead_letter(self, *, outbox_id: str) -> bool: ...
def supersede_unattempted(
self,
*,
outbox_id: str,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def supersede_after_remote_absence(
self,
*,
outbox_id: str,
observed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def claim_next(
self,
*,
lease_owner: str,
now: datetime,
lease_duration: timedelta,
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
) -> KnowledgeFSLifecycleOutbox | None: ...
def acknowledge(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
) -> bool: ...
def schedule_retry(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
next_attempt_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
def mark_dead_letter(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool: ...
__all__ = ["KnowledgeFSLifecycleOutboxRepository"]
@@ -0,0 +1,35 @@
"""Durable SQLAlchemy sink for sanitized KnowledgeFS capability issuance audits."""
from __future__ import annotations
from typing import cast, override
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import KnowledgeFSCapabilityClaimsSummary, KnowledgeFSCapabilityIssuanceAudit
from services.knowledge_fs_capability import CapabilityIssuanceAuditEvent, CapabilityIssuanceAuditor
class SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor(CapabilityIssuanceAuditor):
"""Commit the audit independently so token issuance fails closed on audit failure."""
def __init__(self, session_maker: sessionmaker[Session]):
self._session_maker = session_maker
@override
def record(self, event: CapabilityIssuanceAuditEvent) -> None:
serialized = event.model_dump(mode="json")
trace_id = cast(str, serialized.pop("trace_id"))
jti_hash = cast(str, serialized.pop("jti_hash"))
audit = KnowledgeFSCapabilityIssuanceAudit(
tenant_id=event.namespace_id,
control_space_id=event.control_space_id,
trace_id=trace_id,
jti_hash=jti_hash,
claims_summary=cast(KnowledgeFSCapabilityClaimsSummary, serialized),
)
with self._session_maker.begin() as session:
session.add(audit)
__all__ = ["SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor"]
@@ -0,0 +1,185 @@
"""Transactional persistence for pre-signing KnowledgeFS capability reservations."""
from __future__ import annotations
from datetime import datetime, timedelta
from typing import cast
import sqlalchemy as sa
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs import (
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSCapabilityIssuanceReservationStatus,
KnowledgeFSCapabilityReservationSummary,
)
from services.knowledge_fs_capability import (
CAPABILITY_ISSUANCE_PROFILES,
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
CapabilityIssueRequest,
)
_TERMINAL_RETENTION = timedelta(days=1)
class KnowledgeFSCapabilityIssuanceReservationError(RuntimeError):
"""A retry attempted to change or lose a durable issuance binding."""
class SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository:
def __init__(self, session: Session) -> None:
self._session = session
def reserve(self, request: CapabilityIssueRequest) -> KnowledgeFSCapabilityIssuanceReservation:
summary = _request_summary(request)
reservation = self._session.scalar(
sa.select(KnowledgeFSCapabilityIssuanceReservation)
.where(
KnowledgeFSCapabilityIssuanceReservation.tenant_id == request.namespace_id,
KnowledgeFSCapabilityIssuanceReservation.grant_id == request.grant_id,
)
.with_for_update()
)
if reservation is None:
reservation = KnowledgeFSCapabilityIssuanceReservation(
tenant_id=request.namespace_id,
control_space_id=request.control_space_id,
grant_id=request.grant_id,
trace_id=request.trace_id,
subject=summary["subject"],
caller_kind=request.caller_kind,
request_summary=summary,
)
self._session.add(reservation)
self._session.flush()
return reservation
if (
reservation.control_space_id != request.control_space_id
or reservation.trace_id != request.trace_id
or reservation.subject != summary["subject"]
or reservation.caller_kind != request.caller_kind
or reservation.request_summary != summary
):
raise KnowledgeFSCapabilityIssuanceReservationError(
"Capability grant retry does not match its durable reservation"
)
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.FAILED:
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED
reservation.failed_at = None
reservation.failure_code = None
reservation.cleanup_after = None
reservation.row_version += 1
self._session.flush()
return reservation
def mark_issued(
self,
*,
tenant_id: str,
grant_id: str,
issued_at: datetime,
token_expires_at: datetime,
) -> None:
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED
reservation.issued_at = issued_at
reservation.token_expires_at = token_expires_at
reservation.failed_at = None
reservation.failure_code = None
reservation.cleanup_after = token_expires_at + _TERMINAL_RETENTION
reservation.row_version += 1
self._session.flush()
def mark_failed(
self,
*,
tenant_id: str,
grant_id: str,
failed_at: datetime,
failure_code: str,
) -> None:
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED:
return
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.FAILED
reservation.issued_at = None
reservation.token_expires_at = None
reservation.failed_at = failed_at
reservation.failure_code = failure_code[:128]
reservation.cleanup_after = failed_at + _TERMINAL_RETENTION
reservation.row_version += 1
self._session.flush()
def cleanup_terminal(self, *, before: datetime, limit: int = 1_000) -> int:
if limit <= 0:
raise ValueError("Capability reservation cleanup limit must be positive")
reservation_ids = tuple(
self._session.scalars(
sa.select(KnowledgeFSCapabilityIssuanceReservation.id)
.where(
KnowledgeFSCapabilityIssuanceReservation.status
!= KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
KnowledgeFSCapabilityIssuanceReservation.cleanup_after <= before,
)
.order_by(
KnowledgeFSCapabilityIssuanceReservation.cleanup_after,
KnowledgeFSCapabilityIssuanceReservation.id,
)
.limit(limit)
.with_for_update()
)
)
if not reservation_ids:
return 0
result = self._session.execute(
sa.delete(KnowledgeFSCapabilityIssuanceReservation).where(
KnowledgeFSCapabilityIssuanceReservation.id.in_(reservation_ids)
)
)
return cast(CursorResult[tuple[object, ...]], result).rowcount or 0
def _locked(self, *, tenant_id: str, grant_id: str) -> KnowledgeFSCapabilityIssuanceReservation:
reservation = self._session.scalar(
sa.select(KnowledgeFSCapabilityIssuanceReservation)
.where(
KnowledgeFSCapabilityIssuanceReservation.tenant_id == tenant_id,
KnowledgeFSCapabilityIssuanceReservation.grant_id == grant_id,
)
.with_for_update()
)
if reservation is None:
raise KnowledgeFSCapabilityIssuanceReservationError("Capability issuance reservation is missing")
return reservation
def _request_summary(request: CapabilityIssueRequest) -> KnowledgeFSCapabilityReservationSummary:
operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[request.operation_id]
profile = CAPABILITY_ISSUANCE_PROFILES[request.caller_kind]
subject = f"{profile.subject_prefix}:{request.principal_id}"
return cast(
KnowledgeFSCapabilityReservationSummary,
{
"action": operation.action,
"actor": request.actor,
"authz_revision": request.authz_revision.model_dump(mode="json"),
"caller_kind": request.caller_kind,
"content_policy_revision": request.content_policy_revision,
"content_scope_ids": list(request.content_scope_ids),
"control_space_id": request.control_space_id,
"grant_id": request.grant_id,
"namespace_id": request.namespace_id,
"operation_id": request.operation_id,
"resource_id": request.resource.id,
"resource_parent_id": request.resource.parent_id,
"resource_type": request.resource.type,
"subject": subject,
"trace_id": request.trace_id,
},
)
__all__ = [
"KnowledgeFSCapabilityIssuanceReservationError",
"SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository",
]
@@ -0,0 +1,87 @@
"""SQLAlchemy cleanup-authorization repository with status and version CAS."""
from __future__ import annotations
from typing import cast, override
from sqlalchemy import select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs_cleanup import KnowledgeFSCleanupAuthorization
from repositories.knowledge_fs_cleanup_repository import (
KnowledgeFSCleanupAuthorizationCASUpdate,
KnowledgeFSCleanupAuthorizationRepository,
)
class SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(KnowledgeFSCleanupAuthorizationRepository):
_session: Session
def __init__(self, session: Session):
self._session = session
@override
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization:
self._session.add(authorization)
self._session.flush()
return authorization
@override
def get(
self,
*,
tenant_id: str,
ledger_id: str,
request_id: str,
) -> KnowledgeFSCleanupAuthorization | None:
statement = (
select(KnowledgeFSCleanupAuthorization)
.where(
KnowledgeFSCleanupAuthorization.tenant_id == tenant_id,
KnowledgeFSCleanupAuthorization.ledger_id == ledger_id,
KnowledgeFSCleanupAuthorization.request_id == request_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool:
values: dict[str, object] = {
"status": update_values.new_status,
"row_version": KnowledgeFSCleanupAuthorization.row_version + 1,
}
for field_name in (
"approved_by_account_id",
"approved_at",
"approval_expires_at",
"approved_ledger_cas_version",
"started_by_account_id",
"started_at",
"started_ledger_cas_version",
"completed_by_account_id",
"completed_at",
"completion_evidence",
"completed_ledger_cas_version",
):
value = getattr(update_values, field_name)
if value is not None:
values[field_name] = value
statement = (
update(KnowledgeFSCleanupAuthorization)
.where(
KnowledgeFSCleanupAuthorization.tenant_id == update_values.tenant_id,
KnowledgeFSCleanupAuthorization.ledger_id == update_values.ledger_id,
KnowledgeFSCleanupAuthorization.request_id == update_values.request_id,
KnowledgeFSCleanupAuthorization.status == update_values.expected_status,
KnowledgeFSCleanupAuthorization.row_version == update_values.expected_row_version,
)
.values(**values)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
__all__ = ["SQLAlchemyKnowledgeFSCleanupAuthorizationRepository"]
@@ -0,0 +1,187 @@
"""SQLAlchemy implementation of the KnowledgeFS control-space repository."""
from __future__ import annotations
from datetime import datetime
from typing import cast, override
from sqlalchemy import case, select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
from repositories.knowledge_fs_control_space_repository import (
KnowledgeFSControlSpaceCASUpdate,
KnowledgeFSControlSpaceRepository,
)
class SQLAlchemyKnowledgeFSControlSpaceRepository(KnowledgeFSControlSpaceRepository):
"""Caller-transaction-owned repository with tenant and CAS guards."""
def __init__(self, session: Session):
self._session = session
@override
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace:
self._session.add(control_space)
self._session.flush()
return control_space
@override
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.provisioning_key == provisioning_key)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_by_knowledge_space_id(self, *, tenant_id: str, knowledge_space_id: str) -> KnowledgeFSControlSpace | None:
statement = (
select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.knowledge_space_id == knowledge_space_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]:
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
.execution_options(populate_existing=True)
)
return tuple(self._session.scalars(statement).all())
@override
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]:
if limit <= 0:
return ()
statement = (
select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED)
.order_by(
case(
(
KnowledgeFSControlSpace.state.in_(
(
KnowledgeFSControlSpaceState.PROVISIONING,
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
)
),
0,
),
else_=1,
),
KnowledgeFSControlSpace.updated_at,
KnowledgeFSControlSpace.id,
)
.limit(limit)
.execution_options(populate_existing=True)
)
return tuple(self._session.scalars(statement).all())
@override
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool:
values: dict[str, object] = {
"state": update_values.new_state,
"resource_version": KnowledgeFSControlSpace.resource_version + 1,
"lifecycle_operation_id": update_values.lifecycle_operation_id,
"last_error_code": update_values.last_error_code,
"last_error_message": update_values.last_error_message,
}
if update_values.knowledge_space_id is not None:
values["knowledge_space_id"] = update_values.knowledge_space_id
if update_values.knowledge_space_revision is not None:
values["knowledge_space_revision"] = update_values.knowledge_space_revision
if update_values.attempted_at is not None:
values["attempt_count"] = KnowledgeFSControlSpace.attempt_count + 1
values["last_attempt_at"] = update_values.attempted_at
statement = (
update(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == update_values.tenant_id,
KnowledgeFSControlSpace.id == update_values.control_space_id,
KnowledgeFSControlSpace.resource_version == update_values.expected_resource_version,
KnowledgeFSControlSpace.state == update_values.expected_state,
)
.values(**values)
.execution_options(synchronize_session=False)
)
if update_values.knowledge_space_id is not None:
statement = statement.where(
(KnowledgeFSControlSpace.knowledge_space_id.is_(None))
| (KnowledgeFSControlSpace.knowledge_space_id == update_values.knowledge_space_id)
)
if update_values.knowledge_space_revision is not None:
statement = statement.where(
KnowledgeFSControlSpace.knowledge_space_revision <= update_values.knowledge_space_revision
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def mark_deletion_irreversible(
self,
*,
tenant_id: str,
control_space_id: str,
lifecycle_operation_id: str,
irreversible_at: datetime,
) -> bool:
statement = (
update(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
KnowledgeFSControlSpace.lifecycle_operation_id == lifecycle_operation_id,
KnowledgeFSControlSpace.state.in_(
(
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
KnowledgeFSControlSpaceState.DELETED,
)
),
(KnowledgeFSControlSpace.deletion_irreversible_at.is_(None))
| (KnowledgeFSControlSpace.deletion_irreversible_at == irreversible_at),
)
.values(deletion_irreversible_at=irreversible_at)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]:
statement = (
select(KnowledgeFSControlSpace.id)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
)
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
)
return tuple(self._session.scalars(statement).all())
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceRepository"]
@@ -0,0 +1,36 @@
"""Database aggregation for low-cardinality KnowledgeFS current-state metrics."""
from __future__ import annotations
from sqlalchemy import func, select
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
class SQLAlchemyKnowledgeFSControlSpaceStateCountReader:
"""Read global operational counts without exposing tenant or resource labels."""
_states = (
KnowledgeFSControlSpaceState.PROVISIONING,
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
)
def __init__(self, session_maker: sessionmaker[Session]) -> None:
self._session_maker = session_maker
def __call__(self) -> dict[str, int]:
statement = (
select(KnowledgeFSControlSpace.state, func.count())
.where(KnowledgeFSControlSpace.state.in_(self._states))
.group_by(KnowledgeFSControlSpace.state)
)
with self._session_maker() as session:
rows = session.execute(statement).all()
counts = {state.value: 0 for state in self._states}
counts.update({state.value: int(count) for state, count in rows})
return counts
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceStateCountReader"]
@@ -0,0 +1,442 @@
"""SQLAlchemy repository for the KnowledgeFS workspace cutover ledger."""
from __future__ import annotations
from datetime import datetime
from typing import cast, override
from sqlalchemy import func, select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs_cutover import (
KnowledgeFSMigrationIssue,
KnowledgeFSMigrationIssueStatus,
KnowledgeFSMigrationQuarantine,
KnowledgeFSMigrationQuarantineDisposition,
KnowledgeFSMigrationQuarantineKind,
KnowledgeFSShadowAuthorizationDiff,
KnowledgeFSShadowAuthorizationObservation,
KnowledgeFSWorkspaceCutoverLedger,
)
from repositories.knowledge_fs_cutover_repository import (
KnowledgeFSCutoverCASUpdate,
KnowledgeFSCutoverRepository,
KnowledgeFSQuarantineCASUpdate,
KnowledgeFSShadowDiffCASUpdate,
)
class SQLAlchemyKnowledgeFSCutoverRepository(KnowledgeFSCutoverRepository):
"""Persist one tenant ledger and evidence using tenant-scoped queries and CAS writes."""
_session: Session
def __init__(self, session: Session):
self._session = session
@override
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger:
self._session.add(ledger)
self._session.flush()
return ledger
@override
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None:
statement = (
select(KnowledgeFSWorkspaceCutoverLedger)
.where(KnowledgeFSWorkspaceCutoverLedger.tenant_id == tenant_id)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]:
statement = select(KnowledgeFSWorkspaceCutoverLedger).order_by(KnowledgeFSWorkspaceCutoverLedger.tenant_id)
return tuple(self._session.scalars(statement).all())
@override
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool:
values: dict[str, object] = {
"phase": update_values.new_phase,
"cas_version": KnowledgeFSWorkspaceCutoverLedger.cas_version + 1,
}
for field_name in (
"source_revision_watermark",
"final_revision_watermark",
"applied_revision_watermark",
"source_task_watermark",
"final_task_watermark",
"applied_task_watermark",
"shadow_started_at",
"shadow_completed_at",
"shadow_evidence_digest",
"shadow_observation_count",
"shadow_window_started_at",
"shadow_window_ended_at",
"shadow_traffic_zero",
"shadow_traffic_zero_evidence",
"shadow_latest_observed_revision",
"shadow_producer",
"shadow_completed_by_operator",
"shadow_completed_by_account_id",
"remote_freeze_id",
"remote_freeze_revision",
"remote_freeze_digest",
"remote_freeze_task_watermark",
"remote_freeze_control_space_id",
"remote_freeze_frozen_at",
"remote_freeze_updated_at",
"remote_freeze_acknowledged_at",
"remote_freeze_applied",
"remote_freeze_replayed",
"remote_activation_id",
"remote_activation_revision",
"remote_activation_digest",
"remote_activation_control_space_id",
"remote_activation_activated_at",
"remote_activation_updated_at",
"remote_activation_acknowledged_at",
"remote_activation_applied",
"remote_activation_replayed",
"freeze_at",
"cutover_at",
"rolled_back_at",
"rollback_cutoff_at",
"observation_started_at",
"observation_window_ends_at",
"observation_completed_at",
"maximum_task_expires_at",
"irreversible_cleanup_at",
"smoke_results",
"legacy_dependency_report",
"legacy_dependency_checked_at",
"legacy_dependency_ready",
"product_routes_enabled",
"capability_v2_enabled",
"integrated_mode_enabled",
"legacy_acl_read_only",
):
value = getattr(update_values, field_name)
if value is not None:
values[field_name] = value
if update_values.clear_smoke_results:
values["smoke_results"] = None
statement = (
update(KnowledgeFSWorkspaceCutoverLedger)
.where(
KnowledgeFSWorkspaceCutoverLedger.tenant_id == update_values.tenant_id,
KnowledgeFSWorkspaceCutoverLedger.phase == update_values.expected_phase,
KnowledgeFSWorkspaceCutoverLedger.cas_version == update_values.expected_cas_version,
)
.values(**values)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue:
self._session.add(issue)
self._session.flush()
return issue
@override
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None:
statement = select(KnowledgeFSMigrationIssue).where(
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
KnowledgeFSMigrationIssue.issue_key == issue_key,
)
return self._session.scalar(statement)
@override
def set_issue_status(
self,
*,
tenant_id: str,
ledger_id: str,
issue_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool:
values: dict[str, object] = {"status": new_status}
if new_status is KnowledgeFSMigrationIssueStatus.APPROVED_FAIL_CLOSED:
values.update(approved_at=changed_at, approved_by_account_id=account_id)
elif new_status is KnowledgeFSMigrationIssueStatus.RESOLVED:
values.update(resolved_at=changed_at, resolved_by_account_id=account_id)
statement = (
update(KnowledgeFSMigrationIssue)
.where(
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
KnowledgeFSMigrationIssue.issue_key == issue_key,
KnowledgeFSMigrationIssue.status == expected_status,
)
.values(**values)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool:
statement = (
update(KnowledgeFSShadowAuthorizationDiff)
.where(
KnowledgeFSShadowAuthorizationDiff.tenant_id == update_values.tenant_id,
KnowledgeFSShadowAuthorizationDiff.ledger_id == update_values.ledger_id,
KnowledgeFSShadowAuthorizationDiff.diff_key == update_values.diff_key,
KnowledgeFSShadowAuthorizationDiff.row_version == update_values.expected_row_version,
)
.values(
control_space_id=update_values.control_space_id,
principal=update_values.principal,
legacy_allowed=update_values.legacy_allowed,
dify_allowed=update_values.dify_allowed,
decision=update_values.decision,
reason=update_values.reason,
observed_revision=update_values.observed_revision,
status=update_values.status,
current_evidence_digest=update_values.current_evidence_digest,
last_observed_at=update_values.last_observed_at,
approved_by_account_id=None,
approved_at=None,
resolved_by_account_id=None,
resolved_at=update_values.last_observed_at,
row_version=KnowledgeFSShadowAuthorizationDiff.row_version + 1,
)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def add_shadow_observation(
self, observation: KnowledgeFSShadowAuthorizationObservation
) -> KnowledgeFSShadowAuthorizationObservation:
self._session.add(observation)
self._session.flush()
return observation
@override
def get_shadow_observation(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
evidence_digest: str,
) -> KnowledgeFSShadowAuthorizationObservation | None:
statement = select(KnowledgeFSShadowAuthorizationObservation).where(
KnowledgeFSShadowAuthorizationObservation.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationObservation.ledger_id == ledger_id,
KnowledgeFSShadowAuthorizationObservation.diff_key == diff_key,
KnowledgeFSShadowAuthorizationObservation.evidence_digest == evidence_digest,
)
return self._session.scalar(statement)
@override
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine:
self._session.add(item)
self._session.flush()
return item
@override
def get_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kind: KnowledgeFSMigrationQuarantineKind,
source_id: str,
) -> KnowledgeFSMigrationQuarantine | None:
statement = select(KnowledgeFSMigrationQuarantine).where(
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
KnowledgeFSMigrationQuarantine.source_kind == source_kind,
KnowledgeFSMigrationQuarantine.source_id == source_id,
)
return self._session.scalar(statement)
@override
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool:
statement = (
update(KnowledgeFSMigrationQuarantine)
.where(
KnowledgeFSMigrationQuarantine.tenant_id == update_values.tenant_id,
KnowledgeFSMigrationQuarantine.ledger_id == update_values.ledger_id,
KnowledgeFSMigrationQuarantine.source_kind == update_values.source_kind,
KnowledgeFSMigrationQuarantine.source_id == update_values.source_id,
KnowledgeFSMigrationQuarantine.disposition == update_values.expected_disposition,
KnowledgeFSMigrationQuarantine.row_version == update_values.expected_row_version,
)
.values(
disposition=KnowledgeFSMigrationQuarantineDisposition.RESOLVED,
resolved_by_operator=update_values.resolved_by_operator,
resolved_by_account_id=update_values.resolved_by_account_id,
evidence=update_values.evidence,
resolved_at=update_values.resolved_at,
row_version=KnowledgeFSMigrationQuarantine.row_version + 1,
)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff:
self._session.add(diff)
self._session.flush()
return diff
@override
def get_shadow_diff(
self, *, tenant_id: str, ledger_id: str, diff_key: str
) -> KnowledgeFSShadowAuthorizationDiff | None:
statement = select(KnowledgeFSShadowAuthorizationDiff).where(
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
KnowledgeFSShadowAuthorizationDiff.diff_key == diff_key,
)
return self._session.scalar(statement)
@override
def set_shadow_diff_status(
self,
*,
tenant_id: str,
ledger_id: str,
diff_key: str,
expected_status: KnowledgeFSMigrationIssueStatus,
new_status: KnowledgeFSMigrationIssueStatus,
account_id: str,
changed_at: datetime,
) -> bool:
values: dict[str, object] = {
"status": new_status,
"row_version": KnowledgeFSShadowAuthorizationDiff.row_version + 1,
}
if new_status is KnowledgeFSMigrationIssueStatus.APPROVED_FAIL_CLOSED:
values.update(approved_at=changed_at, approved_by_account_id=account_id)
elif new_status is KnowledgeFSMigrationIssueStatus.RESOLVED:
values.update(resolved_at=changed_at, resolved_by_account_id=account_id)
statement = (
update(KnowledgeFSShadowAuthorizationDiff)
.where(
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
KnowledgeFSShadowAuthorizationDiff.diff_key == diff_key,
KnowledgeFSShadowAuthorizationDiff.status == expected_status,
)
.values(**values)
.execution_options(synchronize_session=False)
)
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
@override
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int:
statement = (
select(func.count())
.select_from(KnowledgeFSMigrationIssue)
.where(
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
KnowledgeFSMigrationIssue.status == KnowledgeFSMigrationIssueStatus.OPEN,
)
)
return self._session.scalar(statement) or 0
@override
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int:
statement = (
select(func.count())
.select_from(KnowledgeFSShadowAuthorizationDiff)
.where(
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
KnowledgeFSShadowAuthorizationDiff.status == KnowledgeFSMigrationIssueStatus.OPEN,
)
)
return self._session.scalar(statement) or 0
@override
def count_unresolved_quarantine(
self,
*,
tenant_id: str,
ledger_id: str,
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
) -> int:
if not source_kinds:
return 0
statement = (
select(func.count())
.select_from(KnowledgeFSMigrationQuarantine)
.where(
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
KnowledgeFSMigrationQuarantine.source_kind.in_(source_kinds),
KnowledgeFSMigrationQuarantine.disposition != KnowledgeFSMigrationQuarantineDisposition.RESOLVED,
)
)
return self._session.scalar(statement) or 0
@override
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]:
statement = (
select(KnowledgeFSMigrationIssue)
.where(
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
)
.order_by(KnowledgeFSMigrationIssue.created_at, KnowledgeFSMigrationIssue.id)
)
return tuple(self._session.scalars(statement).all())
@override
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]:
statement = (
select(KnowledgeFSMigrationQuarantine)
.where(
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
)
.order_by(KnowledgeFSMigrationQuarantine.created_at, KnowledgeFSMigrationQuarantine.id)
)
return tuple(self._session.scalars(statement).all())
@override
def list_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]:
statement = (
select(KnowledgeFSShadowAuthorizationDiff)
.where(
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
)
.order_by(KnowledgeFSShadowAuthorizationDiff.created_at, KnowledgeFSShadowAuthorizationDiff.id)
)
return tuple(self._session.scalars(statement).all())
@override
def list_shadow_observations(
self, *, tenant_id: str, ledger_id: str
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]:
statement = (
select(KnowledgeFSShadowAuthorizationObservation)
.where(
KnowledgeFSShadowAuthorizationObservation.tenant_id == tenant_id,
KnowledgeFSShadowAuthorizationObservation.ledger_id == ledger_id,
)
.order_by(
KnowledgeFSShadowAuthorizationObservation.observed_at,
KnowledgeFSShadowAuthorizationObservation.evidence_digest,
)
)
return tuple(self._session.scalars(statement).all())
__all__ = ["SQLAlchemyKnowledgeFSCutoverRepository"]
@@ -0,0 +1,394 @@
"""SQLAlchemy CAS implementation for the KnowledgeFS lifecycle outbox."""
from __future__ import annotations
import logging
from datetime import datetime, timedelta
from typing import cast, override
import sqlalchemy as sa
from sqlalchemy import select, update
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session
from models.knowledge_fs import (
KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
)
from repositories.knowledge_fs_lifecycle_outbox_repository import KnowledgeFSLifecycleOutboxRepository
from services.knowledge_fs.observability import (
KnowledgeFSLifecycleTaskMetric,
KnowledgeFSOperationalMetricsPort,
get_knowledge_fs_operational_metrics,
)
logger = logging.getLogger(__name__)
class SQLAlchemyKnowledgeFSLifecycleOutboxRepository(KnowledgeFSLifecycleOutboxRepository):
"""Claim and settle commands inside a caller-owned database transaction."""
def __init__(self, session: Session, metrics: KnowledgeFSOperationalMetricsPort | None = None):
self._session = session
self._metrics = metrics or get_knowledge_fs_operational_metrics()
@override
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox:
self._session.add(command)
self._session.flush()
try:
self._metrics.record_lifecycle_task(KnowledgeFSLifecycleTaskMetric(None, command.operation.value, "queued"))
except Exception:
logger.warning(
"KnowledgeFS lifecycle metric export failed operation=%s status=queued",
command.operation.value,
exc_info=True,
)
return command
@override
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None:
statement = (
select(KnowledgeFSLifecycleOutbox)
.where(KnowledgeFSLifecycleOutbox.id == outbox_id)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None:
statement = (
select(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
KnowledgeFSLifecycleOutbox.operation_id == operation_id,
)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_open_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None:
statement = (
select(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
KnowledgeFSLifecycleOutbox.control_space_id == control_space_id,
KnowledgeFSLifecycleOutbox.operation == operation,
KnowledgeFSLifecycleOutbox.status.in_(
(
KnowledgeFSLifecycleOutboxStatus.PENDING,
KnowledgeFSLifecycleOutboxStatus.PROCESSING,
KnowledgeFSLifecycleOutboxStatus.RETRY,
)
),
)
.order_by(KnowledgeFSLifecycleOutbox.created_at.desc(), KnowledgeFSLifecycleOutbox.id.desc())
.limit(1)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def find_latest_for_control_space(
self,
*,
tenant_id: str,
control_space_id: str,
operation: KnowledgeFSLifecycleOperation,
) -> KnowledgeFSLifecycleOutbox | None:
statement = (
select(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
KnowledgeFSLifecycleOutbox.control_space_id == control_space_id,
KnowledgeFSLifecycleOutbox.operation == operation,
)
.order_by(KnowledgeFSLifecycleOutbox.created_at.desc(), KnowledgeFSLifecycleOutbox.id.desc())
.limit(1)
.execution_options(populate_existing=True)
)
return self._session.scalar(statement)
@override
def reactivate_dead_letter(self, *, outbox_id: str) -> bool:
statement = (
update(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.id == outbox_id,
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.RETRY,
next_attempt_at=None,
completed_at=None,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
@override
def supersede_unattempted(
self,
*,
outbox_id: str,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool:
statement = (
update(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.id == outbox_id,
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PENDING,
KnowledgeFSLifecycleOutbox.attempt_count == 0,
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
completed_at=completed_at,
last_error_code=error_code,
last_error_message=error_message,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
@override
def supersede_after_remote_absence(
self,
*,
outbox_id: str,
observed_at: datetime,
error_code: str,
error_message: str,
) -> bool:
safely_unleased = sa.and_(
KnowledgeFSLifecycleOutbox.status.in_(
(
KnowledgeFSLifecycleOutboxStatus.PENDING,
KnowledgeFSLifecycleOutboxStatus.RETRY,
KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
)
),
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
)
expired_lease = sa.and_(
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
KnowledgeFSLifecycleOutbox.lease_expires_at <= observed_at,
)
statement = (
update(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.id == outbox_id,
sa.or_(safely_unleased, expired_lease),
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
lease_owner=None,
lease_expires_at=None,
completed_at=sa.func.coalesce(KnowledgeFSLifecycleOutbox.completed_at, observed_at),
last_error_code=error_code,
last_error_message=error_message,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
@override
def claim_next(
self,
*,
lease_owner: str,
now: datetime,
lease_duration: timedelta,
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
) -> KnowledgeFSLifecycleOutbox | None:
if not lease_owner.strip():
raise ValueError("lease_owner must not be blank")
if lease_duration <= timedelta(0):
raise ValueError("lease_duration must be positive")
if not allowed_operations:
return None
ready_to_attempt = sa.and_(
KnowledgeFSLifecycleOutbox.status.in_(
(KnowledgeFSLifecycleOutboxStatus.PENDING, KnowledgeFSLifecycleOutboxStatus.RETRY)
),
sa.or_(
KnowledgeFSLifecycleOutbox.next_attempt_at.is_(None),
KnowledgeFSLifecycleOutbox.next_attempt_at <= now,
),
)
expired_lease = sa.and_(
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
KnowledgeFSLifecycleOutbox.lease_expires_at <= now,
)
candidate_statement = (
select(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.operation.in_(allowed_operations),
sa.or_(ready_to_attempt, expired_lease),
)
.order_by(
sa.func.coalesce(KnowledgeFSLifecycleOutbox.next_attempt_at, KnowledgeFSLifecycleOutbox.created_at),
KnowledgeFSLifecycleOutbox.id,
)
.limit(1)
.with_for_update(skip_locked=True)
)
candidate = self._session.scalar(candidate_statement)
if candidate is None:
return None
lease_expires_at = now + lease_duration
claim_statement = update(KnowledgeFSLifecycleOutbox).where(
KnowledgeFSLifecycleOutbox.id == candidate.id,
KnowledgeFSLifecycleOutbox.status == candidate.status,
KnowledgeFSLifecycleOutbox.attempt_count == candidate.attempt_count,
)
if candidate.status is KnowledgeFSLifecycleOutboxStatus.PROCESSING:
claim_statement = claim_statement.where(
KnowledgeFSLifecycleOutbox.lease_owner == candidate.lease_owner,
KnowledgeFSLifecycleOutbox.lease_expires_at == candidate.lease_expires_at,
KnowledgeFSLifecycleOutbox.lease_expires_at <= now,
)
else:
claim_statement = claim_statement.where(
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
)
result = self._session.execute(
claim_statement.values(
status=KnowledgeFSLifecycleOutboxStatus.PROCESSING,
lease_owner=lease_owner,
lease_expires_at=lease_expires_at,
last_attempt_at=now,
attempt_count=candidate.attempt_count + 1,
completed_at=None,
).execution_options(synchronize_session=False)
)
if (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) != 1:
return None
return self.get(outbox_id=candidate.id)
@override
def acknowledge(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
) -> bool:
statement = (
update(KnowledgeFSLifecycleOutbox)
.where(
KnowledgeFSLifecycleOutbox.id == outbox_id,
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
KnowledgeFSLifecycleOutbox.lease_owner == lease_owner,
KnowledgeFSLifecycleOutbox.lease_expires_at == expected_lease_expires_at,
KnowledgeFSLifecycleOutbox.lease_expires_at > completed_at,
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.SUCCEEDED,
lease_owner=None,
lease_expires_at=None,
completed_at=completed_at,
last_error_code=None,
last_error_message=None,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
@override
def schedule_retry(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
next_attempt_at: datetime,
error_code: str,
error_message: str,
) -> bool:
statement = (
self._leased_update(
outbox_id=outbox_id,
lease_owner=lease_owner,
expected_lease_expires_at=expected_lease_expires_at,
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.RETRY,
lease_owner=None,
lease_expires_at=None,
next_attempt_at=next_attempt_at,
completed_at=None,
last_error_code=error_code,
last_error_message=error_message,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
@override
def mark_dead_letter(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
completed_at: datetime,
error_code: str,
error_message: str,
) -> bool:
statement = (
self._leased_update(
outbox_id=outbox_id,
lease_owner=lease_owner,
expected_lease_expires_at=expected_lease_expires_at,
)
.values(
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
lease_owner=None,
lease_expires_at=None,
completed_at=completed_at,
last_error_code=error_code,
last_error_message=error_message,
)
.execution_options(synchronize_session=False)
)
return self._updated_once(statement)
def _leased_update(
self,
*,
outbox_id: str,
lease_owner: str,
expected_lease_expires_at: datetime,
) -> sa.Update:
return update(KnowledgeFSLifecycleOutbox).where(
KnowledgeFSLifecycleOutbox.id == outbox_id,
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
KnowledgeFSLifecycleOutbox.lease_owner == lease_owner,
KnowledgeFSLifecycleOutbox.lease_expires_at == expected_lease_expires_at,
)
def _updated_once(self, statement: sa.Update) -> bool:
result = self._session.execute(statement)
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
__all__ = ["SQLAlchemyKnowledgeFSLifecycleOutboxRepository"]
+48
View File
@@ -1380,12 +1380,31 @@ class TenantService:
.where(TenantAccountJoin.tenant_id == tenant.id, TenantAccountJoin.account_id == account.id)
.limit(1)
)
previous_role = TenantAccountRole(ta.role) if ta else None
if ta:
ta.role = TenantAccountRole(role)
else:
ta = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole(role))
session.add(ta)
requested_role = TenantAccountRole(role)
if previous_role is None or previous_role is not requested_role:
from services.knowledge_fs.membership_changes import (
KnowledgeFSWorkspaceMembershipChange,
apply_workspace_membership_change,
)
apply_workspace_membership_change(
session=session,
tenant_id=str(tenant.id),
actor_account_id=str(account.id),
account_ids=(str(account.id),),
change=(
KnowledgeFSWorkspaceMembershipChange.MEMBER_ADDED
if previous_role is None
else KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED
),
)
session.commit()
if dify_config.BILLING_ENABLED:
BillingService.clean_billing_info_cache(tenant.id)
@@ -1796,6 +1815,20 @@ class TenantService:
)
.values(maintainer=owner_id)
)
from services.knowledge_fs.membership_changes import (
KnowledgeFSWorkspaceMembershipChange,
apply_workspace_membership_change,
)
apply_workspace_membership_change(
session=session,
tenant_id=str(tenant.id),
actor_account_id=str(operator.id),
account_ids=(str(account_id),),
change=KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED,
removed_account_id=str(account_id),
replacement_owner_account_id=str(owner_id),
)
session.delete(ta)
# Clean up orphaned pending accounts (invited but never activated)
@@ -1863,6 +1896,7 @@ class TenantService:
if target_member_join.role == new_role:
raise RoleAlreadyAssignedError("The provided role is already assigned to the member.")
affected_account_ids = [str(member.id)]
if new_role == "owner":
# Find the current owner and change their role to 'admin'
current_owner_join = session.scalar(
@@ -1886,6 +1920,8 @@ class TenantService:
role_ids=[admin_role_id],
session=session,
)
if current_owner_join and str(current_owner_join.account_id) != str(member.id):
affected_account_ids.append(str(current_owner_join.account_id))
# Update the role of the target member
if dify_config.RBAC_ENABLED:
@@ -1903,6 +1939,18 @@ class TenantService:
)
else:
target_member_join.role = new_tenant_role
from services.knowledge_fs.membership_changes import (
KnowledgeFSWorkspaceMembershipChange,
apply_workspace_membership_change,
)
apply_workspace_membership_change(
session=session,
tenant_id=str(tenant.id),
actor_account_id=str(operator.id),
account_ids=tuple(affected_account_ids),
change=KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED,
)
session.commit()
@staticmethod
+15 -3
View File
@@ -19,9 +19,9 @@ from __future__ import annotations
from sqlalchemy.orm import Session
from core.agent.entities import AgentToolEntity
from core.app.entities.app_invoke_entities import InvokeFrom
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
from core.tools.entities.tool_entities import ToolProviderType
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolProviderType
from core.tools.errors import (
ToolInvokeError,
ToolNotFoundError,
@@ -64,16 +64,28 @@ class AgentToolInnerService:
credential_id=request.tool.credential_id,
)
try:
dify_run_context = DifyRunContext(
tenant_id=request.caller.tenant_id,
app_id=request.caller.app_id,
user_id=request.caller.user_id,
user_from=UserFrom(request.caller.user_from),
invoke_from=InvokeFrom.value_of(request.caller.invoke_from),
trace_session_id=request.caller.trace_session_id,
)
tool_runtime = ToolManager.get_agent_tool_runtime(
tenant_id=request.caller.tenant_id,
app_id=request.caller.app_id,
agent_tool=agent_tool,
user_id=request.caller.user_id,
invoke_from=InvokeFrom.value_of(request.caller.invoke_from),
invoke_from=dify_run_context.invoke_from,
variable_pool=None,
allow_file_parameters=True,
use_default_for_missing_form_parameters=True,
)
tool_runtime.runtime.dify_run_context = dify_run_context
tool_runtime.runtime.tool_invoke_from = (
ToolInvokeFrom.WORKFLOW if request.caller.workflow_id is not None else ToolInvokeFrom.AGENT
)
messages = ToolEngine.generic_invoke(
session=session,
tool=tool_runtime,
+6 -1
View File
@@ -208,7 +208,12 @@ class DatasourceProviderService:
if credential_id:
datasource_provider = session.scalar(
select(DatasourceProvider)
.where(DatasourceProvider.tenant_id == tenant_id, DatasourceProvider.id == credential_id)
.where(
DatasourceProvider.tenant_id == tenant_id,
DatasourceProvider.id == credential_id,
DatasourceProvider.provider == provider,
DatasourceProvider.plugin_id == plugin_id,
)
.limit(1)
)
else:
+141 -4
View File
@@ -46,6 +46,7 @@ class RBACResourceType(StrEnum):
APP = "app"
DATASET = "dataset"
KNOWLEDGE_FS = "knowledge_space"
class RBACRoleType(StrEnum):
@@ -287,9 +288,9 @@ class ResourcePermissionSnapshot(_RBACModel):
overrides: list[ResourcePermissionKeys] = Field(default_factory=list)
def permission_keys_by_resource_ids(self, resource_ids: list[str]) -> dict[str, list[str]]:
result = {str(resource_id): list(self.default_permission_keys) for resource_id in resource_ids}
result = {resource_id: list(self.default_permission_keys) for resource_id in resource_ids}
for override in self.overrides:
resource_id = str(override.resource_id)
resource_id = override.resource_id
if resource_id in result:
result[resource_id] = list(override.permission_keys)
return result
@@ -608,7 +609,43 @@ def _legacy_resource_permission_keys_batch(
permission_keys = snapshot.app.default_permission_keys
else:
permission_keys = snapshot.dataset.default_permission_keys
return {str(resource_id): list(permission_keys) for resource_id in resource_ids}
return {resource_id: list(permission_keys) for resource_id in resource_ids}
def _commit_knowledge_fs_rbac_invalidation(
*,
session: Session,
tenant_id: str,
actor_account_id: str,
member_account_id: str | None = None,
) -> None:
"""Commit one local invalidation after its remote RBAC mutation succeeded."""
from services.knowledge_fs.membership_changes import (
KnowledgeFSWorkspaceMembershipChange,
apply_workspace_membership_change,
apply_workspace_rbac_role_change,
)
try:
if member_account_id is None:
apply_workspace_rbac_role_change(session=session, tenant_id=tenant_id)
else:
apply_workspace_membership_change(
session=session,
tenant_id=tenant_id,
actor_account_id=actor_account_id,
account_ids=(member_account_id,),
change=KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED,
)
session.commit()
except Exception:
session.rollback()
logger.exception(
"KnowledgeFS RBAC invalidation failed after remote mutation tenant_id=%s actor_account_id=%s",
tenant_id,
actor_account_id,
)
raise
# ---------- Mutation request models ----------
@@ -752,7 +789,7 @@ def _inner_call(
def _resource_id_params(resource_type: RBACResourceType | str, resource_id: str) -> dict[str, str]:
resource_type_value = resource_type.value if isinstance(resource_type, RBACResourceType) else str(resource_type)
resource_type_value = resource_type.value if isinstance(resource_type, RBACResourceType) else resource_type
resource_id = resource_id.strip()
if resource_type_value == RBACResourceType.APP.value:
return {"resource_type": resource_type_value, "app_id": resource_id}
@@ -1703,6 +1740,71 @@ class RBACService:
)
return data
class KnowledgeFSRoleMutations:
"""Couple successful RBAC writes to durable KnowledgeFS invalidation.
Enterprise RBAC is remote, so the remote mutation must succeed before
this service commits the local epoch and revoke outbox transaction. A
local failure is rolled back and propagated so callers cannot mistake
a partially invalidated authorization change for success.
"""
@staticmethod
def replace_member_roles(
tenant_id: str,
actor_account_id: str,
member_account_id: str,
role_ids: list[str],
*,
session: Session,
) -> MemberRolesResponse:
result = RBACService.MemberRoles.replace(
tenant_id,
actor_account_id,
member_account_id,
role_ids,
session=session,
)
_commit_knowledge_fs_rbac_invalidation(
session=session,
tenant_id=tenant_id,
actor_account_id=actor_account_id,
member_account_id=member_account_id,
)
return result
@staticmethod
def update_role(
tenant_id: str,
actor_account_id: str,
role_id: str,
payload: RoleMutation,
*,
session: Session,
) -> RBACRole:
result = RBACService.Roles.update(tenant_id, actor_account_id, role_id, payload)
_commit_knowledge_fs_rbac_invalidation(
session=session,
tenant_id=tenant_id,
actor_account_id=actor_account_id,
)
return result
@staticmethod
def delete_role(
tenant_id: str,
actor_account_id: str,
role_id: str,
*,
session: Session,
) -> None:
RBACService.Roles.delete(tenant_id, actor_account_id, role_id)
_commit_knowledge_fs_rbac_invalidation(
session=session,
tenant_id=tenant_id,
actor_account_id=actor_account_id,
)
class CheckAccess:
"""Call the ``/inner/api/rbac/check-access`` endpoint."""
@@ -1786,6 +1888,41 @@ class RBACService:
)
return _parse_resource_permission_keys_batch(data, resource_id_key="dataset_id")
class KnowledgeFSPermissions:
"""Batch permission lookup for independent KnowledgeFS control-space IDs."""
@staticmethod
def batch_get(
tenant_id: str,
account_id: str | None,
control_space_ids: list[str],
*,
session: Session,
) -> dict[str, list[str]]:
if not control_space_ids:
return {}
if not dify_config.RBAC_ENABLED:
_ = session
permissions = [
"knowledge_space_read",
"knowledge_space_create",
"knowledge_space_edit",
"knowledge_space_delete",
"knowledge_space_access_config",
"knowledge_space_api_key_manage",
"knowledge_space_document_write",
"knowledge_space_query",
]
return dict.fromkeys(control_space_ids, permissions)
data = _inner_call(
"POST",
f"{_INNER_PREFIX}/knowledge-fs/permission-keys/batch",
tenant_id=tenant_id,
account_id=account_id,
json={"control_space_ids": control_space_ids},
)
return _parse_resource_permission_keys_batch(data, resource_id_key="control_space_id")
class MyPermissions:
@staticmethod
def get(
@@ -22,6 +22,7 @@ class AgentToolInvokeCaller(BaseModel):
node_execution_id: str | None = None
agent_id: str | None = None
agent_config_version_id: str | None = None
trace_session_id: str | None = Field(default=None, exclude_if=lambda value: value is None)
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+1
View File
@@ -0,0 +1 @@
"""Dify-owned KnowledgeFS control-plane services."""
@@ -0,0 +1,186 @@
"""Fail-closed Agent and Workflow admission for KnowledgeFS control-spaces."""
from __future__ import annotations
from typing import NamedTuple
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import naive_utc_now
from models.knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSAppSpaceJoinStatus,
KnowledgeFSAppSpaceJoinType,
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpaceState,
KnowledgeFSExternalAccessPolicy,
)
from services.knowledge_fs.product_operations import product_operation_action
from services.knowledge_fs.revocation_commands import (
KnowledgeFSRevocationCommandPort,
KnowledgeFSRevocationCommandProducer,
)
class KnowledgeFSAppAdmissionError(RuntimeError):
"""The app is not explicitly bound to an enabled KnowledgeFS channel."""
class KnowledgeFSAppPrincipalProfile(NamedTuple):
tenant_id: str
control_space_id: str
app_id: str
join_id: str
caller_kind: KnowledgeFSAppSpaceJoinType
action: str
knowledge_space_id: str
knowledge_space_revision: int
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
content_policy_revision: int
class KnowledgeFSAppAdmissionService:
def __init__(
self,
session_maker: sessionmaker[Session],
*,
revocations: KnowledgeFSRevocationCommandPort | None = None,
) -> None:
self._session_maker = session_maker
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
def revoke_binding(
self,
*,
tenant_id: str,
app_id: str,
control_space_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
revoked_by_account_id: str,
) -> None:
with self._session_maker.begin() as session:
join = session.scalar(
sa.select(AppKnowledgeFSSpaceJoin)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
AppKnowledgeFSSpaceJoin.app_id == app_id,
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
)
.with_for_update()
)
if join is None:
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding was not found")
if join.status is not KnowledgeFSAppSpaceJoinStatus.ACTIVE:
return
join.status = KnowledgeFSAppSpaceJoinStatus.REVOKED
join.revision += 1
join.revoked_at = naive_utc_now()
join.revoked_by_account_id = revoked_by_account_id
revision = session.scalar(
sa.select(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
)
.with_for_update()
)
if revision is None:
raise KnowledgeFSAppAdmissionError("KnowledgeFS authorization revision is missing")
revision.external_access_epoch += 1
self._revocations.enqueue_principal_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
subject=f"dify-app:{app_id}",
reason_code="app_binding_revoked",
caller_kinds=(caller_kind.value,),
)
def admit(
self,
*,
tenant_id: str,
app_id: str,
control_space_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
operation_id: str,
) -> KnowledgeFSAppPrincipalProfile:
try:
action = product_operation_action(operation_id)
except KeyError as exc:
raise KnowledgeFSAppAdmissionError("KnowledgeFS operation is not registered") from exc
with self._session_maker() as session:
row = session.execute(
sa.select(
AppKnowledgeFSSpaceJoin,
KnowledgeFSControlSpace,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSAuthorizationRevision,
)
.join(
KnowledgeFSControlSpace,
sa.and_(
KnowledgeFSControlSpace.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
KnowledgeFSControlSpace.id == AppKnowledgeFSSpaceJoin.control_space_id,
),
)
.join(
KnowledgeFSExternalAccessPolicy,
sa.and_(
KnowledgeFSExternalAccessPolicy.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
),
)
.join(
KnowledgeFSAuthorizationRevision,
sa.and_(
KnowledgeFSAuthorizationRevision.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
),
)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
AppKnowledgeFSSpaceJoin.app_id == app_id,
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
AppKnowledgeFSSpaceJoin.status == KnowledgeFSAppSpaceJoinStatus.ACTIVE,
)
).one_or_none()
if row is None:
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding is not enabled")
join, control_space, policy, revision = row._t
channel_enabled = (
policy.agent_enabled if caller_kind is KnowledgeFSAppSpaceJoinType.AGENT else policy.workflow_enabled
)
if (
not channel_enabled
or control_space.state is not KnowledgeFSControlSpaceState.ACTIVE
or control_space.knowledge_space_id is None
):
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding is not enabled")
return KnowledgeFSAppPrincipalProfile(
tenant_id=tenant_id,
control_space_id=control_space_id,
app_id=app_id,
join_id=join.id,
caller_kind=caller_kind,
action=action,
knowledge_space_id=control_space.knowledge_space_id,
knowledge_space_revision=control_space.knowledge_space_revision,
membership_epoch=revision.membership_epoch,
space_acl_epoch=revision.space_acl_epoch,
external_access_epoch=revision.external_access_epoch,
content_policy_revision=revision.content_policy_revision,
)
__all__ = [
"KnowledgeFSAppAdmissionError",
"KnowledgeFSAppAdmissionService",
"KnowledgeFSAppPrincipalProfile",
]
@@ -0,0 +1,267 @@
"""Authorized lifecycle management for Agent and Workflow KnowledgeFS bindings."""
from __future__ import annotations
from typing import Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import naive_utc_now
from models.enums import AppStatus
from models.knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSAppSpaceJoinStatus,
KnowledgeFSAppSpaceJoinType,
KnowledgeFSAuthorizationRevision,
)
from models.model import App, AppMode
from services.knowledge_fs.product_dto import (
KnowledgeFSAppBindingListResponse,
KnowledgeFSAppBindingPayload,
KnowledgeFSAppBindingResponse,
)
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
from services.knowledge_fs.revocation_commands import KnowledgeFSRevocationCommandPort
class KnowledgeFSAppBindingManagementError(RuntimeError):
"""An app binding cannot be created or changed safely."""
class KnowledgeFSAppBindingAuthorizationPort(Protocol):
def authorize_control_space(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
permission: KnowledgeFSProductPermission,
) -> object: ...
class KnowledgeFSAppCatalogPort(Protocol):
def supports_binding(
self,
*,
session: Session,
tenant_id: str,
app_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
) -> bool: ...
class SQLKnowledgeFSAppCatalog:
_MODES_BY_CALLER = {
KnowledgeFSAppSpaceJoinType.AGENT: (AppMode.AGENT, AppMode.AGENT_CHAT),
KnowledgeFSAppSpaceJoinType.WORKFLOW: (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW),
}
def supports_binding(
self,
*,
session: Session,
tenant_id: str,
app_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
) -> bool:
return (
session.scalar(
sa.select(App.id)
.where(
App.id == app_id,
App.tenant_id == tenant_id,
App.status == AppStatus.NORMAL,
App.mode.in_(self._MODES_BY_CALLER[caller_kind]),
)
.limit(1)
)
is not None
)
class KnowledgeFSAppBindingManagementService:
def __init__(
self,
session_maker: sessionmaker[Session],
*,
product: KnowledgeFSAppBindingAuthorizationPort,
apps: KnowledgeFSAppCatalogPort,
revocations: KnowledgeFSRevocationCommandPort,
) -> None:
self._session_maker = session_maker
self._product = product
self._apps = apps
self._revocations = revocations
def list(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
) -> KnowledgeFSAppBindingListResponse:
self._authorize(
tenant_id=tenant_id,
actor_account_id=actor_account_id,
control_space_id=control_space_id,
)
with self._session_maker() as session:
bindings = tuple(
session.scalars(
sa.select(AppKnowledgeFSSpaceJoin)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
)
.order_by(AppKnowledgeFSSpaceJoin.app_id, AppKnowledgeFSSpaceJoin.join_type)
)
)
return KnowledgeFSAppBindingListResponse(data=[_response(binding) for binding in bindings])
def upsert(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
payload: KnowledgeFSAppBindingPayload,
) -> KnowledgeFSAppBindingResponse:
self._authorize(
tenant_id=tenant_id,
actor_account_id=actor_account_id,
control_space_id=control_space_id,
)
with self._session_maker.begin() as session:
if not self._apps.supports_binding(
session=session,
tenant_id=tenant_id,
app_id=payload.app_id,
caller_kind=payload.caller_kind,
):
raise KnowledgeFSAppBindingManagementError("App is not eligible for this KnowledgeFS caller channel")
authorization_revision = _authorization_revision(
session,
tenant_id=tenant_id,
control_space_id=control_space_id,
)
binding = session.scalar(
sa.select(AppKnowledgeFSSpaceJoin)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
AppKnowledgeFSSpaceJoin.app_id == payload.app_id,
AppKnowledgeFSSpaceJoin.join_type == payload.caller_kind,
)
.with_for_update()
)
if binding is None:
binding = AppKnowledgeFSSpaceJoin(
tenant_id=tenant_id,
control_space_id=control_space_id,
app_id=payload.app_id,
join_type=payload.caller_kind,
created_by_account_id=actor_account_id,
)
session.add(binding)
session.flush()
authorization_revision.external_access_epoch += 1
elif binding.status is KnowledgeFSAppSpaceJoinStatus.REVOKED:
binding.status = KnowledgeFSAppSpaceJoinStatus.ACTIVE
binding.revision += 1
binding.revoked_at = None
binding.revoked_by_account_id = None
authorization_revision.external_access_epoch += 1
return _response(binding)
def revoke(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
app_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
) -> None:
self._authorize(
tenant_id=tenant_id,
actor_account_id=actor_account_id,
control_space_id=control_space_id,
)
with self._session_maker.begin() as session:
authorization_revision = _authorization_revision(
session,
tenant_id=tenant_id,
control_space_id=control_space_id,
)
binding = session.scalar(
sa.select(AppKnowledgeFSSpaceJoin)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
AppKnowledgeFSSpaceJoin.app_id == app_id,
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
)
.with_for_update()
)
if binding is None:
raise KnowledgeFSAppBindingManagementError("KnowledgeFS app binding was not found")
if binding.status is KnowledgeFSAppSpaceJoinStatus.REVOKED:
return
binding.status = KnowledgeFSAppSpaceJoinStatus.REVOKED
binding.revision += 1
binding.revoked_at = naive_utc_now()
binding.revoked_by_account_id = actor_account_id
authorization_revision.external_access_epoch += 1
self._revocations.enqueue_principal_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
subject=f"dify-app:{app_id}",
reason_code="app_binding_revoked",
caller_kinds=(caller_kind.value,),
)
def _authorize(self, *, tenant_id: str, actor_account_id: str, control_space_id: str) -> None:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
def _authorization_revision(
session: Session,
*,
tenant_id: str,
control_space_id: str,
) -> KnowledgeFSAuthorizationRevision:
revision = session.scalar(
sa.select(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
)
.with_for_update()
)
if revision is None:
raise KnowledgeFSAppBindingManagementError("KnowledgeFS authorization revision is missing")
return revision
def _response(binding: AppKnowledgeFSSpaceJoin) -> KnowledgeFSAppBindingResponse:
return KnowledgeFSAppBindingResponse(
id=binding.id,
app_id=binding.app_id,
caller_kind=binding.join_type,
status=binding.status,
revision=binding.revision,
)
__all__ = [
"KnowledgeFSAppBindingManagementError",
"KnowledgeFSAppBindingManagementService",
"SQLKnowledgeFSAppCatalog",
]
@@ -0,0 +1,144 @@
"""Single production entry point for Agent and Workflow KnowledgeFS capabilities."""
from __future__ import annotations
from collections.abc import Generator
from contextlib import contextmanager
from typing import Literal, cast
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
from core.app.entities.app_invoke_entities import DifyRunContext
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
from services.knowledge_fs.app_admission_service import KnowledgeFSAppAdmissionService
from services.knowledge_fs.capability_broker import KnowledgeFSCapabilityBroker, KnowledgeFSIssuedProductCapability
from services.knowledge_fs.operation_admission import KnowledgeFSOperationAdmissionService
from services.knowledge_fs.product_dto import (
KnowledgeFSResearchTaskCreatePayload,
KnowledgeFSResearchTaskResponse,
)
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemotePort,
KnowledgeFSRemoteJSONRequest,
)
class KnowledgeResourceRef(BaseModel):
"""A typed app configuration reference to one Dify-owned KnowledgeFS control-space."""
kind: Literal["knowledge_fs"]
control_space_id: str = Field(min_length=1, max_length=1_000)
model_config = ConfigDict(extra="forbid", frozen=True)
@field_validator("control_space_id")
@classmethod
def normalize_control_space_id(cls, value: str) -> str:
normalized = value.strip()
if not normalized:
raise ValueError("KnowledgeFS control-space reference is required")
return normalized
class KnowledgeFSAppExecutionCapabilityService:
def __init__(
self,
*,
admission: KnowledgeFSAppAdmissionService,
broker: KnowledgeFSCapabilityBroker,
operation_admission: KnowledgeFSOperationAdmissionService,
remote: KnowledgeFSProductRemotePort,
) -> None:
self._admission = admission
self._broker = broker
self._operation_admission = operation_admission
self._remote = remote
def issue(
self,
*,
tenant_id: str,
app_id: str,
control_space_id: str,
caller_kind: KnowledgeFSAppSpaceJoinType,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
profile = self._admission.admit(
tenant_id=tenant_id,
app_id=app_id,
control_space_id=control_space_id,
caller_kind=caller_kind,
operation_id=operation_id,
)
return self._broker.issue_app(
profile=profile,
operation_id=operation_id,
resource_id=resource_id,
trace_id=trace_id,
)
def create_research_task(
self,
*,
run_context: DifyRunContext,
caller_kind: KnowledgeFSAppSpaceJoinType,
resource: KnowledgeResourceRef,
payload: KnowledgeFSResearchTaskCreatePayload,
) -> KnowledgeFSResearchTaskResponse:
"""Create one Research task through app admission and a bounded product operation."""
operation_id = "createResearchTask"
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
if (
not is_product_operation_ready(operation_id)
or operation.transport != "json"
or operation.kfs_path is None
or "{" in operation.kfs_path
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app Research task creation is unavailable")
with self._admitted(tenant_id=run_context.tenant_id, operation_id=operation_id):
issued = self.issue(
tenant_id=run_context.tenant_id,
app_id=run_context.app_id,
control_space_id=resource.control_space_id,
caller_kind=caller_kind,
operation_id=operation_id,
trace_id=run_context.trace_session_id,
)
remote_payload = cast(
dict[str, JsonValue],
payload.model_dump(mode="json", exclude_none=True, by_alias=True),
)
remote_payload["knowledgeSpaceId"] = issued.knowledge_space_id
raw = self._remote.execute_json(
KnowledgeFSRemoteJSONRequest(
operation_id=operation_id,
method=operation.method,
path=operation.kfs_path,
namespace_id=run_context.tenant_id,
knowledge_space_id=issued.knowledge_space_id,
capability_token=issued.token,
trace_id=issued.trace_id,
payload=remote_payload,
)
)
response = KnowledgeFSResearchTaskResponse.model_validate(raw)
return response
@contextmanager
def _admitted(self, *, tenant_id: str, operation_id: str) -> Generator[None, None, None]:
charge = self._operation_admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
try:
yield
except BaseException:
charge.refund()
raise
else:
charge.commit()
__all__ = ["KnowledgeFSAppExecutionCapabilityService", "KnowledgeResourceRef"]
@@ -0,0 +1,160 @@
"""Issue one namespace Capability over an exact, already-authorized Space batch."""
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import NamedTuple, Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import (
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpaceState,
)
from services.knowledge_fs.cutover_runtime_gate import KnowledgeFSWorkspaceRuntimeGatePort
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
from services.knowledge_fs_capability import (
CapabilityAuthzRevision,
CapabilityIssueRequest,
CapabilityResource,
IssuedKnowledgeFSCapability,
)
MAX_BATCH_SPACE_SUMMARIES = 100
class KnowledgeFSBatchSpaceBinding(NamedTuple):
control_space_id: str
knowledge_space_id: str
class KnowledgeFSIssuedBatchCapability(NamedTuple):
token: str
expires_at: datetime
knowledge_space_ids: tuple[str, ...]
trace_id: str
class KnowledgeFSBatchCapabilityIssuerPort(Protocol):
def issue_interactive(
self,
*,
tenant_id: str,
account_id: str,
bindings: tuple[KnowledgeFSBatchSpaceBinding, ...],
trace_id: str,
) -> KnowledgeFSIssuedBatchCapability: ...
class KnowledgeFSCapabilityIssuerPort(Protocol):
def issue(self, request: CapabilityIssueRequest) -> IssuedKnowledgeFSCapability: ...
class KnowledgeFSBatchCapabilityBroker:
"""Revalidates exact tenant/registration bindings before signing the scoped ID set."""
def __init__(
self,
session_maker: sessionmaker[Session],
*,
cutover_gate: KnowledgeFSWorkspaceRuntimeGatePort,
issuer: KnowledgeFSCapabilityIssuerPort | None,
) -> None:
self._session_maker = session_maker
self._cutover_gate = cutover_gate
self._issuer = issuer
def issue_interactive(
self,
*,
tenant_id: str,
account_id: str,
bindings: tuple[KnowledgeFSBatchSpaceBinding, ...],
trace_id: str,
) -> KnowledgeFSIssuedBatchCapability:
self._cutover_gate.require_capability_v2(tenant_id=tenant_id)
if self._issuer is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Capability v2 is disabled")
_validate_bindings(bindings)
control_space_ids = tuple(binding.control_space_id for binding in bindings)
with self._session_maker() as session:
rows = session.execute(
sa.select(KnowledgeFSControlSpace, KnowledgeFSAuthorizationRevision)
.join(
KnowledgeFSAuthorizationRevision,
sa.and_(
KnowledgeFSAuthorizationRevision.tenant_id == KnowledgeFSControlSpace.tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == KnowledgeFSControlSpace.id,
),
)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id.in_(control_space_ids),
KnowledgeFSControlSpace.state == KnowledgeFSControlSpaceState.ACTIVE,
)
).all()
rows_by_control_id = {space.id: (space, revision) for space, revision in rows}
if len(rows_by_control_id) != len(bindings):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch authorization snapshot is incomplete")
ordered_revisions: list[KnowledgeFSAuthorizationRevision] = []
for binding in bindings:
space, revision = rows_by_control_id[binding.control_space_id]
if space.knowledge_space_id != binding.knowledge_space_id:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch registration changed")
ordered_revisions.append(revision)
knowledge_space_ids = tuple(binding.knowledge_space_id for binding in bindings)
digest = hashlib.sha256("\0".join((*control_space_ids, *knowledge_space_ids)).encode()).hexdigest()[:24]
issued = self._issuer.issue(
CapabilityIssueRequest(
actor=f"dify-account:{account_id}",
authz_revision=CapabilityAuthzRevision(
membership_epoch=max(revision.membership_epoch for revision in ordered_revisions),
space_acl_epoch=max(revision.space_acl_epoch for revision in ordered_revisions),
external_access_epoch=max(revision.external_access_epoch for revision in ordered_revisions),
credential_revision=None,
),
caller_kind="interactive",
content_policy_revision=max(revision.content_policy_revision for revision in ordered_revisions),
content_scope_ids=knowledge_space_ids,
control_space_id=bindings[0].control_space_id,
grant_id=f"account:{account_id}:batch:{digest}",
namespace_id=tenant_id,
operation_id="batchKnowledgeSpaceProductSummaries",
principal_id=account_id,
resource=CapabilityResource(type="namespace", id=tenant_id),
trace_id=trace_id,
)
)
return KnowledgeFSIssuedBatchCapability(
token=issued.token,
expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC),
knowledge_space_ids=knowledge_space_ids,
trace_id=trace_id,
)
def _validate_bindings(bindings: tuple[KnowledgeFSBatchSpaceBinding, ...]) -> None:
if not bindings or len(bindings) > MAX_BATCH_SPACE_SUMMARIES:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch size must be between 1 and 100")
control_ids = tuple(binding.control_space_id.strip() for binding in bindings)
space_ids = tuple(binding.knowledge_space_id.strip() for binding in bindings)
if (
any(not identifier for identifier in (*control_ids, *space_ids))
or len(set(control_ids)) != len(control_ids)
or len(set(space_ids)) != len(space_ids)
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch bindings must be unique and non-empty")
__all__ = [
"MAX_BATCH_SPACE_SUMMARIES",
"KnowledgeFSBatchCapabilityBroker",
"KnowledgeFSBatchCapabilityIssuerPort",
"KnowledgeFSBatchSpaceBinding",
"KnowledgeFSIssuedBatchCapability",
]
@@ -0,0 +1,492 @@
"""Issue operation capabilities from already-authorized Dify product state."""
from __future__ import annotations
import uuid
from datetime import UTC, datetime
from typing import NamedTuple, Protocol, cast
import sqlalchemy as sa
from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import naive_utc_now
from models.knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSApiCredential,
KnowledgeFSApiCredentialStatus,
KnowledgeFSAppSpaceJoinStatus,
KnowledgeFSAppSpaceJoinType,
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpaceState,
KnowledgeFSExternalAccessPolicy,
)
from repositories.sqlalchemy_knowledge_fs_capability_issuance_reservation_repository import (
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository,
)
from services.knowledge_fs.app_admission_service import KnowledgeFSAppPrincipalProfile
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
from services.knowledge_fs.cutover_runtime_gate import KnowledgeFSWorkspaceRuntimeGatePort
from services.knowledge_fs.product_operations import (
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
KnowledgeFSProductOperation,
is_product_operation_ready,
)
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
from services.knowledge_fs.product_service import KnowledgeFSProductService
from services.knowledge_fs_capability import (
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
CapabilityAuthzRevision,
CapabilityCallerKind,
CapabilityIssueRequest,
CapabilityResource,
IssuedKnowledgeFSCapability,
)
class KnowledgeFSCapabilityIssuerPort(Protocol):
def issue(self, request: CapabilityIssueRequest) -> IssuedKnowledgeFSCapability: ...
class KnowledgeFSIssuedProductCapability(NamedTuple):
token: str
expires_at: datetime
operation_id: str
knowledge_space_id: str
knowledge_space_revision: int
trace_id: str
class KnowledgeFSAuthorizationSnapshot(NamedTuple):
authz_revision: CapabilityAuthzRevision
content_policy_revision: int
class KnowledgeFSCapabilityBroker:
def __init__(
self,
session_maker: sessionmaker[Session],
*,
cutover_gate: KnowledgeFSWorkspaceRuntimeGatePort,
product: KnowledgeFSProductService,
issuer: KnowledgeFSCapabilityIssuerPort | None,
) -> None:
self._session_maker = session_maker
self._cutover_gate = cutover_gate
self._product = product
self._issuer = issuer
def issue_interactive(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
self._cutover_gate.require_capability_v2(tenant_id=tenant_id)
product_operation, capability_operation_id = _operation_contract(operation_id)
issuer = self._require_issuer()
normalized_trace_id = _trace_id(trace_id)
with self._session_maker.begin() as session:
revision = _lock_authorization_revision(
session,
tenant_id=tenant_id,
control_space_id=control_space_id,
)
authorized = self._product.authorize_control_space_in_session(
session=session,
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
permission=product_operation.permission,
require_active=True,
)
space = authorized.control_space
knowledge_space_id = space.knowledge_space_id
if knowledge_space_id is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS control-space is not registered")
request = _issue_request(
capability_operation_id=capability_operation_id,
tenant_id=tenant_id,
control_space_id=control_space_id,
knowledge_space_id=knowledge_space_id,
principal_id=account_id,
actor=f"dify-account:{account_id}",
caller_kind="interactive",
credential_revision=None,
revision=revision,
resource_id=resource_id,
trace_id=normalized_trace_id,
)
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
knowledge_space_revision = space.knowledge_space_revision
return self._issue_reserved(
issuer=issuer,
request=request,
operation_id=operation_id,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
def issue_service(
self,
*,
profile: KnowledgeFSServiceCredentialProfile,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
self._cutover_gate.require_capability_v2(tenant_id=profile.tenant_id)
_, capability_operation_id = _operation_contract(operation_id)
issuer = self._require_issuer()
normalized_trace_id = _trace_id(trace_id)
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
with self._session_maker.begin() as session:
revision = _lock_authorization_revision(
session,
tenant_id=profile.tenant_id,
control_space_id=profile.control_space_id,
)
credential, space, policy = _load_service_authorization(session, profile=profile)
if (
credential.status is not KnowledgeFSApiCredentialStatus.ACTIVE
or credential.principal != profile.principal_id
or (credential.expires_at is not None and credential.expires_at <= naive_utc_now())
or policy is None
or not policy.service_api_enabled
or capability_operation.action not in credential.allowed_actions
or space.state is not KnowledgeFSControlSpaceState.ACTIVE
or space.knowledge_space_id is None
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS credential is no longer authorized")
knowledge_space_id = space.knowledge_space_id
request = _issue_request(
capability_operation_id=capability_operation_id,
tenant_id=profile.tenant_id,
control_space_id=profile.control_space_id,
knowledge_space_id=knowledge_space_id,
principal_id=credential.principal,
actor=f"dify-kfs-credential:{credential.principal}",
caller_kind="service",
credential_revision=credential.revision,
revision=revision,
resource_id=resource_id,
trace_id=normalized_trace_id,
)
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
knowledge_space_revision = space.knowledge_space_revision
return self._issue_reserved(
issuer=issuer,
request=request,
operation_id=operation_id,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
def issue_app(
self,
*,
profile: KnowledgeFSAppPrincipalProfile,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
self._cutover_gate.require_capability_v2(tenant_id=profile.tenant_id)
_, capability_operation_id = _operation_contract(operation_id)
issuer = self._require_issuer()
normalized_trace_id = _trace_id(trace_id)
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
with self._session_maker.begin() as session:
revision = _lock_authorization_revision(
session,
tenant_id=profile.tenant_id,
control_space_id=profile.control_space_id,
)
join, space, policy = _load_app_authorization(session, profile=profile)
caller_kind = _app_caller_kind(profile.caller_kind)
channel_enabled = (
policy.agent_enabled
if profile.caller_kind is KnowledgeFSAppSpaceJoinType.AGENT
else policy.workflow_enabled
)
if (
join.status is not KnowledgeFSAppSpaceJoinStatus.ACTIVE
or capability_operation.action != profile.action
or not channel_enabled
or space.state is not KnowledgeFSControlSpaceState.ACTIVE
or space.knowledge_space_id is None
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app binding is no longer authorized")
knowledge_space_id = space.knowledge_space_id
request = _issue_request(
capability_operation_id=capability_operation_id,
tenant_id=profile.tenant_id,
control_space_id=profile.control_space_id,
knowledge_space_id=knowledge_space_id,
principal_id=profile.app_id,
actor=f"dify-app:{profile.app_id}",
caller_kind=caller_kind,
credential_revision=None,
revision=revision,
resource_id=resource_id,
trace_id=normalized_trace_id,
)
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
knowledge_space_revision = space.knowledge_space_revision
return self._issue_reserved(
issuer=issuer,
request=request,
operation_id=operation_id,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
def _require_issuer(self) -> KnowledgeFSCapabilityIssuerPort:
if self._issuer is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Capability v2 is disabled")
return self._issuer
def _issue_reserved(
self,
*,
issuer: KnowledgeFSCapabilityIssuerPort,
request: CapabilityIssueRequest,
operation_id: str,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> KnowledgeFSIssuedProductCapability:
try:
issued = issuer.issue(request)
with self._session_maker.begin() as session:
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).mark_issued(
tenant_id=request.namespace_id,
grant_id=request.grant_id,
issued_at=datetime.fromtimestamp(issued.claims.iat, tz=UTC).replace(tzinfo=None),
token_expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC).replace(tzinfo=None),
)
except Exception as issuance_error:
try:
with self._session_maker.begin() as session:
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).mark_failed(
tenant_id=request.namespace_id,
grant_id=request.grant_id,
failed_at=naive_utc_now(),
failure_code=type(issuance_error).__name__,
)
except Exception as terminal_error:
raise KnowledgeFSOperationUnavailableError(
"KnowledgeFS capability issuance terminal state could not be persisted"
) from terminal_error
raise
return KnowledgeFSIssuedProductCapability(
token=issued.token,
expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC),
operation_id=operation_id,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
trace_id=request.trace_id,
)
def _lock_authorization_revision(
session: Session,
*,
tenant_id: str,
control_space_id: str,
) -> KnowledgeFSAuthorizationSnapshot:
revision = session.scalar(
sa.select(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
)
.with_for_update()
)
if revision is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS authorization revision is missing")
result = session.execute(
sa.update(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.id == revision.id,
KnowledgeFSAuthorizationRevision.membership_epoch == revision.membership_epoch,
KnowledgeFSAuthorizationRevision.space_acl_epoch == revision.space_acl_epoch,
KnowledgeFSAuthorizationRevision.external_access_epoch == revision.external_access_epoch,
KnowledgeFSAuthorizationRevision.content_policy_revision == revision.content_policy_revision,
KnowledgeFSAuthorizationRevision.revoke_sequence == revision.revoke_sequence,
)
.values(updated_at=sa.func.current_timestamp())
)
if cast(CursorResult[tuple[object, ...]], result).rowcount != 1:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS authorization revision changed during issuance")
return KnowledgeFSAuthorizationSnapshot(
authz_revision=CapabilityAuthzRevision(
membership_epoch=revision.membership_epoch,
space_acl_epoch=revision.space_acl_epoch,
external_access_epoch=revision.external_access_epoch,
credential_revision=None,
),
content_policy_revision=revision.content_policy_revision,
)
def _load_service_authorization(
session: Session,
*,
profile: KnowledgeFSServiceCredentialProfile,
) -> tuple[KnowledgeFSApiCredential, KnowledgeFSControlSpace, KnowledgeFSExternalAccessPolicy | None]:
row = session.execute(
sa.select(
KnowledgeFSApiCredential,
KnowledgeFSControlSpace,
KnowledgeFSExternalAccessPolicy,
)
.join(
KnowledgeFSControlSpace,
sa.and_(
KnowledgeFSControlSpace.tenant_id == KnowledgeFSApiCredential.tenant_id,
KnowledgeFSControlSpace.id == KnowledgeFSApiCredential.control_space_id,
),
)
.outerjoin(
KnowledgeFSExternalAccessPolicy,
sa.and_(
KnowledgeFSExternalAccessPolicy.tenant_id == KnowledgeFSApiCredential.tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == KnowledgeFSApiCredential.control_space_id,
),
)
.where(
KnowledgeFSApiCredential.tenant_id == profile.tenant_id,
KnowledgeFSApiCredential.control_space_id == profile.control_space_id,
KnowledgeFSApiCredential.id == profile.credential_id,
)
).one_or_none()
if row is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS credential is no longer authorized")
return row._t
def _load_app_authorization(
session: Session,
*,
profile: KnowledgeFSAppPrincipalProfile,
) -> tuple[AppKnowledgeFSSpaceJoin, KnowledgeFSControlSpace, KnowledgeFSExternalAccessPolicy]:
row = session.execute(
sa.select(
AppKnowledgeFSSpaceJoin,
KnowledgeFSControlSpace,
KnowledgeFSExternalAccessPolicy,
)
.join(
KnowledgeFSControlSpace,
sa.and_(
KnowledgeFSControlSpace.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
KnowledgeFSControlSpace.id == AppKnowledgeFSSpaceJoin.control_space_id,
),
)
.join(
KnowledgeFSExternalAccessPolicy,
sa.and_(
KnowledgeFSExternalAccessPolicy.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
),
)
.where(
AppKnowledgeFSSpaceJoin.tenant_id == profile.tenant_id,
AppKnowledgeFSSpaceJoin.control_space_id == profile.control_space_id,
AppKnowledgeFSSpaceJoin.app_id == profile.app_id,
AppKnowledgeFSSpaceJoin.id == profile.join_id,
AppKnowledgeFSSpaceJoin.join_type == profile.caller_kind,
)
).one_or_none()
if row is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app binding is no longer authorized")
return row._t
def _issue_request(
*,
capability_operation_id: str,
tenant_id: str,
control_space_id: str,
knowledge_space_id: str,
principal_id: str,
actor: str,
caller_kind: CapabilityCallerKind,
credential_revision: int | None,
revision: KnowledgeFSAuthorizationSnapshot,
resource_id: str | None,
trace_id: str,
) -> CapabilityIssueRequest:
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
if capability_operation.resource_type == "knowledge_space":
resource = CapabilityResource(type="knowledge_space", id=knowledge_space_id)
elif capability_operation.resource_type in {
"document",
"job",
"query",
"research_task",
"source",
"upload_session",
}:
if resource_id is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS child resource is required")
resource = CapabilityResource(
type=capability_operation.resource_type,
id=resource_id,
parent_id=knowledge_space_id,
)
else:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product resource resolver is not registered")
grant_id = str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f"dify-kfs-capability:{tenant_id}:{caller_kind}:{principal_id}:{trace_id}",
)
)
return CapabilityIssueRequest(
actor=actor,
authz_revision=revision.authz_revision.model_copy(update={"credential_revision": credential_revision}),
caller_kind=caller_kind,
content_policy_revision=revision.content_policy_revision,
control_space_id=control_space_id,
grant_id=grant_id,
namespace_id=tenant_id,
operation_id=capability_operation_id,
principal_id=principal_id,
resource=resource,
trace_id=trace_id,
)
def _trace_id(trace_id: str | None) -> str:
normalized = (trace_id or str(uuid.uuid4())).strip()
if not normalized:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS capability trace id is required")
return normalized
def _operation_contract(operation_id: str) -> tuple[KnowledgeFSProductOperation, str]:
product_operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
if product_operation is None or not is_product_operation_ready(operation_id):
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
capability_operation_id = product_operation.capability_operation_id
if capability_operation_id is None:
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
return product_operation, capability_operation_id
def _app_caller_kind(caller_kind: KnowledgeFSAppSpaceJoinType) -> CapabilityCallerKind:
if caller_kind is KnowledgeFSAppSpaceJoinType.AGENT:
return "agent"
return "workflow"
__all__ = [
"KnowledgeFSCapabilityBroker",
"KnowledgeFSCapabilityIssuerPort",
"KnowledgeFSIssuedProductCapability",
]
+963
View File
@@ -0,0 +1,963 @@
"""Default-deny P9 cleanup readiness, approval, and irreversible-start fence.
This module never executes schema or data deletion. It validates operator-supplied
production evidence, persists four-eyes approval, and atomically closes rollback
before an external, separately controlled migration may begin.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Callable, Iterable
from datetime import datetime, timedelta
from typing import Literal, NamedTuple
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import ensure_naive_utc, naive_utc_now
from models.knowledge_fs_cleanup import (
KnowledgeFSCleanupAuthorization,
KnowledgeFSCleanupAuthorizationStatus,
KnowledgeFSCleanupTarget,
)
from models.knowledge_fs_cutover import (
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
knowledge_fs_cutover_smoke_results_passed,
)
from repositories.knowledge_fs_cleanup_repository import KnowledgeFSCleanupAuthorizationCASUpdate
from repositories.knowledge_fs_cutover_repository import KnowledgeFSCutoverCASUpdate
from repositories.sqlalchemy_knowledge_fs_cleanup_repository import (
SQLAlchemyKnowledgeFSCleanupAuthorizationRepository,
)
from repositories.sqlalchemy_knowledge_fs_cutover_repository import SQLAlchemyKnowledgeFSCutoverRepository
from services.knowledge_fs.cutover import (
knowledge_fs_remote_activation_evidence_consistent,
knowledge_fs_remote_freeze_evidence_consistent,
)
_PLAN_SCHEMA_VERSION = "knowledge-fs-p9-cleanup/v1"
_START_CONFIRMATION = "START-KNOWLEDGE-FS-IRREVERSIBLE-CLEANUP"
_MAX_APPROVAL_LIFETIME = timedelta(hours=24)
_REQUIRED_TARGETS = frozenset(KnowledgeFSCleanupTarget)
def cleanup_workspace_cohort_digest(tenant_ids: Iterable[str | UUID]) -> str:
"""Return the canonical digest for the complete, sorted cleanup cohort."""
canonical_ids = sorted(str(tenant_id) for tenant_id in tenant_ids)
canonical = json.dumps(canonical_ids, ensure_ascii=False, separators=(",", ":")).encode()
return f"sha256:{hashlib.sha256(canonical).hexdigest()}"
class StrictCleanupInput(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class CleanupReadinessEvidenceInput(StrictCleanupInput):
schema_version: Literal["knowledge-fs-p9-cleanup/v1"]
tenant_id: UUID
request_id: UUID
expected_cas_version: int = Field(ge=0)
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
targets: list[KnowledgeFSCleanupTarget] = Field(min_length=6, max_length=6)
expected_workspace_count: int = Field(gt=0, le=100_000)
workspace_tenant_ids: list[UUID] = Field(min_length=1, max_length=100_000)
workspace_cohort_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
workspace_inventory_reference: str = Field(min_length=1, max_length=1024)
evidence_environment: Literal["production"]
observed_at: datetime
requested_at: datetime
legacy_route_zero_window_started_at: datetime
legacy_route_zero_window_ends_at: datetime
rollback_window_seconds: int = Field(gt=0)
legacy_route_calls: int = Field(ge=0)
legacy_access_route_calls: int = Field(ge=0)
legacy_member_route_calls: int = Field(ge=0)
legacy_api_key_route_calls: int = Field(ge=0)
legacy_route_metric_reference: str = Field(min_length=1, max_length=1024)
maximum_token_expires_at: datetime
backup_reference: str = Field(min_length=1, max_length=1024)
backup_verified_at: datetime
restore_drill_reference: str = Field(min_length=1, max_length=1024)
restore_drill_verified_at: datetime
change_window_approval_reference: str = Field(min_length=1, max_length=1024)
requested_by_account_id: UUID
@field_validator(
"observed_at",
"requested_at",
"legacy_route_zero_window_started_at",
"legacy_route_zero_window_ends_at",
"maximum_token_expires_at",
"backup_verified_at",
"restore_drill_verified_at",
)
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("cleanup evidence timestamps require an explicit timezone")
return value
@model_validator(mode="after")
def require_complete_unique_plan(self) -> CleanupReadinessEvidenceInput:
if frozenset(self.targets) != _REQUIRED_TARGETS or len(set(self.targets)) != len(self.targets):
raise ValueError("cleanup plan must contain every destructive target exactly once")
tenant_ids = [str(tenant_id) for tenant_id in self.workspace_tenant_ids]
if tenant_ids != sorted(tenant_ids) or len(tenant_ids) != len(set(tenant_ids)):
raise ValueError("cleanup Workspace cohort must be unique and sorted")
if str(self.tenant_id) not in tenant_ids:
raise ValueError("cleanup authorization tenant must be in the Workspace cohort")
if self.expected_workspace_count != len(tenant_ids):
raise ValueError("cleanup Workspace count does not match the cohort")
if self.workspace_cohort_digest != cleanup_workspace_cohort_digest(tenant_ids):
raise ValueError("cleanup Workspace cohort digest does not match the canonical tenant list")
if self.legacy_route_calls != (
self.legacy_access_route_calls + self.legacy_member_route_calls + self.legacy_api_key_route_calls
):
raise ValueError("legacy route total does not match access/member/API-key counts")
return self
class CleanupApprovalInput(StrictCleanupInput):
schema_version: Literal["knowledge-fs-p9-cleanup-approval/v1"]
tenant_id: UUID
request_id: UUID
expected_cas_version: int = Field(ge=0)
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
approved_by_account_id: UUID
approved_at: datetime
approval_expires_at: datetime
@field_validator("approved_at", "approval_expires_at")
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("cleanup approval timestamps require an explicit timezone")
return value
class CleanupStartInput(StrictCleanupInput):
schema_version: Literal["knowledge-fs-p9-cleanup-start/v1"]
tenant_id: UUID
request_id: UUID
expected_cas_version: int = Field(ge=0)
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
started_by_account_id: UUID
started_at: datetime
confirmation: Literal["START-KNOWLEDGE-FS-IRREVERSIBLE-CLEANUP"]
@field_validator("started_at")
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("cleanup start timestamp requires an explicit timezone")
return value
class CleanupArchivedRowCounts(StrictCleanupInput):
knowledge_space_members: int = Field(ge=0)
knowledge_space_access_policies: int = Field(ge=0)
knowledge_space_access_policy_members: int = Field(ge=0)
knowledge_space_api_access: int = Field(ge=0)
knowledge_space_api_keys: int = Field(ge=0)
knowledge_space_permission_snapshots: int = Field(ge=0)
class CleanupCompletionChecks(StrictCleanupInput):
legacy_foreign_keys_remaining: Literal[0]
legacy_tables_remaining: Literal[0]
legacy_routes_registered: Literal[0]
legacy_v1_auth_acceptances: Literal[0]
raw_proxy_routes_registered: Literal[0]
post_cleanup_smoke_passed: Literal[True]
recovery_material_verified: Literal[True]
class CleanupCompletionEvidenceInput(StrictCleanupInput):
schema_version: Literal["knowledge-fs-p9-cleanup-completion/v1"]
tenant_id: UUID
request_id: UUID
expected_cas_version: int = Field(ge=0)
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
migration_bundle_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
database_engine: Literal["postgresql", "tidb"]
migration_revision: str = Field(min_length=1, max_length=255)
archived_row_counts: CleanupArchivedRowCounts
checks: CleanupCompletionChecks
archive_reference: str = Field(min_length=1, max_length=1024)
catalog_verification_reference: str = Field(min_length=1, max_length=1024)
route_metric_reference: str = Field(min_length=1, max_length=1024)
post_cleanup_smoke_reference: str = Field(min_length=1, max_length=1024)
recovery_material_reference: str = Field(min_length=1, max_length=1024)
completed_by_account_id: UUID
completed_at: datetime
@field_validator("completed_at")
@classmethod
def require_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("cleanup completion timestamp requires an explicit timezone")
return value
@model_validator(mode="after")
def require_reviewed_bundle(self) -> CleanupCompletionEvidenceInput:
if self.migration_bundle_digest != self.plan_digest:
raise ValueError("cleanup completion bundle digest must match the approved plan")
return self
class CleanupReadinessReport(NamedTuple):
tenant_id: str
request_id: str
ready: bool
reasons: tuple[str, ...]
status: str
applied: bool
replayed: bool
operator_attested_evidence: bool
class CleanupApprovalReport(NamedTuple):
tenant_id: str
request_id: str
approvable: bool
reasons: tuple[str, ...]
status: str
applied: bool
replayed: bool
class CleanupStartReport(NamedTuple):
tenant_id: str
request_id: str
startable: bool
reasons: tuple[str, ...]
status: str
applied: bool
replayed: bool
irreversible_cleanup_at: str | None
destructive_actions_executed: bool
class CleanupCompletionReport(NamedTuple):
tenant_id: str
request_id: str
completable: bool
reasons: tuple[str, ...]
status: str
applied: bool
replayed: bool
destructive_actions_executed: bool
class KnowledgeFSCleanupError(RuntimeError):
pass
class KnowledgeFSCleanupNotFoundError(KnowledgeFSCleanupError):
pass
class KnowledgeFSCleanupConflictError(KnowledgeFSCleanupError):
pass
class KnowledgeFSCleanupGateBlockedError(KnowledgeFSCleanupError):
pass
class KnowledgeFSCleanupService:
"""Persist readiness and approval without executing destructive operations."""
_session_maker: sessionmaker[Session]
_clock: Callable[[], datetime]
def __init__(
self,
session_maker: sessionmaker[Session],
clock: Callable[[], datetime] = naive_utc_now,
):
self._session_maker = session_maker
self._clock = clock
def request_cleanup(
self,
payload: CleanupReadinessEvidenceInput,
*,
apply: bool,
) -> CleanupReadinessReport:
tenant_id = str(payload.tenant_id)
request_id = str(payload.request_id)
with self._session_maker.begin() as session:
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
ledger = self._require_ledger(cutover_repository, tenant_id)
reasons = self._readiness_reasons(cutover_repository, ledger, payload)
existing = cleanup_repository.get(
tenant_id=tenant_id,
ledger_id=ledger.id,
request_id=request_id,
)
if existing is not None:
self._assert_request_replay(existing, payload)
replay_payload = payload.model_copy(update={"expected_cas_version": ledger.cas_version})
reasons = self._readiness_reasons(cutover_repository, ledger, replay_payload)
return CleanupReadinessReport(
tenant_id,
request_id,
not reasons,
reasons,
existing.status.value,
apply,
True,
True,
)
if not apply:
return CleanupReadinessReport(
tenant_id,
request_id,
not reasons,
reasons,
"dry_run",
False,
False,
True,
)
if reasons:
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
self._cas_ledger(
cutover_repository,
KnowledgeFSCutoverCASUpdate(
tenant_id=tenant_id,
expected_phase=ledger.phase,
expected_cas_version=ledger.cas_version,
new_phase=ledger.phase,
),
)
cleanup_repository.add(
KnowledgeFSCleanupAuthorization(
tenant_id=tenant_id,
ledger_id=ledger.id,
request_id=request_id,
plan_schema_version=_PLAN_SCHEMA_VERSION,
plan_digest=payload.plan_digest,
targets=[target.value for target in payload.targets],
readiness_evidence=payload.model_dump(mode="json"),
requested_by_account_id=str(payload.requested_by_account_id),
requested_at=ensure_naive_utc(payload.requested_at),
readiness_ledger_cas_version=ledger.cas_version + 1,
)
)
return CleanupReadinessReport(
tenant_id,
request_id,
True,
(),
KnowledgeFSCleanupAuthorizationStatus.REQUESTED.value,
True,
False,
True,
)
def approve_cleanup(self, payload: CleanupApprovalInput, *, apply: bool) -> CleanupApprovalReport:
tenant_id = str(payload.tenant_id)
request_id = str(payload.request_id)
with self._session_maker.begin() as session:
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
ledger = self._require_ledger(cutover_repository, tenant_id)
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.APPROVED:
self._assert_approval_replay(authorization, payload)
return CleanupApprovalReport(tenant_id, request_id, True, (), "approved", apply, True)
if authorization.status in {
KnowledgeFSCleanupAuthorizationStatus.STARTED,
KnowledgeFSCleanupAuthorizationStatus.COMPLETED,
}:
raise KnowledgeFSCleanupConflictError("Cleanup authorization has already started")
reasons = self._approval_reasons(cutover_repository, ledger, authorization, payload)
if not apply:
return CleanupApprovalReport(
tenant_id,
request_id,
not reasons,
reasons,
authorization.status.value,
False,
False,
)
if reasons:
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
self._cas_ledger(
cutover_repository,
KnowledgeFSCutoverCASUpdate(
tenant_id=tenant_id,
expected_phase=ledger.phase,
expected_cas_version=ledger.cas_version,
new_phase=ledger.phase,
),
)
changed = cleanup_repository.compare_and_set(
KnowledgeFSCleanupAuthorizationCASUpdate(
tenant_id=tenant_id,
ledger_id=ledger.id,
request_id=request_id,
expected_status=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
expected_row_version=authorization.row_version,
new_status=KnowledgeFSCleanupAuthorizationStatus.APPROVED,
approved_by_account_id=str(payload.approved_by_account_id),
approved_at=ensure_naive_utc(payload.approved_at),
approval_expires_at=ensure_naive_utc(payload.approval_expires_at),
approved_ledger_cas_version=ledger.cas_version + 1,
)
)
if not changed:
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during approval")
return CleanupApprovalReport(tenant_id, request_id, True, (), "approved", True, False)
def start_cleanup(self, payload: CleanupStartInput, *, apply: bool) -> CleanupStartReport:
tenant_id = str(payload.tenant_id)
request_id = str(payload.request_id)
with self._session_maker.begin() as session:
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
ledger = self._require_ledger(cutover_repository, tenant_id)
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.STARTED:
self._assert_start_replay(authorization, payload)
return CleanupStartReport(
tenant_id,
request_id,
True,
(),
"started",
apply,
True,
_iso(ledger.irreversible_cleanup_at),
False,
)
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED:
raise KnowledgeFSCleanupConflictError("Cleanup authorization has already completed")
reasons = self._start_reasons(cutover_repository, ledger, authorization, payload)
if not apply:
return CleanupStartReport(
tenant_id,
request_id,
not reasons,
reasons,
authorization.status.value,
False,
False,
_iso(ledger.irreversible_cleanup_at),
False,
)
if reasons:
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
started_at = ensure_naive_utc(payload.started_at)
self._fence_cleanup_cohort(cutover_repository, started_at=started_at)
changed = cleanup_repository.compare_and_set(
KnowledgeFSCleanupAuthorizationCASUpdate(
tenant_id=tenant_id,
ledger_id=ledger.id,
request_id=request_id,
expected_status=KnowledgeFSCleanupAuthorizationStatus.APPROVED,
expected_row_version=authorization.row_version,
new_status=KnowledgeFSCleanupAuthorizationStatus.STARTED,
started_by_account_id=str(payload.started_by_account_id),
started_at=started_at,
started_ledger_cas_version=ledger.cas_version + 1,
)
)
if not changed:
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during start")
return CleanupStartReport(
tenant_id,
request_id,
True,
(),
"started",
True,
False,
_iso(started_at),
False,
)
def complete_cleanup(
self,
payload: CleanupCompletionEvidenceInput,
*,
apply: bool,
) -> CleanupCompletionReport:
tenant_id = str(payload.tenant_id)
request_id = str(payload.request_id)
with self._session_maker.begin() as session:
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
ledger = self._require_ledger(cutover_repository, tenant_id)
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED:
self._assert_completion_replay(authorization, payload)
return CleanupCompletionReport(
tenant_id,
request_id,
True,
(),
"completed",
apply,
True,
True,
)
reasons = self._completion_reasons(cutover_repository, ledger, authorization, payload)
if not apply:
return CleanupCompletionReport(
tenant_id,
request_id,
not reasons,
reasons,
authorization.status.value,
False,
False,
False,
)
if reasons:
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
self._cas_ledger(
cutover_repository,
KnowledgeFSCutoverCASUpdate(
tenant_id=tenant_id,
expected_phase=ledger.phase,
expected_cas_version=ledger.cas_version,
new_phase=ledger.phase,
),
)
completed_at = ensure_naive_utc(payload.completed_at)
changed = cleanup_repository.compare_and_set(
KnowledgeFSCleanupAuthorizationCASUpdate(
tenant_id=tenant_id,
ledger_id=ledger.id,
request_id=request_id,
expected_status=KnowledgeFSCleanupAuthorizationStatus.STARTED,
expected_row_version=authorization.row_version,
new_status=KnowledgeFSCleanupAuthorizationStatus.COMPLETED,
completed_by_account_id=str(payload.completed_by_account_id),
completed_at=completed_at,
completion_evidence=payload.model_dump(mode="json"),
completed_ledger_cas_version=ledger.cas_version + 1,
)
)
if not changed:
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during completion")
return CleanupCompletionReport(
tenant_id,
request_id,
True,
(),
"completed",
True,
False,
True,
)
def status(self, *, tenant_id: str, request_id: str) -> dict[str, object]:
with self._session_maker() as session:
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
ledger = self._require_ledger(cutover_repository, tenant_id)
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
return {
"tenant_id": tenant_id,
"request_id": request_id,
"status": authorization.status.value,
"plan_schema_version": authorization.plan_schema_version,
"plan_digest": authorization.plan_digest,
"targets": authorization.targets,
"readiness_evidence": authorization.readiness_evidence,
"requested_by_account_id": authorization.requested_by_account_id,
"requested_at": _iso(authorization.requested_at),
"readiness_ledger_cas_version": authorization.readiness_ledger_cas_version,
"approved_by_account_id": authorization.approved_by_account_id,
"approved_at": _iso(authorization.approved_at),
"approval_expires_at": _iso(authorization.approval_expires_at),
"approved_ledger_cas_version": authorization.approved_ledger_cas_version,
"started_by_account_id": authorization.started_by_account_id,
"started_at": _iso(authorization.started_at),
"started_ledger_cas_version": authorization.started_ledger_cas_version,
"completed_by_account_id": authorization.completed_by_account_id,
"completed_at": _iso(authorization.completed_at),
"completion_evidence": authorization.completion_evidence,
"completed_ledger_cas_version": authorization.completed_ledger_cas_version,
"row_version": authorization.row_version,
"irreversible_cleanup_at": _iso(ledger.irreversible_cleanup_at),
"destructive_actions_executed": (
authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED
),
"operator_attested_evidence": True,
}
def _readiness_reasons(
self,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
payload: CleanupReadinessEvidenceInput,
*,
allow_irreversible: bool = False,
) -> tuple[str, ...]:
reasons: list[str] = []
now = ensure_naive_utc(self._clock())
observed_at = ensure_naive_utc(payload.observed_at)
requested_at = ensure_naive_utc(payload.requested_at)
zero_window_start = ensure_naive_utc(payload.legacy_route_zero_window_started_at)
zero_window_end = ensure_naive_utc(payload.legacy_route_zero_window_ends_at)
maximum_token_expires_at = ensure_naive_utc(payload.maximum_token_expires_at)
backup_verified_at = ensure_naive_utc(payload.backup_verified_at)
restore_drill_verified_at = ensure_naive_utc(payload.restore_drill_verified_at)
if observed_at > now or requested_at > now:
reasons.append("cleanup evidence cannot be future-dated")
if requested_at < observed_at:
reasons.append("cleanup request predates its evidence observation")
if payload.legacy_access_route_calls != 0:
reasons.append("legacy access route calls are nonzero")
if payload.legacy_member_route_calls != 0:
reasons.append("legacy member route calls are nonzero")
if payload.legacy_api_key_route_calls != 0:
reasons.append("legacy API-key route calls are nonzero")
if payload.legacy_route_calls != 0:
reasons.append("legacy route calls are nonzero")
if zero_window_end > observed_at or zero_window_end < zero_window_start:
reasons.append("legacy route zero window is invalid")
elif zero_window_end - zero_window_start < timedelta(seconds=payload.rollback_window_seconds):
reasons.append("legacy route zero window is shorter than one rollback window")
if maximum_token_expires_at > observed_at:
reasons.append("maximum token TTL has not elapsed")
if backup_verified_at > observed_at or restore_drill_verified_at > observed_at:
reasons.append("backup or restore drill verification is future evidence")
ledgers = repository.list_ledgers()
expected_tenant_ids = tuple(str(tenant_id) for tenant_id in payload.workspace_tenant_ids)
actual_tenant_ids = tuple(candidate.tenant_id for candidate in ledgers)
if len(ledgers) != payload.expected_workspace_count:
reasons.append("persisted Workspace count does not match the cleanup cohort")
if actual_tenant_ids != expected_tenant_ids:
reasons.append("persisted Workspace cohort does not exactly match the cleanup inventory")
if cleanup_workspace_cohort_digest(actual_tenant_ids) != payload.workspace_cohort_digest:
reasons.append("persisted Workspace cohort digest changed")
for candidate in ledgers:
workspace_reasons = self._workspace_readiness_reasons(
repository,
candidate,
payload,
allow_irreversible=allow_irreversible,
)
if candidate.tenant_id == ledger.tenant_id and candidate.cas_version != payload.expected_cas_version:
workspace_reasons.insert(0, "cutover ledger CAS version changed")
for reason in workspace_reasons:
if candidate.tenant_id == ledger.tenant_id:
reasons.append(reason)
else:
reasons.append(f"Workspace {candidate.tenant_id}: {reason}")
return tuple(dict.fromkeys(reasons))
def _workspace_readiness_reasons(
self,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
payload: CleanupReadinessEvidenceInput,
*,
allow_irreversible: bool,
) -> list[str]:
reasons: list[str] = []
now = ensure_naive_utc(self._clock())
observed_at = ensure_naive_utc(payload.observed_at)
zero_window_start = ensure_naive_utc(payload.legacy_route_zero_window_started_at)
backup_verified_at = ensure_naive_utc(payload.backup_verified_at)
restore_drill_verified_at = ensure_naive_utc(payload.restore_drill_verified_at)
if ledger.phase is not KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP:
reasons.append("workspace is not ready_for_cleanup")
if allow_irreversible:
if ledger.irreversible_cleanup_at is None:
reasons.append("irreversible cleanup fence is missing")
elif ledger.irreversible_cleanup_at is not None:
reasons.append("irreversible cleanup has already started")
if ledger.observation_completed_at is None:
reasons.append("production observation is not persisted as complete")
if ledger.observation_window_ends_at is None or ledger.observation_completed_at is None:
reasons.append("observation window evidence is incomplete")
elif ledger.observation_completed_at < ledger.observation_window_ends_at:
reasons.append("persisted observation window has not elapsed")
if ledger.maximum_task_expires_at is None or ledger.observation_completed_at is None:
reasons.append("maximum task TTL evidence is incomplete")
elif ledger.observation_completed_at < ledger.maximum_task_expires_at:
reasons.append("persisted maximum task TTL has not elapsed")
if ledger.observation_completed_at is not None and observed_at < ledger.observation_completed_at:
reasons.append("cleanup evidence predates completed observation")
if (
ledger.rollback_cutoff_at is None
or observed_at < ledger.rollback_cutoff_at
or now < ledger.rollback_cutoff_at
):
reasons.append("rollback cutoff has not elapsed")
if ledger.cutover_at is None or zero_window_start < ledger.cutover_at:
reasons.append("legacy route zero window predates cutover")
if ledger.observation_completed_at is not None and (
backup_verified_at < ledger.observation_completed_at
or restore_drill_verified_at < ledger.observation_completed_at
):
reasons.append("backup and restore drill must be verified after observation completion")
if not ledger.legacy_dependency_ready or ledger.legacy_dependency_checked_at is None:
reasons.append("legacy snapshot/FK dependency dashboard is not ready")
if repository.count_open_issues(tenant_id=ledger.tenant_id, ledger_id=ledger.id) > 0:
reasons.append("open migration issues remain")
if repository.count_unapproved_shadow_diffs(tenant_id=ledger.tenant_id, ledger_id=ledger.id) > 0:
reasons.append("open shadow authorization differences remain")
if (
not knowledge_fs_cutover_smoke_results_passed(ledger.smoke_results)
or ledger.smoke_results is None
or ledger.smoke_results.get("tenant_id") != ledger.tenant_id
):
reasons.append("cutover smoke evidence is incomplete")
if not knowledge_fs_remote_freeze_evidence_consistent(ledger):
reasons.append("remote Workspace freeze evidence is incomplete or inconsistent")
if not knowledge_fs_remote_activation_evidence_consistent(ledger):
reasons.append("remote Workspace activation evidence is incomplete or inconsistent")
if (
ledger.final_revision_watermark is None
or ledger.final_revision_watermark != ledger.applied_revision_watermark
or ledger.final_task_watermark is None
or ledger.final_task_watermark != ledger.applied_task_watermark
):
reasons.append("final delta watermarks are not fully applied")
if not (
ledger.product_routes_enabled
and ledger.capability_v2_enabled
and ledger.integrated_mode_enabled
and ledger.legacy_acl_read_only
):
reasons.append("atomic cutover feature state is not fully enabled")
return reasons
def _approval_reasons(
self,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupApprovalInput,
) -> tuple[str, ...]:
reasons: list[str] = []
approved_at = ensure_naive_utc(payload.approved_at)
expires_at = ensure_naive_utc(payload.approval_expires_at)
now = ensure_naive_utc(self._clock())
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.REQUESTED:
reasons.append("cleanup authorization is not requested")
if ledger.cas_version != payload.expected_cas_version:
reasons.append("cutover ledger CAS version changed")
if authorization.plan_digest != payload.plan_digest:
reasons.append("cleanup plan digest changed")
if authorization.requested_by_account_id == str(payload.approved_by_account_id):
reasons.append("cleanup requires a distinct approver")
if approved_at > now:
reasons.append("cleanup approval cannot be future-dated")
if expires_at <= now or expires_at <= approved_at:
reasons.append("cleanup approval must be currently valid")
if expires_at - approved_at > _MAX_APPROVAL_LIFETIME:
reasons.append("cleanup approval lifetime exceeds 24 hours")
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
reasons.extend(self._readiness_reasons(repository, ledger, evidence))
return tuple(dict.fromkeys(reasons))
def _start_reasons(
self,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupStartInput,
) -> tuple[str, ...]:
reasons: list[str] = []
started_at = ensure_naive_utc(payload.started_at)
now = ensure_naive_utc(self._clock())
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.APPROVED:
reasons.append("cleanup authorization is not approved")
if ledger.cas_version != payload.expected_cas_version:
reasons.append("cutover ledger CAS version changed")
if authorization.plan_digest != payload.plan_digest:
reasons.append("cleanup plan digest changed")
if authorization.approved_at is None or started_at < authorization.approved_at:
reasons.append("cleanup start predates approval")
if started_at > now:
reasons.append("cleanup start cannot be future-dated")
if authorization.approval_expires_at is None or now >= authorization.approval_expires_at:
reasons.append("cleanup approval has expired")
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
reasons.extend(self._readiness_reasons(repository, ledger, evidence))
return tuple(dict.fromkeys(reasons))
def _completion_reasons(
self,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupCompletionEvidenceInput,
) -> tuple[str, ...]:
reasons: list[str] = []
completed_at = ensure_naive_utc(payload.completed_at)
now = ensure_naive_utc(self._clock())
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.STARTED:
reasons.append("cleanup authorization is not started")
if ledger.cas_version != payload.expected_cas_version:
reasons.append("cutover ledger CAS version changed")
if authorization.plan_digest != payload.plan_digest:
reasons.append("cleanup plan digest changed")
if payload.migration_bundle_digest != authorization.plan_digest:
reasons.append("executed migration bundle does not match the approved plan")
if authorization.started_at is None or completed_at < authorization.started_at:
reasons.append("cleanup completion predates the irreversible start")
if completed_at > now:
reasons.append("cleanup completion cannot be future-dated")
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
reasons.extend(
self._readiness_reasons(
repository,
ledger,
evidence,
allow_irreversible=True,
)
)
if authorization.started_at is not None:
for candidate in repository.list_ledgers():
if candidate.irreversible_cleanup_at != authorization.started_at:
reasons.append(f"Workspace {candidate.tenant_id}: irreversible cleanup fence changed")
return tuple(dict.fromkeys(reasons))
@staticmethod
def _require_ledger(
repository: SQLAlchemyKnowledgeFSCutoverRepository,
tenant_id: str,
) -> KnowledgeFSWorkspaceCutoverLedger:
ledger = repository.get_ledger(tenant_id=tenant_id)
if ledger is None:
raise KnowledgeFSCleanupNotFoundError("Workspace cutover ledger was not found")
return ledger
@staticmethod
def _require_authorization(
repository: SQLAlchemyKnowledgeFSCleanupAuthorizationRepository,
ledger: KnowledgeFSWorkspaceCutoverLedger,
request_id: str,
) -> KnowledgeFSCleanupAuthorization:
authorization = repository.get(
tenant_id=ledger.tenant_id,
ledger_id=ledger.id,
request_id=request_id,
)
if authorization is None:
raise KnowledgeFSCleanupNotFoundError("Cleanup authorization was not found")
return authorization
@staticmethod
def _cas_ledger(
repository: SQLAlchemyKnowledgeFSCutoverRepository,
update: KnowledgeFSCutoverCASUpdate,
) -> None:
if not repository.compare_and_set(update):
raise KnowledgeFSCleanupConflictError("Cutover ledger changed during cleanup authorization")
@classmethod
def _fence_cleanup_cohort(
cls,
repository: SQLAlchemyKnowledgeFSCutoverRepository,
*,
started_at: datetime,
) -> None:
ledgers = repository.list_ledgers()
if not ledgers:
raise KnowledgeFSCleanupConflictError("Cleanup Workspace cohort is empty")
for candidate in ledgers:
cls._cas_ledger(
repository,
KnowledgeFSCutoverCASUpdate(
tenant_id=candidate.tenant_id,
expected_phase=candidate.phase,
expected_cas_version=candidate.cas_version,
new_phase=candidate.phase,
irreversible_cleanup_at=started_at,
),
)
@staticmethod
def _assert_request_replay(
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupReadinessEvidenceInput,
) -> None:
if (
authorization.plan_digest != payload.plan_digest
or authorization.targets != [target.value for target in payload.targets]
or authorization.readiness_evidence != payload.model_dump(mode="json")
):
raise KnowledgeFSCleanupConflictError("Cleanup request ID was reused with different evidence")
@staticmethod
def _assert_approval_replay(
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupApprovalInput,
) -> None:
if (
authorization.plan_digest != payload.plan_digest
or authorization.approved_by_account_id != str(payload.approved_by_account_id)
or authorization.approved_at != ensure_naive_utc(payload.approved_at)
or authorization.approval_expires_at != ensure_naive_utc(payload.approval_expires_at)
):
raise KnowledgeFSCleanupConflictError("Cleanup approval replay does not match persisted approval")
@staticmethod
def _assert_start_replay(
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupStartInput,
) -> None:
if (
authorization.plan_digest != payload.plan_digest
or authorization.started_by_account_id != str(payload.started_by_account_id)
or authorization.started_at != ensure_naive_utc(payload.started_at)
):
raise KnowledgeFSCleanupConflictError("Cleanup start replay does not match persisted fence")
@staticmethod
def _assert_completion_replay(
authorization: KnowledgeFSCleanupAuthorization,
payload: CleanupCompletionEvidenceInput,
) -> None:
if (
authorization.plan_digest != payload.plan_digest
or authorization.completed_by_account_id != str(payload.completed_by_account_id)
or authorization.completed_at != ensure_naive_utc(payload.completed_at)
or authorization.completion_evidence != payload.model_dump(mode="json")
):
raise KnowledgeFSCleanupConflictError("Cleanup completion replay does not match persisted evidence")
def _iso(value: datetime | None) -> str | None:
return f"{value.isoformat()}Z" if value is not None else None
__all__ = [
"CleanupApprovalInput",
"CleanupApprovalReport",
"CleanupCompletionEvidenceInput",
"CleanupCompletionReport",
"CleanupReadinessEvidenceInput",
"CleanupReadinessReport",
"CleanupStartInput",
"CleanupStartReport",
"KnowledgeFSCleanupConflictError",
"KnowledgeFSCleanupError",
"KnowledgeFSCleanupGateBlockedError",
"KnowledgeFSCleanupNotFoundError",
"KnowledgeFSCleanupService",
"cleanup_workspace_cohort_digest",
]
@@ -0,0 +1,430 @@
"""Dify-owned KnowledgeFS members, visibility, and external-access mutations."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import naive_utc_now
from models import TenantAccountJoin
from models.knowledge_fs import (
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpacePermissionStatus,
KnowledgeFSControlSpaceVisibility,
KnowledgeFSExternalAccessPolicy,
)
from services.knowledge_fs.product_dto import (
KnowledgeFSExternalAccessPayload,
KnowledgeFSExternalAccessResponse,
KnowledgeFSMemberBindingPayload,
KnowledgeFSPermissionListResponse,
KnowledgeFSPermissionResponse,
)
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
from services.knowledge_fs.product_service import KnowledgeFSProductService
from services.knowledge_fs.revocation_commands import (
KnowledgeFSRevocationCommandPort,
KnowledgeFSRevocationCommandProducer,
)
class KnowledgeFSControlPlaneInvariantError(RuntimeError):
"""Required authorization revision state is absent or inconsistent."""
class KnowledgeFSWorkspaceMemberPort(Protocol):
def are_active_members(self, *, session: Session, tenant_id: str, account_ids: Sequence[str]) -> bool: ...
class SQLKnowledgeFSWorkspaceMemberPort:
def are_active_members(self, *, session: Session, tenant_id: str, account_ids: Sequence[str]) -> bool:
unique_ids = frozenset(account_ids)
if not unique_ids:
return True
found = frozenset(
session.scalars(
sa.select(TenantAccountJoin.account_id).where(
TenantAccountJoin.tenant_id == tenant_id,
TenantAccountJoin.account_id.in_(unique_ids),
)
)
)
return found == unique_ids
class KnowledgeFSControlPlaneService:
"""Mutate Dify authorization state after product authorization succeeds."""
def __init__(
self,
session_maker: sessionmaker[Session],
*,
product: KnowledgeFSProductService,
members: KnowledgeFSWorkspaceMemberPort,
revocations: KnowledgeFSRevocationCommandPort | None = None,
) -> None:
self._session_maker = session_maker
self._product = product
self._members = members
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
def list_permissions(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
) -> KnowledgeFSPermissionListResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
with self._session_maker() as session:
permissions = tuple(
session.scalars(
sa.select(KnowledgeFSControlSpacePermission)
.where(
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
)
.order_by(KnowledgeFSControlSpacePermission.account_id)
)
)
return KnowledgeFSPermissionListResponse(
data=[
KnowledgeFSPermissionResponse(
account_id=permission.account_id,
role=permission.role,
status=permission.status.value,
revision=permission.revision,
)
for permission in permissions
]
)
def replace_members(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
members: Sequence[KnowledgeFSMemberBindingPayload],
) -> KnowledgeFSPermissionListResponse:
authorized = self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
desired = {member.account_id: member.role for member in members}
if len(desired) != len(members) or authorized.control_space.owner_account_id in desired:
raise KnowledgeFSControlPlaneInvariantError("Member bindings must be unique and exclude the owner")
with self._session_maker.begin() as session:
control_space = session.scalar(
sa.select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
)
.with_for_update()
)
if control_space is None:
raise KnowledgeFSControlPlaneInvariantError("Control-space disappeared during member update")
if not self._members.are_active_members(
session=session,
tenant_id=tenant_id,
account_ids=tuple(desired),
):
raise KnowledgeFSControlPlaneInvariantError("Every KnowledgeFS member must belong to the workspace")
existing = {
permission.account_id: permission
for permission in session.scalars(
sa.select(KnowledgeFSControlSpacePermission).where(
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
KnowledgeFSControlSpacePermission.account_id != control_space.owner_account_id,
)
)
}
now = naive_utc_now()
revoke_reasons_by_account: dict[str, str] = {}
for account_id, permission in existing.items():
role = desired.pop(account_id, None)
if role is None:
if permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE:
permission.status = KnowledgeFSControlSpacePermissionStatus.REVOKED
permission.revision += 1
permission.revoked_at = now
permission.revoked_by_account_id = actor_account_id
revoke_reasons_by_account[account_id] = "permission_revoked"
elif (
permission.role is not role
or permission.status is not KnowledgeFSControlSpacePermissionStatus.ACTIVE
):
previous_role = permission.role
was_active = permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE
permission.role = role
permission.status = KnowledgeFSControlSpacePermissionStatus.ACTIVE
permission.revision += 1
permission.revoked_at = None
permission.revoked_by_account_id = None
if was_active and _permission_role_is_narrower(previous_role, role):
revoke_reasons_by_account[account_id] = "permission_role_narrowed"
for account_id, role in desired.items():
session.add(
KnowledgeFSControlSpacePermission(
tenant_id=tenant_id,
control_space_id=control_space_id,
account_id=account_id,
role=role,
granted_by_account_id=actor_account_id,
)
)
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
revision.space_acl_epoch += 1
for account_id, reason_code in revoke_reasons_by_account.items():
self._revocations.enqueue_principal_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
subject=f"dify-account:{account_id}",
reason_code=reason_code,
caller_kinds=("interactive",),
)
return self.list_permissions(
tenant_id=tenant_id,
actor_account_id=actor_account_id,
control_space_id=control_space_id,
)
def update_visibility(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
visibility: KnowledgeFSControlSpaceVisibility,
) -> None:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
with self._session_maker.begin() as session:
control_space = session.scalar(
sa.select(KnowledgeFSControlSpace)
.where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.id == control_space_id,
)
.with_for_update()
)
if control_space is None:
raise KnowledgeFSControlPlaneInvariantError("Control-space disappeared during visibility update")
if control_space.visibility is visibility:
return
previous_visibility = control_space.visibility
control_space.visibility = visibility
control_space.resource_version += 1
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
revision.space_acl_epoch += 1
if _visibility_is_narrower(previous_visibility, visibility):
active_member_subjects = {
f"dify-account:{account_id}"
for account_id in session.scalars(
sa.select(KnowledgeFSControlSpacePermission.account_id).where(
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
KnowledgeFSControlSpacePermission.account_id != control_space.owner_account_id,
)
)
}
if previous_visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS:
allowed_subjects = {f"dify-account:{control_space.owner_account_id}"}
if visibility is KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS:
allowed_subjects.update(active_member_subjects)
self._revocations.enqueue_control_space_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
reason_code="visibility_narrowed",
caller_kinds=("interactive",),
excluded_subjects=tuple(sorted(allowed_subjects)),
)
else:
for subject in sorted(active_member_subjects):
self._revocations.enqueue_principal_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
subject=subject,
reason_code="visibility_narrowed",
caller_kinds=("interactive",),
)
def get_external_access(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
) -> KnowledgeFSExternalAccessResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
with self._session_maker() as session:
policy = session.scalar(
sa.select(KnowledgeFSExternalAccessPolicy).where(
KnowledgeFSExternalAccessPolicy.tenant_id == tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == control_space_id,
)
)
return _external_access_response(policy)
def update_external_access(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
payload: KnowledgeFSExternalAccessPayload,
) -> KnowledgeFSExternalAccessResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
)
with self._session_maker.begin() as session:
policy = session.scalar(
sa.select(KnowledgeFSExternalAccessPolicy)
.where(
KnowledgeFSExternalAccessPolicy.tenant_id == tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == control_space_id,
)
.with_for_update()
)
if policy is None:
policy = KnowledgeFSExternalAccessPolicy(
tenant_id=tenant_id,
control_space_id=control_space_id,
)
session.add(policy)
session.flush()
previous_channels = {
"service": policy.service_api_enabled,
"agent": policy.agent_enabled,
"workflow": policy.workflow_enabled,
"mcp": policy.mcp_enabled,
}
requested_channels = {
"service": payload.service_api_enabled,
"agent": payload.agent_enabled,
"workflow": payload.workflow_enabled,
"mcp": payload.mcp_enabled,
}
if previous_channels == requested_channels:
return _external_access_response(policy)
policy.service_api_enabled = payload.service_api_enabled
policy.agent_enabled = payload.agent_enabled
policy.workflow_enabled = payload.workflow_enabled
policy.mcp_enabled = payload.mcp_enabled
policy.revision += 1
policy.updated_by_account_id = actor_account_id
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
revision.external_access_epoch += 1
disabled_callers = tuple(
caller_kind
for caller_kind, was_enabled in previous_channels.items()
if was_enabled and not requested_channels[caller_kind]
)
if disabled_callers:
self._revocations.enqueue_control_space_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
reason_code="external_access_revoked",
caller_kinds=disabled_callers,
)
session.flush()
response = _external_access_response(policy)
return response
def _permission_role_is_narrower(
previous: KnowledgeFSControlSpacePermissionRole,
current: KnowledgeFSControlSpacePermissionRole,
) -> bool:
rank = {
KnowledgeFSControlSpacePermissionRole.VIEWER: 0,
KnowledgeFSControlSpacePermissionRole.EDITOR: 1,
KnowledgeFSControlSpacePermissionRole.OWNER: 2,
}
return rank[current] < rank[previous]
def _authorization_revision(
session: Session,
*,
tenant_id: str,
control_space_id: str,
) -> KnowledgeFSAuthorizationRevision:
revision = session.scalar(
sa.select(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
)
.with_for_update()
)
if revision is None:
raise KnowledgeFSControlPlaneInvariantError("KnowledgeFS authorization revision is missing")
return revision
def _visibility_is_narrower(
previous: KnowledgeFSControlSpaceVisibility,
requested: KnowledgeFSControlSpaceVisibility,
) -> bool:
return (
previous is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS
and requested
in {
KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS,
KnowledgeFSControlSpaceVisibility.ONLY_ME,
}
) or (
previous is KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS
and requested is KnowledgeFSControlSpaceVisibility.ONLY_ME
)
def _external_access_response(policy: KnowledgeFSExternalAccessPolicy | None) -> KnowledgeFSExternalAccessResponse:
return KnowledgeFSExternalAccessResponse(
service_api_enabled=policy.service_api_enabled if policy else False,
agent_enabled=policy.agent_enabled if policy else False,
workflow_enabled=policy.workflow_enabled if policy else False,
mcp_enabled=policy.mcp_enabled if policy else False,
revision=policy.revision if policy else 0,
)
__all__ = [
"KnowledgeFSControlPlaneInvariantError",
"KnowledgeFSControlPlaneService",
"KnowledgeFSWorkspaceMemberPort",
"SQLKnowledgeFSWorkspaceMemberPort",
]
@@ -0,0 +1,318 @@
"""Durable product entrypoints for KnowledgeFS provision and deletion intent."""
from __future__ import annotations
import uuid
from typing import NamedTuple
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import (
AppKnowledgeFSSpaceJoin,
KnowledgeFSApiCredential,
KnowledgeFSAuthorizationRevision,
KnowledgeFSCapabilityIssuanceAudit,
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpaceState,
KnowledgeFSDeleteCommandPayload,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
KnowledgeFSModelSelectionIntentPayload,
KnowledgeFSProvisionCommandPayload,
KnowledgeFSRetrievalProfileIntentPayload,
)
from repositories.knowledge_fs_control_space_repository import KnowledgeFSControlSpaceCASUpdate
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
SQLAlchemyKnowledgeFSControlSpaceRepository,
)
from repositories.sqlalchemy_knowledge_fs_lifecycle_outbox_repository import (
SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
)
from services.knowledge_fs.control_space_lifecycle import (
KnowledgeFSControlSpaceLifecycleService,
KnowledgeFSControlSpaceNotFoundError,
KnowledgeFSControlSpaceVersionConflictError,
KnowledgeFSWorkspaceDeletionBlockedError,
)
class KnowledgeFSProvisionIntent(NamedTuple):
tenant_id: str
owner_account_id: str
provisioning_key: str
operation_id: str
idempotency_key: str
name: str
slug: str
icon: str | None
description: str | None
model_intent: KnowledgeFSModelSelectionIntentPayload
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
class KnowledgeFSProvisionIntentResult(NamedTuple):
control_space: KnowledgeFSControlSpace
outbox: KnowledgeFSLifecycleOutbox
class KnowledgeFSDeletionIntentResult(NamedTuple):
control_space: KnowledgeFSControlSpace
outbox: KnowledgeFSLifecycleOutbox | None
class KnowledgeFSControlSpaceIntentConflictError(RuntimeError):
"""An idempotency identity was reused for a different lifecycle intent."""
class KnowledgeFSControlSpaceCommandService:
"""Write control state and its outbox command in one database transaction."""
def __init__(self, session_maker: sessionmaker[Session]):
self._session_maker = session_maker
def create_provision_intent(self, intent: KnowledgeFSProvisionIntent) -> KnowledgeFSProvisionIntentResult:
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
existing = control_repository.find_by_provisioning_key(provisioning_key=intent.provisioning_key)
if existing is not None:
command = outbox_repository.get_by_operation_id(
tenant_id=intent.tenant_id,
operation_id=intent.operation_id,
)
if (
existing.tenant_id != intent.tenant_id
or existing.owner_account_id != intent.owner_account_id
or command is None
or command.control_space_id != existing.id
or command.idempotency_key != intent.idempotency_key
):
raise KnowledgeFSControlSpaceIntentConflictError(
"KnowledgeFS provisioning key was reused for a different intent"
)
return KnowledgeFSProvisionIntentResult(existing, command)
control_space = control_repository.add(
KnowledgeFSControlSpace(
tenant_id=intent.tenant_id,
owner_account_id=intent.owner_account_id,
provisioning_key=intent.provisioning_key,
lifecycle_operation_id=intent.operation_id,
)
)
session.add(
KnowledgeFSAuthorizationRevision(
tenant_id=intent.tenant_id,
control_space_id=control_space.id,
)
)
session.add(
KnowledgeFSControlSpacePermission(
tenant_id=intent.tenant_id,
control_space_id=control_space.id,
account_id=intent.owner_account_id,
role=KnowledgeFSControlSpacePermissionRole.OWNER,
granted_by_account_id=intent.owner_account_id,
)
)
payload = KnowledgeFSProvisionCommandPayload(
schema_version=1,
idempotency_key=intent.idempotency_key,
expected_revision=0,
provisioning_key=intent.provisioning_key,
name=intent.name,
icon=intent.icon,
description=intent.description,
slug=intent.slug,
model_intent=KnowledgeFSModelSelectionIntentPayload(**intent.model_intent),
profile_intent=KnowledgeFSRetrievalProfileIntentPayload(**intent.profile_intent),
)
command = outbox_repository.add(
KnowledgeFSLifecycleOutbox(
tenant_id=intent.tenant_id,
control_space_id=control_space.id,
operation_id=intent.operation_id,
idempotency_key=intent.idempotency_key,
operation=KnowledgeFSLifecycleOperation.PROVISION,
command_payload=payload,
expected_control_space_version=control_space.resource_version,
expected_knowledge_space_revision=0,
)
)
return KnowledgeFSProvisionIntentResult(control_space, command)
def request_deletion(
self,
*,
tenant_id: str,
control_space_id: str,
operation_id: str,
idempotency_key: str,
) -> KnowledgeFSDeletionIntentResult:
"""Use the same durable path for single, batch, and workspace cleanup."""
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
control_space = control_repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
if control_space is None:
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space was not found in this tenant")
if control_space.state is KnowledgeFSControlSpaceState.DELETED:
return KnowledgeFSDeletionIntentResult(control_space, None)
replay = outbox_repository.get_by_operation_id(tenant_id=tenant_id, operation_id=operation_id)
if replay is not None:
if replay.control_space_id != control_space_id or replay.idempotency_key != idempotency_key:
raise KnowledgeFSControlSpaceIntentConflictError(
"KnowledgeFS deletion operation was reused for a different intent"
)
if (
replay.operation is KnowledgeFSLifecycleOperation.DELETE
and replay.status is KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER
):
if not outbox_repository.reactivate_dead_letter(outbox_id=replay.id):
raise KnowledgeFSControlSpaceVersionConflictError(
"KnowledgeFS permanent cleanup command changed before reactivation"
)
replay = outbox_repository.get(outbox_id=replay.id)
if replay is None:
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS cleanup command disappeared")
return KnowledgeFSDeletionIntentResult(control_space, replay)
open_command = outbox_repository.find_open_for_control_space(
tenant_id=tenant_id,
control_space_id=control_space_id,
operation=KnowledgeFSLifecycleOperation.DELETE,
)
if open_command is not None:
return KnowledgeFSDeletionIntentResult(control_space, open_command)
if control_space.state is KnowledgeFSControlSpaceState.DELETING:
changed = control_repository.compare_and_set_lifecycle(
KnowledgeFSControlSpaceCASUpdate(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=control_space.resource_version,
expected_state=KnowledgeFSControlSpaceState.DELETING,
new_state=KnowledgeFSControlSpaceState.DELETING,
lifecycle_operation_id=operation_id,
)
)
if not changed:
raise KnowledgeFSControlSpaceVersionConflictError(
"KnowledgeFS control-space changed while restoring permanent cleanup"
)
transitioned = control_repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
if transitioned is None:
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space disappeared")
else:
transitioned = KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=control_space.resource_version,
new_state=KnowledgeFSControlSpaceState.DELETING,
lifecycle_operation_id=operation_id,
)
payload = KnowledgeFSDeleteCommandPayload(
schema_version=1,
idempotency_key=idempotency_key,
expected_revision=transitioned.knowledge_space_revision,
knowledge_space_id=transitioned.knowledge_space_id,
provisioning_key=transitioned.provisioning_key,
)
command = outbox_repository.add(
KnowledgeFSLifecycleOutbox(
tenant_id=tenant_id,
control_space_id=control_space_id,
operation_id=operation_id,
idempotency_key=idempotency_key,
operation=KnowledgeFSLifecycleOperation.DELETE,
command_payload=payload,
expected_control_space_version=transitioned.resource_version,
expected_knowledge_space_revision=transitioned.knowledge_space_revision,
)
)
return KnowledgeFSDeletionIntentResult(transitioned, command)
def request_workspace_cleanup(self, *, tenant_id: str) -> tuple[KnowledgeFSDeletionIntentResult, ...]:
"""Enumerate independent control-spaces; never hard-delete them with the workspace."""
with self._session_maker() as session:
control_spaces = SQLAlchemyKnowledgeFSControlSpaceRepository(session).list_for_tenant(tenant_id=tenant_id)
control_space_ids = tuple(
control_space.id
for control_space in control_spaces
if control_space.state is not KnowledgeFSControlSpaceState.DELETED
)
commands: list[KnowledgeFSDeletionIntentResult] = []
for control_space_id in control_space_ids:
operation_id = str(
uuid.uuid5(uuid.NAMESPACE_URL, f"dify-kfs-workspace-cleanup:{tenant_id}:{control_space_id}")
)
commands.append(
self.request_deletion(
tenant_id=tenant_id,
control_space_id=control_space_id,
operation_id=operation_id,
idempotency_key=f"workspace-cleanup:{tenant_id}:{control_space_id}",
)
)
return tuple(commands)
def assert_workspace_deletion_allowed(self, *, tenant_id: str) -> None:
with self._session_maker() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
KnowledgeFSControlSpaceLifecycleService(repository).assert_workspace_deletion_allowed(tenant_id=tenant_id)
def finalize_workspace_deletion(self, *, tenant_id: str) -> int:
"""Remove local control-plane rows only after every remote Space is durably deleted.
The control-space Workspace FK remains ``RESTRICT`` until this explicit finalizer
succeeds, so a caller cannot hard-delete the Workspace and strand remote resources.
"""
with self._session_maker.begin() as session:
control_spaces = tuple(
session.scalars(
sa.select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
.with_for_update()
)
)
blocking_ids = tuple(
control_space.id
for control_space in control_spaces
if control_space.state is not KnowledgeFSControlSpaceState.DELETED
)
if blocking_ids:
raise KnowledgeFSWorkspaceDeletionBlockedError(blocking_ids)
for model in (
KnowledgeFSLifecycleOutbox,
KnowledgeFSCapabilityIssuanceAudit,
KnowledgeFSCapabilityIssuanceReservation,
KnowledgeFSAuthorizationRevision,
AppKnowledgeFSSpaceJoin,
KnowledgeFSApiCredential,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSControlSpacePermission,
):
session.execute(sa.delete(model).where(model.tenant_id == tenant_id))
session.execute(sa.delete(KnowledgeFSControlSpace).where(KnowledgeFSControlSpace.tenant_id == tenant_id))
return len(control_spaces)
__all__ = [
"KnowledgeFSControlSpaceCommandService",
"KnowledgeFSControlSpaceIntentConflictError",
"KnowledgeFSDeletionIntentResult",
"KnowledgeFSProvisionIntent",
"KnowledgeFSProvisionIntentResult",
]
@@ -0,0 +1,215 @@
"""Local CAS lifecycle rules for KnowledgeFS control-spaces.
P1A persists lifecycle intent only. This service deliberately has no
KnowledgeFS client or task dispatcher, so deploying it cannot emit P1B remote
traffic.
"""
from __future__ import annotations
import logging
from collections.abc import Mapping
from datetime import datetime
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
from repositories.knowledge_fs_control_space_repository import (
KnowledgeFSControlSpaceCASUpdate,
KnowledgeFSControlSpaceRepository,
)
from services.knowledge_fs.observability import (
KnowledgeFSControlSpaceStateMetric,
KnowledgeFSOperationalMetricsPort,
get_knowledge_fs_operational_metrics,
)
logger = logging.getLogger(__name__)
class KnowledgeFSControlSpaceLifecycleError(RuntimeError):
"""Base error for a rejected local control-space lifecycle operation."""
class KnowledgeFSControlSpaceNotFoundError(KnowledgeFSControlSpaceLifecycleError):
pass
class KnowledgeFSControlSpaceVersionConflictError(KnowledgeFSControlSpaceLifecycleError):
pass
class KnowledgeFSInvalidLifecycleTransitionError(KnowledgeFSControlSpaceLifecycleError):
pass
class KnowledgeFSSpaceRegistrationConflictError(KnowledgeFSControlSpaceLifecycleError):
pass
class KnowledgeFSWorkspaceDeletionBlockedError(KnowledgeFSControlSpaceLifecycleError):
def __init__(self, control_space_ids: tuple[str, ...]):
self.control_space_ids = control_space_ids
super().__init__("Workspace deletion is blocked by non-terminal KnowledgeFS control-spaces")
_ALLOWED_TRANSITIONS: Mapping[KnowledgeFSControlSpaceState, frozenset[KnowledgeFSControlSpaceState]] = {
KnowledgeFSControlSpaceState.PROVISIONING: frozenset(
{
KnowledgeFSControlSpaceState.ACTIVE,
KnowledgeFSControlSpaceState.DELETING,
KnowledgeFSControlSpaceState.ERROR,
}
),
KnowledgeFSControlSpaceState.ACTIVE: frozenset(
{KnowledgeFSControlSpaceState.DELETING, KnowledgeFSControlSpaceState.ERROR}
),
KnowledgeFSControlSpaceState.DELETING: frozenset(
{KnowledgeFSControlSpaceState.DELETED, KnowledgeFSControlSpaceState.ERROR}
),
KnowledgeFSControlSpaceState.ERROR: frozenset(
{
KnowledgeFSControlSpaceState.PROVISIONING,
KnowledgeFSControlSpaceState.ACTIVE,
KnowledgeFSControlSpaceState.DELETING,
}
),
KnowledgeFSControlSpaceState.DELETED: frozenset(),
}
class KnowledgeFSControlSpaceLifecycleService:
"""Validate lifecycle invariants and persist transitions through a CAS."""
def __init__(
self,
repository: KnowledgeFSControlSpaceRepository,
*,
metrics: KnowledgeFSOperationalMetricsPort | None = None,
) -> None:
self._repository = repository
self._metrics = metrics or get_knowledge_fs_operational_metrics()
def transition(
self,
*,
tenant_id: str,
control_space_id: str,
expected_resource_version: int,
new_state: KnowledgeFSControlSpaceState,
lifecycle_operation_id: str,
knowledge_space_id: str | None = None,
knowledge_space_revision: int | None = None,
last_error_code: str | None = None,
last_error_message: str | None = None,
) -> KnowledgeFSControlSpace:
control_space = self._repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
if control_space is None:
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space was not found in this tenant")
if control_space.resource_version != expected_resource_version:
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS control-space resource version changed")
if control_space.deletion_irreversible_at is not None and new_state in {
KnowledgeFSControlSpaceState.ACTIVE,
KnowledgeFSControlSpaceState.PROVISIONING,
}:
raise KnowledgeFSInvalidLifecycleTransitionError(
"KnowledgeFS control-space deletion is irreversible and cannot be recovered"
)
if new_state not in _ALLOWED_TRANSITIONS[control_space.state]:
raise KnowledgeFSInvalidLifecycleTransitionError(
f"Cannot transition KnowledgeFS control-space from {control_space.state} to {new_state}"
)
registered_space_id = control_space.knowledge_space_id
if registered_space_id is not None and knowledge_space_id not in {None, registered_space_id}:
raise KnowledgeFSSpaceRegistrationConflictError("KnowledgeFS Space registration is immutable")
effective_space_id = knowledge_space_id or registered_space_id
if new_state is KnowledgeFSControlSpaceState.ACTIVE and effective_space_id is None:
raise KnowledgeFSSpaceRegistrationConflictError("An active control-space must have a KnowledgeFS Space ID")
if knowledge_space_revision is not None and knowledge_space_revision < control_space.knowledge_space_revision:
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS Space revision moved backwards")
previous_state = control_space.state
previous_updated_at = control_space.updated_at
changed = self._repository.compare_and_set_lifecycle(
KnowledgeFSControlSpaceCASUpdate(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=expected_resource_version,
expected_state=control_space.state,
new_state=new_state,
lifecycle_operation_id=lifecycle_operation_id,
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
last_error_code=last_error_code,
last_error_message=last_error_message,
)
)
if not changed:
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS control-space changed during transition")
transitioned = self._repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
if transitioned is None:
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space disappeared after transition")
self._record_state_metric(
previous_state=previous_state,
previous_updated_at=previous_updated_at,
transitioned=transitioned,
)
return transitioned
def _record_state_metric(
self,
*,
previous_state: KnowledgeFSControlSpaceState,
previous_updated_at: datetime,
transitioned: KnowledgeFSControlSpace,
) -> None:
try:
duration_seconds = max(0.0, (transitioned.updated_at - previous_updated_at).total_seconds())
self._metrics.record_control_space_state(
KnowledgeFSControlSpaceStateMetric(
duration_seconds,
previous_state.value,
transitioned.state.value,
)
)
except Exception:
logger.warning("KnowledgeFS control-space state metric export failed", exc_info=True)
def mark_deletion_irreversible(
self,
*,
tenant_id: str,
control_space_id: str,
lifecycle_operation_id: str,
irreversible_at: datetime,
) -> None:
"""Persist the monotonic point after which product recovery is forbidden."""
changed = self._repository.mark_deletion_irreversible(
tenant_id=tenant_id,
control_space_id=control_space_id,
lifecycle_operation_id=lifecycle_operation_id,
irreversible_at=irreversible_at,
)
if not changed:
raise KnowledgeFSControlSpaceVersionConflictError(
"KnowledgeFS control-space changed before deletion became irreversible"
)
def assert_workspace_deletion_allowed(self, *, tenant_id: str) -> None:
"""Fence tenant deletion until every control-space reaches ``deleted``."""
blocking_ids = self._repository.list_workspace_deletion_blockers(tenant_id=tenant_id)
if blocking_ids:
raise KnowledgeFSWorkspaceDeletionBlockedError(blocking_ids)
__all__ = [
"KnowledgeFSControlSpaceLifecycleError",
"KnowledgeFSControlSpaceLifecycleService",
"KnowledgeFSControlSpaceNotFoundError",
"KnowledgeFSControlSpaceVersionConflictError",
"KnowledgeFSInvalidLifecycleTransitionError",
"KnowledgeFSSpaceRegistrationConflictError",
"KnowledgeFSWorkspaceDeletionBlockedError",
]
@@ -0,0 +1,174 @@
"""Operator-only registration, backfill, repair, and dry-run helpers."""
from __future__ import annotations
from collections import Counter
from collections.abc import Iterable
from typing import NamedTuple
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import (
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpaceState,
)
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
SQLAlchemyKnowledgeFSControlSpaceRepository,
)
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceIntentConflictError
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
class KnowledgeFSControlSpaceRegistration(NamedTuple):
tenant_id: str
owner_account_id: str
provisioning_key: str
knowledge_space_id: str
knowledge_space_revision: int
class KnowledgeFSControlSpaceDryRunReport(NamedTuple):
total: int
by_state: dict[str, int]
irreversible_deletions: int
class KnowledgeFSControlSpaceBackfillReport(NamedTuple):
candidates: int
registered: int
replays: int
class KnowledgeFSControlSpaceManagementService:
def __init__(self, session_maker: sessionmaker[Session]):
self._session_maker = session_maker
def dry_run(self, *, tenant_id: str | None = None) -> KnowledgeFSControlSpaceDryRunReport:
with self._session_maker() as session:
if tenant_id is None:
from sqlalchemy import select
control_spaces = tuple(session.scalars(select(KnowledgeFSControlSpace)))
else:
control_spaces = SQLAlchemyKnowledgeFSControlSpaceRepository(session).list_for_tenant(
tenant_id=tenant_id
)
counts = Counter(control_space.state.value for control_space in control_spaces)
return KnowledgeFSControlSpaceDryRunReport(
total=len(control_spaces),
by_state=dict(sorted(counts.items())),
irreversible_deletions=sum(
control_space.deletion_irreversible_at is not None for control_space in control_spaces
),
)
def register(self, registration: KnowledgeFSControlSpaceRegistration) -> tuple[KnowledgeFSControlSpace, bool]:
with self._session_maker.begin() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
by_key = repository.find_by_provisioning_key(provisioning_key=registration.provisioning_key)
by_space = repository.find_by_knowledge_space_id(
tenant_id=registration.tenant_id,
knowledge_space_id=registration.knowledge_space_id,
)
existing = by_key or by_space
if existing is not None:
if (
existing.tenant_id != registration.tenant_id
or existing.owner_account_id != registration.owner_account_id
or existing.provisioning_key != registration.provisioning_key
or existing.knowledge_space_id != registration.knowledge_space_id
):
raise KnowledgeFSControlSpaceIntentConflictError(
"KnowledgeFS registration conflicts with an existing local identity"
)
self._ensure_owner_permission(session, control_space=existing)
return existing, True
control_space = repository.add(
KnowledgeFSControlSpace(
tenant_id=registration.tenant_id,
owner_account_id=registration.owner_account_id,
provisioning_key=registration.provisioning_key,
knowledge_space_id=registration.knowledge_space_id,
knowledge_space_revision=registration.knowledge_space_revision,
state=KnowledgeFSControlSpaceState.ACTIVE,
lifecycle_operation_id="management-register",
)
)
session.add(
KnowledgeFSAuthorizationRevision(
tenant_id=registration.tenant_id,
control_space_id=control_space.id,
)
)
self._ensure_owner_permission(session, control_space=control_space)
return control_space, False
@staticmethod
def _ensure_owner_permission(session: Session, *, control_space: KnowledgeFSControlSpace) -> None:
permission = session.scalar(
sa.select(KnowledgeFSControlSpacePermission).where(
KnowledgeFSControlSpacePermission.tenant_id == control_space.tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == control_space.id,
KnowledgeFSControlSpacePermission.account_id == control_space.owner_account_id,
)
)
if permission is None:
session.add(
KnowledgeFSControlSpacePermission(
tenant_id=control_space.tenant_id,
control_space_id=control_space.id,
account_id=control_space.owner_account_id,
role=KnowledgeFSControlSpacePermissionRole.OWNER,
granted_by_account_id=control_space.owner_account_id,
)
)
def backfill(
self,
registrations: Iterable[KnowledgeFSControlSpaceRegistration],
*,
apply: bool,
) -> KnowledgeFSControlSpaceBackfillReport:
candidates = tuple(registrations)
if not apply:
return KnowledgeFSControlSpaceBackfillReport(len(candidates), 0, 0)
registered = 0
replays = 0
for registration in candidates:
_, replayed = self.register(registration)
registered += not replayed
replays += replayed
return KnowledgeFSControlSpaceBackfillReport(len(candidates), registered, replays)
def repair_registration(
self,
*,
tenant_id: str,
control_space_id: str,
expected_resource_version: int,
knowledge_space_id: str,
knowledge_space_revision: int,
) -> KnowledgeFSControlSpace:
with self._session_maker.begin() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
return KnowledgeFSControlSpaceLifecycleService(repository).transition(
tenant_id=tenant_id,
control_space_id=control_space_id,
expected_resource_version=expected_resource_version,
new_state=KnowledgeFSControlSpaceState.ACTIVE,
lifecycle_operation_id="management-repair",
knowledge_space_id=knowledge_space_id,
knowledge_space_revision=knowledge_space_revision,
)
__all__ = [
"KnowledgeFSControlSpaceBackfillReport",
"KnowledgeFSControlSpaceDryRunReport",
"KnowledgeFSControlSpaceManagementService",
"KnowledgeFSControlSpaceRegistration",
]
@@ -0,0 +1,379 @@
"""Resource-bound KnowledgeFS service credentials and fail-closed validation."""
from __future__ import annotations
import hashlib
import secrets
from collections.abc import Sequence
from datetime import UTC, datetime
from typing import NamedTuple, Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from libs.datetime_utils import naive_utc_now
from models.knowledge_fs import (
KnowledgeFSApiCredential,
KnowledgeFSApiCredentialStatus,
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpaceState,
KnowledgeFSExternalAccessPolicy,
)
from services.knowledge_fs.product_dto import (
KnowledgeFSCredentialCreatePayload,
KnowledgeFSCredentialCreateResponse,
KnowledgeFSCredentialItemResponse,
KnowledgeFSCredentialListResponse,
)
from services.knowledge_fs.product_operations import (
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
KnowledgeFSProductPermission,
is_product_operation_ready,
)
from services.knowledge_fs.product_service import KnowledgeFSProductService
from services.knowledge_fs.revocation_commands import (
KnowledgeFSRevocationCommandPort,
KnowledgeFSRevocationCommandProducer,
)
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
_SECRET_PREFIX = "kfs_"
_DISPLAY_PREFIX_LENGTH = 12
class KnowledgeFSCredentialError(RuntimeError):
"""Base class for stable credential failures."""
class KnowledgeFSCredentialValidationError(KnowledgeFSCredentialError):
"""A bearer credential is invalid, disabled, expired, or out of scope."""
class KnowledgeFSCredentialPolicyError(KnowledgeFSCredentialError):
"""A credential management request violates the product policy."""
class KnowledgeFSServiceCredentialProfile(NamedTuple):
tenant_id: str
control_space_id: str
credential_id: str
principal_id: str
allowed_actions: frozenset[str]
knowledge_space_id: str
knowledge_space_revision: int
membership_epoch: int
space_acl_epoch: int
external_access_epoch: int
content_policy_revision: int
credential_revision: int
expires_at: datetime | None
class KnowledgeFSCredentialProfileCache(Protocol):
def get(self, credential_hash: str) -> KnowledgeFSServiceCredentialProfile | None: ...
def set(self, credential_hash: str, profile: KnowledgeFSServiceCredentialProfile) -> None: ...
def invalidate(self, credential_hash: str) -> None: ...
class NullKnowledgeFSCredentialProfileCache:
"""Safe default: no cross-request authorization state is retained."""
def get(self, credential_hash: str) -> KnowledgeFSServiceCredentialProfile | None:
_ = credential_hash
return None
def set(self, credential_hash: str, profile: KnowledgeFSServiceCredentialProfile) -> None:
_ = (credential_hash, profile)
def invalidate(self, credential_hash: str) -> None:
_ = credential_hash
def knowledge_fs_service_credential_actions() -> frozenset[str]:
"""Return only actions backed by an exact P2 product-operation mapping."""
actions: set[str] = set()
for product_operation_id, product_operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
if not is_product_operation_ready(product_operation_id):
continue
capability_operation_id = product_operation.capability_operation_id
if capability_operation_id is None:
continue
actions.add(KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id].action)
return frozenset(actions)
class KnowledgeFSCredentialService:
def __init__(
self,
session_maker: sessionmaker[Session],
*,
product: KnowledgeFSProductService,
cache: KnowledgeFSCredentialProfileCache | None = None,
revocations: KnowledgeFSRevocationCommandPort | None = None,
) -> None:
self._session_maker = session_maker
self._product = product
self._cache = cache or NullKnowledgeFSCredentialProfileCache()
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
def create(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
payload: KnowledgeFSCredentialCreatePayload,
) -> KnowledgeFSCredentialCreateResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
require_active=True,
)
allowed_actions = _validate_allowed_actions(payload.allowed_actions)
expires_at = _normalize_expiry(payload.expires_at)
if expires_at is not None and expires_at <= naive_utc_now():
raise KnowledgeFSCredentialPolicyError("Credential expiry must be in the future")
raw_credential = f"{_SECRET_PREFIX}{secrets.token_urlsafe(32)}"
credential_hash = hash_knowledge_fs_credential(raw_credential)
credential = KnowledgeFSApiCredential(
tenant_id=tenant_id,
control_space_id=control_space_id,
credential_hash=credential_hash,
credential_prefix=raw_credential[:_DISPLAY_PREFIX_LENGTH],
credential_last4=raw_credential[-4:],
principal="pending",
allowed_actions=list(allowed_actions),
expires_at=expires_at,
created_by_account_id=actor_account_id,
)
credential.principal = credential.id
with self._session_maker.begin() as session:
session.add(credential)
return KnowledgeFSCredentialCreateResponse(
id=credential.id,
credential=raw_credential,
credential_prefix=credential.credential_prefix,
credential_last4=credential.credential_last4,
principal=credential.principal,
allowed_actions=list(allowed_actions),
expires_at=credential.expires_at,
)
def list(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
) -> KnowledgeFSCredentialListResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
)
with self._session_maker() as session:
credentials = tuple(
session.scalars(
sa.select(KnowledgeFSApiCredential)
.where(
KnowledgeFSApiCredential.tenant_id == tenant_id,
KnowledgeFSApiCredential.control_space_id == control_space_id,
)
.order_by(KnowledgeFSApiCredential.created_at.desc(), KnowledgeFSApiCredential.id.desc())
)
)
return KnowledgeFSCredentialListResponse(data=[_credential_item(item) for item in credentials])
def revoke(
self,
*,
tenant_id: str,
actor_account_id: str,
control_space_id: str,
credential_id: str,
reason: str = "revoked_by_user",
) -> None:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=actor_account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
)
credential_hash: str | None = None
with self._session_maker.begin() as session:
credential = session.scalar(
sa.select(KnowledgeFSApiCredential)
.where(
KnowledgeFSApiCredential.tenant_id == tenant_id,
KnowledgeFSApiCredential.control_space_id == control_space_id,
KnowledgeFSApiCredential.id == credential_id,
)
.with_for_update()
)
if credential is None:
raise KnowledgeFSCredentialPolicyError("KnowledgeFS credential was not found")
credential_hash = credential.credential_hash
if credential.status is KnowledgeFSApiCredentialStatus.ACTIVE:
credential.status = KnowledgeFSApiCredentialStatus.REVOKED
credential.revision += 1
credential.revoked_at = naive_utc_now()
credential.revoked_by_account_id = actor_account_id
credential.revoke_reason = reason[:255]
self._revocations.enqueue_principal_grants(
session=session,
tenant_id=tenant_id,
control_space_id=control_space_id,
subject=f"dify-kfs-credential:{credential.principal}",
reason_code="credential_revoked",
caller_kinds=("service",),
)
if credential_hash is not None:
self._cache.invalidate(credential_hash)
def validate_service_credential(
self,
*,
raw_credential: str,
required_action: str,
) -> KnowledgeFSServiceCredentialProfile:
if not raw_credential.startswith(_SECRET_PREFIX) or len(raw_credential) < 32:
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
credential_hash = hash_knowledge_fs_credential(raw_credential)
cached = self._cache.get(credential_hash)
if cached is not None:
if cached.expires_at is not None and cached.expires_at <= naive_utc_now():
self._cache.invalidate(credential_hash)
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
if required_action not in cached.allowed_actions:
raise KnowledgeFSCredentialValidationError("Credential action is not allowed")
return cached
with self._session_maker.begin() as session:
row = session.execute(
sa.select(
KnowledgeFSApiCredential,
KnowledgeFSControlSpace,
KnowledgeFSExternalAccessPolicy,
KnowledgeFSAuthorizationRevision,
)
.join(
KnowledgeFSControlSpace,
sa.and_(
KnowledgeFSControlSpace.tenant_id == KnowledgeFSApiCredential.tenant_id,
KnowledgeFSControlSpace.id == KnowledgeFSApiCredential.control_space_id,
),
)
.outerjoin(
KnowledgeFSExternalAccessPolicy,
sa.and_(
KnowledgeFSExternalAccessPolicy.tenant_id == KnowledgeFSApiCredential.tenant_id,
KnowledgeFSExternalAccessPolicy.control_space_id == KnowledgeFSApiCredential.control_space_id,
),
)
.join(
KnowledgeFSAuthorizationRevision,
sa.and_(
KnowledgeFSAuthorizationRevision.tenant_id == KnowledgeFSApiCredential.tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id == KnowledgeFSApiCredential.control_space_id,
),
)
.where(KnowledgeFSApiCredential.credential_hash == credential_hash)
.with_for_update(of=KnowledgeFSApiCredential)
).one_or_none()
if row is None:
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
credential, control_space, policy, revision = row._t
now = naive_utc_now()
if credential.expires_at is not None and credential.expires_at <= now:
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
if (
credential.status is not KnowledgeFSApiCredentialStatus.ACTIVE
or control_space.state is not KnowledgeFSControlSpaceState.ACTIVE
or control_space.knowledge_space_id is None
or policy is None
or not policy.service_api_enabled
or required_action not in credential.allowed_actions
):
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
credential.last_used_at = now
profile = KnowledgeFSServiceCredentialProfile(
tenant_id=credential.tenant_id,
control_space_id=credential.control_space_id,
credential_id=credential.id,
principal_id=credential.principal,
allowed_actions=frozenset(credential.allowed_actions),
knowledge_space_id=control_space.knowledge_space_id,
knowledge_space_revision=control_space.knowledge_space_revision,
membership_epoch=revision.membership_epoch,
space_acl_epoch=revision.space_acl_epoch,
external_access_epoch=revision.external_access_epoch,
content_policy_revision=revision.content_policy_revision,
credential_revision=credential.revision,
expires_at=credential.expires_at,
)
self._cache.set(credential_hash, profile)
return profile
def hash_knowledge_fs_credential(raw_credential: str) -> str:
return f"sha256:{hashlib.sha256(raw_credential.encode()).hexdigest()}"
def _validate_allowed_actions(actions: Sequence[str]) -> tuple[str, ...]:
normalized = tuple(dict.fromkeys(action.strip() for action in actions if action.strip()))
if not normalized or len(normalized) != len(actions):
raise KnowledgeFSCredentialPolicyError("Credential actions must be unique and non-empty")
supported = knowledge_fs_service_credential_actions()
if not set(normalized).issubset(supported):
raise KnowledgeFSCredentialPolicyError("Credential action is not registered for the service profile")
return tuple(sorted(normalized))
def _normalize_expiry(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None or value.utcoffset() is None:
return value
return value.astimezone(UTC).replace(tzinfo=None)
def _credential_item(credential: KnowledgeFSApiCredential) -> KnowledgeFSCredentialItemResponse:
status = credential.status
if (
status is KnowledgeFSApiCredentialStatus.ACTIVE
and credential.expires_at is not None
and credential.expires_at <= naive_utc_now()
):
status = KnowledgeFSApiCredentialStatus.EXPIRED
return KnowledgeFSCredentialItemResponse(
id=credential.id,
credential_prefix=credential.credential_prefix,
credential_last4=credential.credential_last4,
principal=credential.principal,
allowed_actions=list(credential.allowed_actions),
status=status.value,
revision=credential.revision,
expires_at=credential.expires_at,
last_used_at=credential.last_used_at,
)
__all__ = [
"KnowledgeFSCredentialError",
"KnowledgeFSCredentialPolicyError",
"KnowledgeFSCredentialProfileCache",
"KnowledgeFSCredentialService",
"KnowledgeFSCredentialValidationError",
"KnowledgeFSServiceCredentialProfile",
"NullKnowledgeFSCredentialProfileCache",
"hash_knowledge_fs_credential",
"knowledge_fs_service_credential_actions",
]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,65 @@
"""Fail-closed runtime admission for per-Workspace KnowledgeFS cutover."""
from __future__ import annotations
from typing import Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs_cutover import (
KnowledgeFSWorkspaceCutoverLedger,
KnowledgeFSWorkspaceCutoverPhase,
)
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
_TRAFFIC_PHASES = frozenset(
{
KnowledgeFSWorkspaceCutoverPhase.CUTOVER,
KnowledgeFSWorkspaceCutoverPhase.OBSERVING,
KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP,
}
)
class KnowledgeFSWorkspaceRuntimeGatePort(Protocol):
"""Workspace-scoped admission checked before product traffic or token issuance."""
def require_product_routes(self, *, tenant_id: str) -> None: ...
def require_capability_v2(self, *, tenant_id: str) -> None: ...
class SQLKnowledgeFSWorkspaceRuntimeGate:
"""Read the atomic cutover row and reject missing or partial rollout state."""
def __init__(self, session_maker: sessionmaker[Session]) -> None:
self._session_maker = session_maker
def require_product_routes(self, *, tenant_id: str) -> None:
self._require_complete_cutover(tenant_id=tenant_id)
def require_capability_v2(self, *, tenant_id: str) -> None:
self._require_complete_cutover(tenant_id=tenant_id)
def _require_complete_cutover(self, *, tenant_id: str) -> None:
with self._session_maker() as session:
ledger = session.scalar(
sa.select(KnowledgeFSWorkspaceCutoverLedger).where(
KnowledgeFSWorkspaceCutoverLedger.tenant_id == tenant_id
)
)
if (
ledger is None
or ledger.phase not in _TRAFFIC_PHASES
or ledger.cutover_at is None
or ledger.rolled_back_at is not None
or not ledger.product_routes_enabled
or not ledger.capability_v2_enabled
or not ledger.integrated_mode_enabled
or not ledger.legacy_acl_read_only
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Workspace is not cut over for product traffic")
__all__ = ["KnowledgeFSWorkspaceRuntimeGatePort", "SQLKnowledgeFSWorkspaceRuntimeGate"]
+998
View File
@@ -0,0 +1,998 @@
"""Typed KnowledgeFS BFF facade over operation capabilities and bounded transports."""
from __future__ import annotations
from collections.abc import Callable, Generator
from contextlib import contextmanager
from typing import Literal
from pydantic import BaseModel, JsonValue
from services.knowledge_fs.capability_broker import KnowledgeFSCapabilityBroker
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
from services.knowledge_fs.operation_admission import KnowledgeFSOperationAdmissionService
from services.knowledge_fs.product_dto import (
KnowledgeFSAnswerTraceResponse,
KnowledgeFSBulkDeletionAcceptedResponse,
KnowledgeFSBulkDocumentDeletePayload,
KnowledgeFSBulkJobResponse,
KnowledgeFSDocumentChunkListResponse,
KnowledgeFSDocumentChunkResponse,
KnowledgeFSDocumentCompilationJobResponse,
KnowledgeFSDocumentCreatePayload,
KnowledgeFSDocumentDeletePayload,
KnowledgeFSDocumentListResponse,
KnowledgeFSDocumentMetadataPayload,
KnowledgeFSDocumentOutlineResponse,
KnowledgeFSDocumentReindexPayload,
KnowledgeFSDocumentReindexResponse,
KnowledgeFSDocumentResponse,
KnowledgeFSDocumentRevisionListResponse,
KnowledgeFSDurableDeletionAcceptedResponse,
KnowledgeFSLogicalDocumentResponse,
KnowledgeFSQueryCreatePayload,
KnowledgeFSQueryResponse,
KnowledgeFSResearchTaskCreatePayload,
KnowledgeFSResearchTaskListResponse,
KnowledgeFSResearchTaskPartialListResponse,
KnowledgeFSResearchTaskPlanPayload,
KnowledgeFSResearchTaskPlanResponse,
KnowledgeFSResearchTaskResponse,
KnowledgeFSSettingsPayload,
KnowledgeFSSettingsResponse,
KnowledgeFSSmallFileUploadResponse,
KnowledgeFSSourceCrawlResponse,
KnowledgeFSSourceCreatePayload,
KnowledgeFSSourceCredentialTestResponse,
KnowledgeFSSourceDeletePayload,
KnowledgeFSSourceFilesResponse,
KnowledgeFSSourceImportFilesPayload,
KnowledgeFSSourceImportPagesPayload,
KnowledgeFSSourceImportResponse,
KnowledgeFSSourceListResponse,
KnowledgeFSSourcePagesResponse,
KnowledgeFSSourceResponse,
KnowledgeFSSourceUpdatePayload,
KnowledgeFSSpaceUpdatePayload,
KnowledgeFSTraceEntryListResponse,
KnowledgeFSTraceListResponse,
)
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemotePort,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSRemoteBinaryRequest,
KnowledgeFSRemoteJSONRequest,
)
class KnowledgeFSDataFacade:
def __init__(
self,
*,
admission: KnowledgeFSOperationAdmissionService,
broker: KnowledgeFSCapabilityBroker,
remote: KnowledgeFSProductRemotePort,
) -> None:
self._admission = admission
self._broker = broker
self._remote = remote
def get_settings(self, *, tenant_id: str, account_id: str, control_space_id: str) -> KnowledgeFSSettingsResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getSettings",
)
return KnowledgeFSSettingsResponse.model_validate(raw)
def update_space(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSSpaceUpdatePayload,
) -> JsonValue:
return self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="updateSpace",
payload=payload,
)
def update_settings(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSSettingsPayload,
) -> KnowledgeFSSettingsResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="updateSettings",
payload=payload,
)
return KnowledgeFSSettingsResponse.model_validate(raw)
def list_documents(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
cursor: str | None,
) -> KnowledgeFSDocumentListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listDocuments",
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSDocumentListResponse.model_validate(raw)
def create_document(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSDocumentCreatePayload,
) -> KnowledgeFSDocumentResponse:
_ = (tenant_id, account_id, control_space_id, payload)
raise KnowledgeFSOperationUnavailableError(
"Buffered KnowledgeFS document creation is deprecated; use the P6 direct-upload capability flow"
)
def upload_small_file(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
upload_session_id: str,
body_reader: Callable[[int], bytes],
) -> KnowledgeFSSmallFileUploadResponse:
operation_id = "uploadSmallFile"
_assert_binary_bff_ready(operation_id)
with self._admitted(tenant_id=tenant_id, operation_id=operation_id):
issued = self._broker.issue_interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
resource_id=upload_session_id,
)
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
body = body_reader(operation.max_request_bytes)
if not isinstance(body, bytes) or not body:
raise KnowledgeFSProductRequestRejectedError(status_code=422)
if len(body) > operation.max_request_bytes:
raise KnowledgeFSProductRequestRejectedError(status_code=413)
if operation.kfs_path is None:
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
path = _resolve_product_path(
template=operation.kfs_path,
knowledge_space_id=issued.knowledge_space_id,
resource_id=upload_session_id,
resource_resolver=operation.resource_resolver,
path_parameters=(),
)
raw = self._remote.execute_binary(
KnowledgeFSRemoteBinaryRequest(
operation_id=operation_id,
method=operation.method,
path=path,
namespace_id=tenant_id,
knowledge_space_id=issued.knowledge_space_id,
capability_token=issued.token,
trace_id=issued.trace_id,
body=body,
query=(("knowledgeSpaceId", issued.knowledge_space_id),),
)
)
return KnowledgeFSSmallFileUploadResponse.model_validate(raw)
def get_document(
self, *, tenant_id: str, account_id: str, control_space_id: str, document_id: str
) -> KnowledgeFSDocumentResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getDocument",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
)
return KnowledgeFSDocumentResponse.model_validate(raw)
def get_document_outline(
self, *, tenant_id: str, account_id: str, control_space_id: str, document_id: str
) -> KnowledgeFSDocumentOutlineResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getDocumentOutline",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
)
return KnowledgeFSDocumentOutlineResponse.model_validate(raw)
def list_document_revisions(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
document_id: str,
cursor: str | None = None,
) -> KnowledgeFSDocumentRevisionListResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listDocumentRevisions",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSDocumentRevisionListResponse.model_validate(raw)
def update_document_metadata(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
document_id: str,
payload: KnowledgeFSDocumentMetadataPayload,
) -> KnowledgeFSLogicalDocumentResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="updateDocumentMetadata",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
payload=payload,
)
return KnowledgeFSLogicalDocumentResponse.model_validate(raw)
def list_document_chunks(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
document_id: str,
revision: int,
cursor: str | None = None,
query_text: str | None = None,
) -> KnowledgeFSDocumentChunkListResponse:
query = tuple((name, value) for name, value in (("cursor", cursor), ("query", query_text)) if value)
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listDocumentChunks",
resource_id=document_id,
path_parameters=(("documentId", document_id), ("revision", str(revision))),
query=query,
)
return KnowledgeFSDocumentChunkListResponse.model_validate(raw)
def get_document_chunk(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
document_id: str,
revision: int,
chunk_id: str,
) -> KnowledgeFSDocumentChunkResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getDocumentChunk",
resource_id=document_id,
path_parameters=(
("documentId", document_id),
("revision", str(revision)),
("chunkId", chunk_id),
),
)
return KnowledgeFSDocumentChunkResponse.model_validate(raw)
def delete_document(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
document_id: str,
payload: KnowledgeFSDocumentDeletePayload,
idempotency_key: str,
) -> KnowledgeFSDurableDeletionAcceptedResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="deleteDocument",
resource_id=document_id,
path_parameters=(("documentId", document_id),),
payload=payload,
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSDurableDeletionAcceptedResponse.model_validate(raw)
def bulk_delete_documents(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSBulkDocumentDeletePayload,
idempotency_key: str,
) -> KnowledgeFSBulkDeletionAcceptedResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="bulkDeleteDocuments",
payload=payload,
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSBulkDeletionAcceptedResponse.model_validate(raw)
def reindex_documents(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSDocumentReindexPayload,
) -> KnowledgeFSDocumentReindexResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="reindexDocuments",
payload=payload,
)
return KnowledgeFSDocumentReindexResponse.model_validate(raw)
def get_compilation_job(
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
) -> KnowledgeFSDocumentCompilationJobResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getCompilationJob",
resource_id=job_id,
)
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
def cancel_compilation_job(
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
) -> KnowledgeFSDocumentCompilationJobResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="cancelCompilationJob",
resource_id=job_id,
)
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
def retry_compilation_job(
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
) -> KnowledgeFSDocumentCompilationJobResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="retryCompilationJob",
resource_id=job_id,
)
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
def get_bulk_job(
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
) -> KnowledgeFSBulkJobResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getBulkJob",
resource_id=job_id,
)
return KnowledgeFSBulkJobResponse.model_validate(raw)
def list_sources(
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
) -> KnowledgeFSSourceListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listSources",
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSSourceListResponse.model_validate(raw)
def create_source(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSSourceCreatePayload,
) -> KnowledgeFSSourceResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="createSource",
payload=payload,
)
return KnowledgeFSSourceResponse.model_validate(raw)
def get_source(
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
) -> KnowledgeFSSourceResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
)
return KnowledgeFSSourceResponse.model_validate(raw)
def update_source(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
payload: KnowledgeFSSourceUpdatePayload,
) -> KnowledgeFSSourceResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="updateSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
payload=payload,
)
return KnowledgeFSSourceResponse.model_validate(raw)
def delete_source(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
payload: KnowledgeFSSourceDeletePayload,
documents: Literal["cascade", "keep"],
idempotency_key: str,
) -> KnowledgeFSDurableDeletionAcceptedResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="deleteSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
payload=payload,
query=(("documents", documents),),
headers=(("Idempotency-Key", idempotency_key),),
)
return KnowledgeFSDurableDeletionAcceptedResponse.model_validate(raw)
def test_source(
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
) -> KnowledgeFSSourceCredentialTestResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="testSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
)
return KnowledgeFSSourceCredentialTestResponse.model_validate(raw)
def crawl_source(
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
) -> KnowledgeFSSourceCrawlResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="crawlSource",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
)
return KnowledgeFSSourceCrawlResponse.model_validate(raw)
def list_source_pages(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
cursor: str | None = None,
limit: int = 50,
) -> KnowledgeFSSourcePagesResponse:
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listSourcePages",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
query=query,
)
return KnowledgeFSSourcePagesResponse.model_validate(raw)
def import_source_pages(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
payload: KnowledgeFSSourceImportPagesPayload,
) -> KnowledgeFSSourceImportResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="importSourcePages",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
payload=payload,
)
return KnowledgeFSSourceImportResponse.model_validate(raw)
def list_source_files(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
query: tuple[tuple[str, str], ...] = (),
) -> KnowledgeFSSourceFilesResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listSourceFiles",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
query=query,
)
return KnowledgeFSSourceFilesResponse.model_validate(raw)
def import_source_files(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
source_id: str,
payload: KnowledgeFSSourceImportFilesPayload,
) -> KnowledgeFSSourceImportResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="importSourceFiles",
resource_id=source_id,
path_parameters=(("sourceId", source_id),),
payload=payload,
)
return KnowledgeFSSourceImportResponse.model_validate(raw)
def create_query(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSQueryCreatePayload,
) -> KnowledgeFSQueryResponse:
_ = (tenant_id, account_id, control_space_id, payload)
raise KnowledgeFSOperationUnavailableError(
"Buffered KnowledgeFS query creation is deprecated; use the queries/admission direct flow"
)
def create_research_task(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSResearchTaskCreatePayload,
) -> KnowledgeFSResearchTaskResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="createResearchTask",
payload=payload,
bind_space_in_body=True,
)
return KnowledgeFSResearchTaskResponse.model_validate(raw)
def plan_research_task(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSResearchTaskPlanPayload,
) -> KnowledgeFSResearchTaskPlanResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="planResearchTask",
payload=payload,
bind_space_in_body=True,
)
return KnowledgeFSResearchTaskPlanResponse.model_validate(raw)
def get_research_task(
self, *, tenant_id: str, account_id: str, control_space_id: str, task_id: str
) -> KnowledgeFSResearchTaskResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getResearchTask",
resource_id=task_id,
)
return KnowledgeFSResearchTaskResponse.model_validate(raw)
def list_research_task_partials(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
task_id: str,
cursor: str | None = None,
limit: int = 25,
) -> KnowledgeFSResearchTaskPartialListResponse:
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listResearchTaskPartials",
resource_id=task_id,
query=query,
)
return KnowledgeFSResearchTaskPartialListResponse.model_validate(raw)
def cancel_research_task(
self, *, tenant_id: str, account_id: str, control_space_id: str, task_id: str
) -> KnowledgeFSResearchTaskResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="cancelResearchTask",
resource_id=task_id,
)
return KnowledgeFSResearchTaskResponse.model_validate(raw)
def list_research_tasks(
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
) -> KnowledgeFSResearchTaskListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listResearchTasks",
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSResearchTaskListResponse.model_validate(raw)
def list_traces(
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
) -> KnowledgeFSTraceListResponse:
raw = self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="listTraces",
query=(("cursor", cursor),) if cursor else (),
)
return KnowledgeFSTraceListResponse.model_validate(raw)
def get_trace(
self, *, tenant_id: str, account_id: str, control_space_id: str, trace_id: str
) -> KnowledgeFSAnswerTraceResponse:
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id="getTrace",
resource_id=trace_id,
path_parameters=(("traceId", trace_id),),
)
return KnowledgeFSAnswerTraceResponse.model_validate(raw)
def list_trace_entries(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
trace_id: str,
kind: Literal["conflicts", "evidence", "missing"],
cursor: str | None = None,
limit: int = 100,
) -> KnowledgeFSTraceEntryListResponse:
operation_id = {
"conflicts": "listTraceConflicts",
"evidence": "listTraceEvidence",
"missing": "listTraceMissing",
}[kind]
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
raw = self._interactive_child(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
resource_id=trace_id,
path_parameters=(("traceId", trace_id),),
query=query,
)
return KnowledgeFSTraceEntryListResponse.model_validate(raw)
def execute_service(
self,
*,
profile: KnowledgeFSServiceCredentialProfile,
operation_id: str,
payload: BaseModel | None = None,
query: tuple[tuple[str, str], ...] = (),
bind_space_in_body: bool = False,
resource_id: str | None = None,
path_parameters: tuple[tuple[str, str], ...] = (),
headers: tuple[tuple[str, str], ...] = (),
) -> JsonValue:
_assert_json_bff_ready(operation_id)
with self._admitted(tenant_id=profile.tenant_id, operation_id=operation_id):
issued = self._broker.issue_service(profile=profile, operation_id=operation_id, resource_id=resource_id)
return self._execute(
operation_id=operation_id,
namespace_id=profile.tenant_id,
knowledge_space_id=profile.knowledge_space_id,
knowledge_space_revision=profile.knowledge_space_revision,
capability_token=issued.token,
trace_id=issued.trace_id,
payload=payload,
query=query,
bind_space_in_body=bind_space_in_body,
resource_id=resource_id,
path_parameters=path_parameters,
headers=headers,
)
def _interactive(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
payload: BaseModel | None = None,
query: tuple[tuple[str, str], ...] = (),
bind_space_in_body: bool = False,
resource_id: str | None = None,
path_parameters: tuple[tuple[str, str], ...] = (),
headers: tuple[tuple[str, str], ...] = (),
) -> JsonValue:
_assert_json_bff_ready(operation_id)
with self._admitted(tenant_id=tenant_id, operation_id=operation_id):
issued = self._broker.issue_interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
resource_id=resource_id,
)
return self._execute(
operation_id=operation_id,
namespace_id=tenant_id,
knowledge_space_id=issued.knowledge_space_id,
knowledge_space_revision=issued.knowledge_space_revision,
capability_token=issued.token,
trace_id=issued.trace_id,
payload=payload,
query=query,
bind_space_in_body=bind_space_in_body,
resource_id=resource_id,
path_parameters=path_parameters,
headers=headers,
)
@contextmanager
def _admitted(self, *, tenant_id: str, operation_id: str) -> Generator[None]:
charge = self._admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
try:
yield
except BaseException:
charge.refund()
raise
else:
charge.commit()
def _interactive_child(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
resource_id: str,
payload: BaseModel | None = None,
query: tuple[tuple[str, str], ...] = (),
path_parameters: tuple[tuple[str, str], ...] = (),
headers: tuple[tuple[str, str], ...] = (),
) -> JsonValue:
return self._interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
resource_id=resource_id,
payload=payload,
query=query,
path_parameters=path_parameters,
headers=headers,
)
def _execute(
self,
*,
operation_id: str,
namespace_id: str,
knowledge_space_id: str,
knowledge_space_revision: int,
capability_token: str,
trace_id: str,
payload: BaseModel | None,
query: tuple[tuple[str, str], ...],
bind_space_in_body: bool,
resource_id: str | None,
path_parameters: tuple[tuple[str, str], ...],
headers: tuple[tuple[str, str], ...],
) -> JsonValue:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
if operation.kfs_path is None:
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
path = _resolve_product_path(
template=operation.kfs_path,
knowledge_space_id=knowledge_space_id,
resource_id=resource_id,
resource_resolver=operation.resource_resolver,
path_parameters=path_parameters,
)
remote_payload: JsonValue | None = (
payload.model_dump(mode="json", exclude_none=True, by_alias=True) if payload is not None else None
)
if operation_id == "updateSpace":
if knowledge_space_revision <= 0 or not isinstance(payload, KnowledgeFSSpaceUpdatePayload):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS metadata revision is not available")
remote_payload = {"expectedRevision": knowledge_space_revision}
if payload.name is not None:
remote_payload["name"] = payload.name
if payload.icon is not None:
remote_payload["iconRef"] = payload.icon
if payload.description is not None:
remote_payload["description"] = payload.description
if bind_space_in_body:
if not isinstance(remote_payload, dict):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request body binding is unavailable")
remote_payload["knowledgeSpaceId"] = knowledge_space_id
if operation.resource_resolver in {"job", "query", "research_task"} and not path.startswith(
"/knowledge-spaces/"
):
query = (*query, ("knowledgeSpaceId", knowledge_space_id))
return self._remote.execute_json(
KnowledgeFSRemoteJSONRequest(
operation_id=operation_id,
method=operation.method,
path=path,
namespace_id=namespace_id,
knowledge_space_id=knowledge_space_id,
capability_token=capability_token,
trace_id=trace_id,
payload=remote_payload,
query=query,
headers=headers,
)
)
def _resolve_product_path(
*,
template: str,
knowledge_space_id: str,
resource_id: str | None,
resource_resolver: str,
path_parameters: tuple[tuple[str, str], ...],
) -> str:
bindings = {name: _path_segment(value) for name, value in path_parameters}
if template.startswith("/knowledge-spaces/{id}") or resource_resolver in {"knowledge_space", "namespace"}:
bindings["id"] = _path_segment(knowledge_space_id)
elif "{id}" in template:
if resource_id is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS child resource is required")
bindings["id"] = _path_segment(resource_id)
path = template
for name, value in bindings.items():
path = path.replace(f"{{{name}}}", value)
if "{" in path or "}" in path:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product path resolver is not registered")
return path
def _path_segment(value: str) -> str:
normalized = value.strip()
if (
not normalized
or len(normalized) > 1_000
or normalized in {".", ".."}
or any(character in normalized for character in ("/", "%", "?", "#", "\\"))
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product path parameter is invalid")
return normalized
def _assert_ready(operation_id: str) -> None:
if not is_product_operation_ready(operation_id):
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
def _assert_json_bff_ready(operation_id: str) -> None:
_assert_ready(operation_id)
if KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].transport != "json":
raise KnowledgeFSOperationUnavailableError(
f"KnowledgeFS operation requires a direct transport and cannot use the buffered BFF: {operation_id}"
)
def _assert_binary_bff_ready(operation_id: str) -> None:
_assert_ready(operation_id)
if KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].transport != "binary":
raise KnowledgeFSOperationUnavailableError(
f"KnowledgeFS operation does not allow the bounded binary BFF: {operation_id}"
)
__all__ = ["KnowledgeFSDataFacade"]
+240
View File
@@ -0,0 +1,240 @@
"""Remote port for the KnowledgeFS lifecycle data-plane."""
from __future__ import annotations
from datetime import datetime
from enum import StrEnum
from typing import Literal, NamedTuple, Protocol
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from models.knowledge_fs import KnowledgeFSModelSelectionIntentPayload, KnowledgeFSRetrievalProfileIntentPayload
class KnowledgeFSRemoteSpace(NamedTuple):
namespace_id: str
knowledge_space_id: str
provisioning_key: str
revision: int
class KnowledgeFSIntegratedProvisionRequest(NamedTuple):
namespace_id: str
control_space_id: str
operation_id: str
idempotency_key: str
provisioning_key: str
name: str
slug: str
icon: str | None
description: str | None
model_intent: KnowledgeFSModelSelectionIntentPayload
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
class KnowledgeFSIntegratedDeletionRequest(NamedTuple):
namespace_id: str
control_space_id: str
operation_id: str
idempotency_key: str
knowledge_space_id: str | None
provisioning_key: str
expected_revision: int
class KnowledgeFSCapabilityGrantRevokeRequest(NamedTuple):
namespace_id: str
control_space_id: str
operation_id: str
idempotency_key: str
knowledge_space_id: str
grant_id: str
event_id: str
reason_code: str
revoke_sequence: int
expected_revision: int
class KnowledgeFSCapabilityGrantRevokeAck(NamedTuple):
applied: bool
highest_revoke_sequence: int
state: Literal["active", "revoked"]
class _StrictLifecycleModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class KnowledgeFSDifyIntegrationFreezeRequest(_StrictLifecycleModel):
"""Tenant-scoped monotonic freeze command sent before final-delta capture."""
namespace_id: str = Field(min_length=1, max_length=255)
control_space_id: str = Field(min_length=1, max_length=255)
freeze_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991)
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991)
@field_validator("namespace_id", "control_space_id", "freeze_id")
@classmethod
def validate_normalized_identifier(cls, value: str) -> str:
if value.strip() != value:
raise ValueError("freeze identifiers must be normalized")
return value
class KnowledgeFSDifyIntegrationFreezeAck(_StrictLifecycleModel):
"""Exact durable KnowledgeFS acknowledgement required before local freeze publication."""
namespace_id: str = Field(min_length=1, max_length=255)
freeze_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991)
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991)
frozen_at: datetime
updated_at: datetime
frozen: Literal[True]
applied: bool
replayed: bool
@field_validator("namespace_id", "freeze_id")
@classmethod
def validate_normalized_identifier(cls, value: str) -> str:
if value.strip() != value:
raise ValueError("freeze identifiers must be normalized")
return value
@field_validator("frozen_at", "updated_at")
@classmethod
def validate_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("freeze timestamps must include a timezone")
return value
@model_validator(mode="after")
def validate_outcome(self) -> KnowledgeFSDifyIntegrationFreezeAck:
if self.applied is self.replayed:
raise ValueError("freeze acknowledgement must be either applied or replayed")
return self
class KnowledgeFSDifyIntegrationActivationRequest(_StrictLifecycleModel):
"""Tenant-scoped monotonic activation command sent only after local cutover gates pass."""
namespace_id: str = Field(min_length=1, max_length=255)
control_space_id: str = Field(min_length=1, max_length=255)
activation_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991)
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
@field_validator("namespace_id", "control_space_id", "activation_id")
@classmethod
def validate_normalized_identifier(cls, value: str) -> str:
if value.strip() != value:
raise ValueError("activation identifiers must be normalized")
return value
class KnowledgeFSDifyIntegrationActivationAck(_StrictLifecycleModel):
"""Exact durable KnowledgeFS acknowledgement required before local routes are enabled."""
namespace_id: str = Field(min_length=1, max_length=255)
activation_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991)
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
activated_at: datetime
updated_at: datetime
active: Literal[True]
applied: bool
replayed: bool
@field_validator("namespace_id", "activation_id")
@classmethod
def validate_normalized_identifier(cls, value: str) -> str:
if value.strip() != value:
raise ValueError("activation identifiers must be normalized")
return value
@field_validator("activated_at", "updated_at")
@classmethod
def validate_timezone(cls, value: datetime) -> datetime:
if value.tzinfo is None or value.utcoffset() is None:
raise ValueError("activation timestamps must include a timezone")
return value
@model_validator(mode="after")
def validate_outcome(self) -> KnowledgeFSDifyIntegrationActivationAck:
if self.applied is self.replayed:
raise ValueError("activation acknowledgement must be either applied or replayed")
return self
class KnowledgeFSDeletionPhase(StrEnum):
ACCEPTED = "accepted"
IRREVERSIBLE = "irreversible"
COMPLETED = "completed"
class KnowledgeFSDeletionProgress(NamedTuple):
phase: KnowledgeFSDeletionPhase
revision: int
irreversible_at: datetime | None
class KnowledgeFSLifecycleRemotePort(Protocol):
"""Explicit hexagonal boundary; tests inject a fake with no HTTP dependency."""
def provision_integrated_space(self, request: KnowledgeFSIntegratedProvisionRequest) -> KnowledgeFSRemoteSpace: ...
def request_integrated_deletion(
self, request: KnowledgeFSIntegratedDeletionRequest
) -> KnowledgeFSDeletionProgress: ...
def revoke_capability_grant(
self, request: KnowledgeFSCapabilityGrantRevokeRequest
) -> KnowledgeFSCapabilityGrantRevokeAck: ...
def activate_dify_workspace_integration(
self, request: KnowledgeFSDifyIntegrationActivationRequest
) -> KnowledgeFSDifyIntegrationActivationAck: ...
def freeze_dify_workspace_integration(
self, request: KnowledgeFSDifyIntegrationFreezeRequest
) -> KnowledgeFSDifyIntegrationFreezeAck: ...
def find_by_provisioning_key(
self,
*,
provisioning_key: str,
control_space_id: str,
) -> KnowledgeFSRemoteSpace | None: ...
def list_spaces(
self,
*,
namespace_id: str,
control_space_id: str,
) -> tuple[KnowledgeFSRemoteSpace, ...]: ...
class KnowledgeFSLifecycleRemoteError(RuntimeError):
def __init__(self, code: str, message: str):
self.code = code
super().__init__(message)
__all__ = [
"KnowledgeFSCapabilityGrantRevokeAck",
"KnowledgeFSCapabilityGrantRevokeRequest",
"KnowledgeFSDeletionPhase",
"KnowledgeFSDeletionProgress",
"KnowledgeFSDifyIntegrationActivationAck",
"KnowledgeFSDifyIntegrationActivationRequest",
"KnowledgeFSDifyIntegrationFreezeAck",
"KnowledgeFSDifyIntegrationFreezeRequest",
"KnowledgeFSIntegratedDeletionRequest",
"KnowledgeFSIntegratedProvisionRequest",
"KnowledgeFSLifecycleRemoteError",
"KnowledgeFSLifecycleRemotePort",
"KnowledgeFSRemoteSpace",
]
@@ -0,0 +1,47 @@
"""Fail-closed rollout gates for the KnowledgeFS lifecycle worker."""
from __future__ import annotations
from typing import NamedTuple
class KnowledgeFSLifecycleWorkerReadiness(NamedTuple):
ready: bool
blockers: tuple[str, ...]
def evaluate_knowledge_fs_lifecycle_worker_readiness(
*,
worker_enabled: bool,
capability_v2_enabled: bool,
integrated_provision_ready: bool,
legacy_acl_freeze_ready: bool,
) -> KnowledgeFSLifecycleWorkerReadiness:
"""Require every security dependency without coupling cleanup to the product flag."""
gates = (
("worker", worker_enabled),
("capability_v2", capability_v2_enabled),
("integrated_provision", integrated_provision_ready),
("legacy_acl_freeze", legacy_acl_freeze_ready),
)
blockers = tuple(name for name, enabled in gates if not enabled)
return KnowledgeFSLifecycleWorkerReadiness(ready=not blockers, blockers=blockers)
def get_configured_knowledge_fs_lifecycle_worker_readiness() -> KnowledgeFSLifecycleWorkerReadiness:
from configs import dify_config
return evaluate_knowledge_fs_lifecycle_worker_readiness(
worker_enabled=dify_config.KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED,
capability_v2_enabled=dify_config.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED,
integrated_provision_ready=dify_config.KNOWLEDGE_FS_INTEGRATED_PROVISION_READY,
legacy_acl_freeze_ready=dify_config.KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY,
)
__all__ = [
"KnowledgeFSLifecycleWorkerReadiness",
"evaluate_knowledge_fs_lifecycle_worker_readiness",
"get_configured_knowledge_fs_lifecycle_worker_readiness",
]
@@ -0,0 +1,578 @@
"""Capability-v2-only HTTP adapter for the KnowledgeFS lifecycle data plane."""
from __future__ import annotations
from datetime import UTC, datetime
from http import HTTPStatus
from typing import Literal
import httpx
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from core.helper import ssrf_proxy
from core.tools.errors import ToolSSRFError
from services.knowledge_fs.lifecycle_port import (
KnowledgeFSCapabilityGrantRevokeAck,
KnowledgeFSCapabilityGrantRevokeRequest,
KnowledgeFSDeletionPhase,
KnowledgeFSDeletionProgress,
KnowledgeFSDifyIntegrationActivationAck,
KnowledgeFSDifyIntegrationActivationRequest,
KnowledgeFSDifyIntegrationFreezeAck,
KnowledgeFSDifyIntegrationFreezeRequest,
KnowledgeFSIntegratedDeletionRequest,
KnowledgeFSIntegratedProvisionRequest,
KnowledgeFSLifecycleRemoteError,
KnowledgeFSRemoteSpace,
)
from services.knowledge_fs_capability import (
CapabilityAuthzRevision,
CapabilityIssueRequest,
CapabilityResource,
KnowledgeFSCapabilityIssuer,
)
_JSON_ADAPTER: TypeAdapter[JsonValue] = TypeAdapter(JsonValue)
_MAX_RESPONSE_BYTES = 1024 * 1024
_MAX_RECONCILIATION_SPACES = 10_000
_PAGE_SIZE = 100
_WORKER_PRINCIPAL = "knowledge-fs-lifecycle"
class _KnowledgeSpaceResponse(BaseModel):
id: str
revision: int = Field(ge=1)
slug: str = Field(min_length=1, max_length=160)
tenant_id: str = Field(min_length=1, max_length=255, alias="tenantId")
model_config = ConfigDict(extra="ignore", populate_by_name=True)
class _KnowledgeSpaceListResponse(BaseModel):
items: list[_KnowledgeSpaceResponse]
next_cursor: str | None = Field(default=None, alias="nextCursor")
model_config = ConfigDict(extra="forbid", populate_by_name=True)
class _DeletionProgressResponse(BaseModel):
phase: Literal["accepted", "irreversible", "completed"]
revision: int = Field(ge=1)
irreversible_at: datetime | None = Field(default=None, alias="irreversibleAt")
model_config = ConfigDict(extra="forbid", populate_by_name=True)
class _CapabilityGrantRevokeResponse(BaseModel):
applied: bool
highest_revoke_sequence: int = Field(ge=0, alias="highestRevokeSequence")
state: Literal["active", "revoked"]
model_config = ConfigDict(extra="forbid", populate_by_name=True)
class _DifyIntegrationActivationBody(BaseModel):
activation_id: str = Field(alias="activationId")
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991, alias="activationRevision")
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$", alias="sourceRevisionDigest")
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True)
class _DifyIntegrationActivationResponse(_DifyIntegrationActivationBody):
namespace_id: str = Field(min_length=1, max_length=255, alias="namespaceId")
activated_at: datetime = Field(alias="activatedAt")
updated_at: datetime = Field(alias="updatedAt")
active: Literal[True]
applied: bool
replayed: bool
class _DifyIntegrationFreezeBody(BaseModel):
freeze_id: str = Field(alias="freezeId")
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991, alias="freezeRevision")
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$", alias="sourceRevisionDigest")
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991, alias="sourceTaskWatermark")
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True)
class _DifyIntegrationFreezeResponse(_DifyIntegrationFreezeBody):
namespace_id: str = Field(min_length=1, max_length=255, alias="namespaceId")
frozen_at: datetime = Field(alias="frozenAt")
updated_at: datetime = Field(alias="updatedAt")
frozen: Literal[True]
applied: bool
replayed: bool
class HTTPKnowledgeFSLifecycleRemoteClient:
"""Deliver durable lifecycle commands with freshly issued internal-worker capabilities."""
def __init__(
self,
*,
base_url: str,
issuer: KnowledgeFSCapabilityIssuer,
timeout_seconds: float,
max_response_bytes: int = _MAX_RESPONSE_BYTES,
) -> None:
self._base_url = base_url.rstrip("/")
self._issuer = issuer
self._timeout_seconds = timeout_seconds
self._max_response_bytes = max_response_bytes
def provision_integrated_space(self, request: KnowledgeFSIntegratedProvisionRequest) -> KnowledgeFSRemoteSpace:
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
operation_id="provisionIntegratedKnowledgeSpace",
operation_identity=request.operation_id,
resource=CapabilityResource(type="namespace", id=request.namespace_id),
)
payload: dict[str, JsonValue] = {
"embeddingProfile": _json_object(request.model_intent, "embedding profile"),
"idempotencyKey": request.provisioning_key,
"name": request.name,
"retrievalProfile": _json_object(request.profile_intent, "retrieval profile"),
"slug": request.slug,
}
if request.icon is not None:
payload["iconRef"] = request.icon
if request.description is not None:
payload["description"] = request.description
response = self._request_json(
method="POST",
path="/internal/knowledge-spaces/provision",
token=token,
trace_id=request.operation_id,
payload=payload,
expected_statuses=(HTTPStatus.CREATED,),
)
try:
space = _KnowledgeSpaceResponse.model_validate(response)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid provision response"
) from exc
if space.tenant_id != request.namespace_id:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_SCOPE_MISMATCH", "KnowledgeFS provision response crossed its namespace"
)
return _remote_space(space, provisioning_key=request.provisioning_key)
def request_integrated_deletion(self, request: KnowledgeFSIntegratedDeletionRequest) -> KnowledgeFSDeletionProgress:
knowledge_space_id = request.knowledge_space_id
expected_revision = request.expected_revision
if knowledge_space_id is None:
recovered = self.find_by_provisioning_key(
provisioning_key=request.provisioning_key,
control_space_id=request.control_space_id,
)
if recovered is None:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_SPACE_NOT_FOUND",
"KnowledgeFS deletion identity could not be recovered",
)
knowledge_space_id = recovered.knowledge_space_id
expected_revision = recovered.revision
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
operation_id="deleteIntegratedKnowledgeSpace",
operation_identity=request.operation_id,
resource=CapabilityResource(type="knowledge_space", id=knowledge_space_id),
)
response = self._request_json(
method="POST",
path=f"/internal/knowledge-spaces/{knowledge_space_id}/delete",
token=token,
trace_id=request.operation_id,
payload={
"controlSpaceId": request.control_space_id,
"expectedRevision": expected_revision,
"idempotencyKey": request.idempotency_key,
"operationId": request.operation_id,
"provisioningKey": request.provisioning_key,
},
expected_statuses=(HTTPStatus.OK, HTTPStatus.ACCEPTED),
)
try:
progress = _DeletionProgressResponse.model_validate(response)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid deletion response"
) from exc
irreversible_at = progress.irreversible_at
if irreversible_at is not None:
if irreversible_at.tzinfo is None or irreversible_at.utcoffset() is None:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID",
"KnowledgeFS deletion timestamp must include a timezone",
)
irreversible_at = irreversible_at.astimezone(UTC).replace(tzinfo=None)
return KnowledgeFSDeletionProgress(
phase=KnowledgeFSDeletionPhase(progress.phase),
revision=progress.revision,
irreversible_at=irreversible_at,
)
def revoke_capability_grant(
self, request: KnowledgeFSCapabilityGrantRevokeRequest
) -> KnowledgeFSCapabilityGrantRevokeAck:
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
operation_id="revokeCapabilityGrant",
operation_identity=request.operation_id,
resource=CapabilityResource(type="knowledge_space", id=request.knowledge_space_id),
)
response = self._request_json(
method="POST",
path=f"/internal/capability-grants/{request.grant_id}/revoke",
token=token,
trace_id=request.operation_id,
payload={
"eventId": request.event_id,
"knowledgeSpaceId": request.knowledge_space_id,
"reasonCode": request.reason_code,
"revokeSequence": request.revoke_sequence,
},
expected_statuses=(HTTPStatus.OK,),
)
try:
ack = _CapabilityGrantRevokeResponse.model_validate(response)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid capability revoke response"
) from exc
return KnowledgeFSCapabilityGrantRevokeAck(
applied=ack.applied,
highest_revoke_sequence=ack.highest_revoke_sequence,
state=ack.state,
)
def activate_dify_workspace_integration(
self, request: KnowledgeFSDifyIntegrationActivationRequest
) -> KnowledgeFSDifyIntegrationActivationAck:
"""Persist one monotonic namespace activation using a stable grant and trace identity."""
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
operation_id="activateDifyWorkspaceIntegration",
operation_identity=request.activation_id,
resource=CapabilityResource(type="namespace", id=request.namespace_id),
)
body = _DifyIntegrationActivationBody(
activationId=request.activation_id,
activationRevision=request.activation_revision,
sourceRevisionDigest=request.source_revision_digest,
)
response = self._request_json(
method="POST",
path="/internal/dify-integration/activate",
token=token,
trace_id=request.activation_id,
payload=_JSON_ADAPTER.validate_python(body.model_dump(mode="json", by_alias=True)),
expected_statuses=(HTTPStatus.OK,),
)
try:
parsed = _DifyIntegrationActivationResponse.model_validate(response)
ack = KnowledgeFSDifyIntegrationActivationAck(
namespace_id=parsed.namespace_id,
activation_id=parsed.activation_id,
activation_revision=parsed.activation_revision,
source_revision_digest=parsed.source_revision_digest,
activated_at=parsed.activated_at,
updated_at=parsed.updated_at,
active=parsed.active,
applied=parsed.applied,
replayed=parsed.replayed,
)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID",
"KnowledgeFS returned an invalid integration activation acknowledgement",
) from exc
if (
ack.namespace_id != request.namespace_id
or ack.activation_id != request.activation_id
or ack.activation_revision != request.activation_revision
or ack.source_revision_digest != request.source_revision_digest
):
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_ACTIVATION_ACK_MISMATCH",
"KnowledgeFS integration activation acknowledgement did not match the request",
)
return ack
def freeze_dify_workspace_integration(
self, request: KnowledgeFSDifyIntegrationFreezeRequest
) -> KnowledgeFSDifyIntegrationFreezeAck:
"""Persist the remote maintenance freeze before Dify captures the final delta."""
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
operation_id="freezeDifyWorkspaceIntegration",
operation_identity=request.freeze_id,
resource=CapabilityResource(type="namespace", id=request.namespace_id),
)
body = _DifyIntegrationFreezeBody(
freezeId=request.freeze_id,
freezeRevision=request.freeze_revision,
sourceRevisionDigest=request.source_revision_digest,
sourceTaskWatermark=request.source_task_watermark,
)
response = self._request_json(
method="POST",
path="/internal/dify-integration/freeze",
token=token,
trace_id=request.freeze_id,
payload=_JSON_ADAPTER.validate_python(body.model_dump(mode="json", by_alias=True)),
expected_statuses=(HTTPStatus.OK,),
)
try:
parsed = _DifyIntegrationFreezeResponse.model_validate(response)
ack = KnowledgeFSDifyIntegrationFreezeAck(
namespace_id=parsed.namespace_id,
freeze_id=parsed.freeze_id,
freeze_revision=parsed.freeze_revision,
source_revision_digest=parsed.source_revision_digest,
source_task_watermark=parsed.source_task_watermark,
frozen_at=parsed.frozen_at,
updated_at=parsed.updated_at,
frozen=parsed.frozen,
applied=parsed.applied,
replayed=parsed.replayed,
)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID",
"KnowledgeFS returned an invalid integration freeze acknowledgement",
) from exc
if (
ack.namespace_id != request.namespace_id
or ack.freeze_id != request.freeze_id
or ack.freeze_revision != request.freeze_revision
or ack.source_revision_digest != request.source_revision_digest
or ack.source_task_watermark != request.source_task_watermark
):
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_FREEZE_ACK_MISMATCH",
"KnowledgeFS integration freeze acknowledgement did not match the request",
)
return ack
def find_by_provisioning_key(
self,
*,
provisioning_key: str,
control_space_id: str,
) -> KnowledgeFSRemoteSpace | None:
namespace_id = _namespace_from_provisioning_key(provisioning_key)
return next(
(
space
for space in self.list_spaces(
namespace_id=namespace_id,
control_space_id=control_space_id,
)
if space.provisioning_key == provisioning_key
),
None,
)
def list_spaces(
self,
*,
namespace_id: str | None = None,
control_space_id: str | None = None,
) -> tuple[KnowledgeFSRemoteSpace, ...]:
if namespace_id is None or not namespace_id.strip():
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_NAMESPACE_REQUIRED",
"KnowledgeFS reconciliation requires an explicit namespace",
)
if control_space_id is None or not control_space_id.strip():
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_CONTROL_SPACE_REQUIRED",
"KnowledgeFS reconciliation requires an auditable control-space",
)
namespace_id = namespace_id.strip()
token = self._issue_capability(
namespace_id=namespace_id,
control_space_id=control_space_id.strip(),
operation_id="listKnowledgeSpaces",
operation_identity=f"reconcile:{namespace_id}",
resource=CapabilityResource(type="namespace", id=namespace_id),
)
spaces: list[KnowledgeFSRemoteSpace] = []
cursor: str | None = None
consumed_cursors: set[str] = set()
while True:
query = (
(("limit", str(_PAGE_SIZE)),) if cursor is None else (("limit", str(_PAGE_SIZE)), ("cursor", cursor))
)
response = self._request_json(
method="GET",
path="/knowledge-spaces",
token=token,
trace_id=f"reconcile:{namespace_id}",
query=query,
payload=None,
expected_statuses=(HTTPStatus.OK,),
)
try:
page = _KnowledgeSpaceListResponse.model_validate(response)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid Space list"
) from exc
for item in page.items:
if item.tenant_id != namespace_id:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_SCOPE_MISMATCH", "KnowledgeFS list response crossed its namespace"
)
spaces.append(_remote_space(item, provisioning_key=f"dify:{namespace_id}:{item.slug}"))
if len(spaces) > _MAX_RECONCILIATION_SPACES:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RECONCILIATION_LIMIT",
"KnowledgeFS reconciliation exceeded its bounded Space limit",
)
cursor = page.next_cursor
if cursor is None:
break
if cursor in consumed_cursors:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_CURSOR_LOOP", "KnowledgeFS repeated a reconciliation cursor"
)
consumed_cursors.add(cursor)
return tuple(spaces)
def _issue_capability(
self,
*,
namespace_id: str,
control_space_id: str,
operation_id: str,
operation_identity: str,
resource: CapabilityResource,
) -> str:
issued = self._issuer.issue(
CapabilityIssueRequest(
actor=f"dify-worker:{_WORKER_PRINCIPAL}",
authz_revision=CapabilityAuthzRevision(
credential_revision=None,
external_access_epoch=0,
membership_epoch=0,
space_acl_epoch=0,
),
caller_kind="internal_worker",
content_policy_revision=0,
control_space_id=control_space_id,
grant_id=operation_identity,
namespace_id=namespace_id,
operation_id=operation_id,
principal_id=_WORKER_PRINCIPAL,
resource=resource,
trace_id=operation_identity,
)
)
return issued.token
def _request_json(
self,
*,
method: Literal["GET", "POST"],
path: str,
token: str,
trace_id: str,
payload: JsonValue | None,
expected_statuses: tuple[HTTPStatus, ...],
query: tuple[tuple[str, str], ...] = (),
) -> JsonValue:
headers = {
"Accept": "application/json",
"Accept-Encoding": "identity",
"Authorization": f"Bearer {token}",
"X-Trace-Id": trace_id,
}
request_kwargs: dict[str, object] = {
"follow_redirects": False,
"headers": headers,
"params": query,
"timeout": self._timeout_seconds,
}
if payload is not None:
headers["Content-Type"] = "application/json"
request_kwargs["json"] = payload
try:
upstream_url = httpx.URL(f"{self._base_url}/").join(path.lstrip("/"))
response = ssrf_proxy.make_request(
method=method,
url=str(upstream_url),
max_retries=0,
stream_response=True,
**request_kwargs,
)
response = ssrf_proxy.buffer_response(response, max_response_bytes=self._max_response_bytes)
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_REQUEST_FAILED", "KnowledgeFS lifecycle request failed"
) from exc
try:
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
if content_type != "application/json" and not content_type.endswith("+json"):
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_MEDIA_TYPE_INVALID",
"KnowledgeFS returned an unsupported lifecycle media type",
)
if response.status_code not in {int(status) for status in expected_statuses}:
raise KnowledgeFSLifecycleRemoteError(
f"KNOWLEDGE_FS_HTTP_{response.status_code}",
f"KnowledgeFS lifecycle request returned HTTP {response.status_code}",
)
try:
return _JSON_ADAPTER.validate_python(response.json())
except (ValueError, ValidationError) as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned invalid lifecycle JSON"
) from exc
finally:
response.close()
def _json_object(value: object, label: str) -> dict[str, JsonValue]:
try:
parsed = _JSON_ADAPTER.validate_python(value)
except ValidationError as exc:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_REQUEST_INVALID", f"KnowledgeFS {label} is not JSON-compatible"
) from exc
if not isinstance(parsed, dict):
raise KnowledgeFSLifecycleRemoteError("KNOWLEDGE_FS_REQUEST_INVALID", f"KnowledgeFS {label} must be an object")
return parsed
def _namespace_from_provisioning_key(provisioning_key: str) -> str:
parts = provisioning_key.split(":", 2)
if len(parts) != 3 or parts[0] != "dify" or not parts[1] or not parts[2]:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_PROVISIONING_KEY_INVALID",
"KnowledgeFS provisioning key does not identify a namespace",
)
return parts[1]
def _remote_space(space: _KnowledgeSpaceResponse, *, provisioning_key: str) -> KnowledgeFSRemoteSpace:
return KnowledgeFSRemoteSpace(
namespace_id=space.tenant_id,
knowledge_space_id=space.id,
provisioning_key=provisioning_key,
revision=space.revision,
)
__all__ = ["HTTPKnowledgeFSLifecycleRemoteClient"]
+627
View File
@@ -0,0 +1,627 @@
"""Lease-driven KnowledgeFS integrated provision and durable deletion saga."""
from __future__ import annotations
import logging
from datetime import datetime, timedelta
from typing import Literal, NamedTuple, cast
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import (
KnowledgeFSControlSpaceState,
KnowledgeFSDeleteCommandPayload,
KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
KnowledgeFSProvisionCommandPayload,
KnowledgeFSRevokeCommandPayload,
)
from repositories.knowledge_fs_control_space_repository import KnowledgeFSControlSpaceCASUpdate
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
SQLAlchemyKnowledgeFSControlSpaceRepository,
)
from repositories.sqlalchemy_knowledge_fs_lifecycle_outbox_repository import (
SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
)
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
from services.knowledge_fs.lifecycle_port import (
KnowledgeFSCapabilityGrantRevokeAck,
KnowledgeFSCapabilityGrantRevokeRequest,
KnowledgeFSDeletionPhase,
KnowledgeFSIntegratedDeletionRequest,
KnowledgeFSIntegratedProvisionRequest,
KnowledgeFSLifecycleRemoteError,
KnowledgeFSLifecycleRemotePort,
KnowledgeFSRemoteSpace,
)
from services.knowledge_fs.observability import (
KnowledgeFSLifecycleTaskMetric,
KnowledgeFSOperationalMetricsPort,
get_knowledge_fs_operational_metrics,
)
logger = logging.getLogger(__name__)
class KnowledgeFSLifecycleDispatchResult(NamedTuple):
claimed: bool
completed: bool
outbox_id: str | None
class KnowledgeFSLifecycleLeaseLostError(RuntimeError):
"""The command lease changed before the worker could settle its result."""
class KnowledgeFSLifecycleSagaRunner:
"""Claim, deliver, and settle one command without holding a DB transaction over HTTP."""
def __init__(
self,
session_maker: sessionmaker[Session],
remote: KnowledgeFSLifecycleRemotePort,
metrics: KnowledgeFSOperationalMetricsPort | None = None,
):
self._session_maker = session_maker
self._remote = remote
self._metrics = metrics or get_knowledge_fs_operational_metrics()
def dispatch_one(
self,
*,
worker_id: str,
now: datetime,
lease_duration: timedelta,
product_enabled: bool,
) -> KnowledgeFSLifecycleDispatchResult:
allowed_operations = tuple(KnowledgeFSLifecycleOperation)
if not product_enabled:
allowed_operations = tuple(
operation
for operation in allowed_operations
if operation is not KnowledgeFSLifecycleOperation.PROVISION
)
with self._session_maker.begin() as session:
command = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session).claim_next(
lease_owner=worker_id,
now=now,
lease_duration=lease_duration,
allowed_operations=allowed_operations,
)
if command is None:
return KnowledgeFSLifecycleDispatchResult(False, False, None)
claimed = _ClaimedCommand.from_model(command)
self._record_task(claimed, status="running")
try:
return self._dispatch_claimed(claimed=claimed, now=now, product_enabled=product_enabled)
except Exception:
self._record_task(claimed, status="dispatch_error")
raise
def _dispatch_claimed(
self,
*,
claimed: _ClaimedCommand,
now: datetime,
product_enabled: bool,
) -> KnowledgeFSLifecycleDispatchResult:
try:
if claimed.operation is KnowledgeFSLifecycleOperation.PROVISION:
if self._settle_provision_without_delivery(claimed=claimed, completed_at=now):
return self._result(claimed, completed=True, settled_at=now)
remote_space = self._remote.provision_integrated_space(_provision_request(claimed))
self._complete_provision(claimed=claimed, remote_space=remote_space, completed_at=now)
return self._result(claimed, completed=True, settled_at=now)
if claimed.operation is KnowledgeFSLifecycleOperation.DELETE:
if self._already_deleted(claimed=claimed, completed_at=now):
return self._result(claimed, completed=True, settled_at=now)
if not product_enabled and self._complete_never_provisioned_deletion(
claimed=claimed,
completed_at=now,
):
return self._result(claimed, completed=True, settled_at=now)
request = self._prepare_deletion_request(claimed)
if request is None:
if not product_enabled and self._complete_remote_absent_deletion(
claimed=claimed,
observed_at=now,
):
return self._result(claimed, completed=True, settled_at=now)
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_SPACE_NOT_FOUND",
"KnowledgeFS deletion identity could not be recovered",
)
progress = self._remote.request_integrated_deletion(request)
completed = self._settle_deletion(claimed=claimed, progress=progress, settled_at=now)
return self._result(claimed, completed=completed, settled_at=now)
if claimed.operation is KnowledgeFSLifecycleOperation.REVOKE:
acknowledgment = self._remote.revoke_capability_grant(_revoke_request(claimed))
completed = self._settle_revoke(
claimed=claimed,
acknowledgment=acknowledgment,
settled_at=now,
)
return self._result(claimed, completed=completed, settled_at=now)
self._retry(
claimed=claimed,
now=now,
error_code="UNSUPPORTED_OPERATION",
error_message=f"Lifecycle operation {claimed.operation} is not dispatched yet",
)
except KnowledgeFSLifecycleRemoteError as exc:
self._retry(claimed=claimed, now=now, error_code=exc.code, error_message=str(exc))
return self._result(claimed, completed=False, settled_at=now)
def _result(
self,
claimed: _ClaimedCommand,
*,
completed: bool,
settled_at: datetime,
) -> KnowledgeFSLifecycleDispatchResult:
self._record_task(
claimed,
status="succeeded" if completed else "retry",
settled_at=settled_at if completed else None,
)
return KnowledgeFSLifecycleDispatchResult(True, completed, claimed.outbox_id)
def _record_task(
self,
claimed: _ClaimedCommand,
*,
status: Literal["dispatch_error", "retry", "running", "succeeded"],
settled_at: datetime | None = None,
) -> None:
try:
duration_seconds = None
if settled_at is not None:
duration_seconds = max(0.0, (settled_at - claimed.created_at).total_seconds())
self._metrics.record_lifecycle_task(
KnowledgeFSLifecycleTaskMetric(duration_seconds, claimed.operation.value, status)
)
except Exception:
logger.warning(
"KnowledgeFS lifecycle metric export failed operation=%s status=%s",
claimed.operation.value,
status,
exc_info=True,
)
def _settle_provision_without_delivery(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
"""ACK a provision already applied or superseded before crossing the HTTP boundary."""
with self._session_maker.begin() as session:
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
still_owned = (
control_space is not None
and control_space.lifecycle_operation_id == claimed.operation_id
and control_space.state is KnowledgeFSControlSpaceState.PROVISIONING
)
if still_owned:
return False
self._ack(
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
claimed=claimed,
completed_at=completed_at,
)
return True
def _complete_never_provisioned_deletion(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
"""Terminally delete a local intent proven never to have crossed the remote boundary."""
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
control_space = control_repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if (
control_space is None
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
or control_space.lifecycle_operation_id != claimed.operation_id
or control_space.knowledge_space_id is not None
):
return False
provision = outbox_repository.find_open_for_control_space(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
operation=KnowledgeFSLifecycleOperation.PROVISION,
)
if (
provision is None
or provision.status is not KnowledgeFSLifecycleOutboxStatus.PENDING
or provision.attempt_count != 0
):
return False
if not outbox_repository.supersede_unattempted(
outbox_id=provision.id,
completed_at=completed_at,
error_code="SUPERSEDED_BY_DELETE",
error_message="Provision was canceled before delivery by permanent cleanup",
):
return False
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
expected_resource_version=control_space.resource_version,
new_state=KnowledgeFSControlSpaceState.DELETED,
lifecycle_operation_id=claimed.operation_id,
)
self._ack(outbox_repository, claimed=claimed, completed_at=completed_at)
return True
def _complete_remote_absent_deletion(self, *, claimed: _ClaimedCommand, observed_at: datetime) -> bool:
"""Finish cleanup after an authoritative remote listing proves the target absent."""
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
control_space = control_repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if (
control_space is None
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
or control_space.lifecycle_operation_id != claimed.operation_id
or control_space.knowledge_space_id is not None
):
return False
provision = outbox_repository.find_latest_for_control_space(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
operation=KnowledgeFSLifecycleOperation.PROVISION,
)
if provision is None or not outbox_repository.supersede_after_remote_absence(
outbox_id=provision.id,
observed_at=observed_at,
error_code="SUPERSEDED_BY_DELETE",
error_message="Remote absence was confirmed by permanent cleanup",
):
return False
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
expected_resource_version=control_space.resource_version,
new_state=KnowledgeFSControlSpaceState.DELETED,
lifecycle_operation_id=claimed.operation_id,
)
self._ack(outbox_repository, claimed=claimed, completed_at=observed_at)
return True
def _prepare_deletion_request(self, claimed: _ClaimedCommand) -> KnowledgeFSIntegratedDeletionRequest | None:
"""Resolve and durably register a lost-ACK remote identity before deletion admission."""
with self._session_maker() as session:
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if (
control_space is None
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
or control_space.lifecycle_operation_id != claimed.operation_id
):
raise KnowledgeFSLifecycleLeaseLostError("Delete command no longer owns the control-space")
if control_space.knowledge_space_id is not None:
return _resolved_deletion_request(
claimed,
knowledge_space_id=control_space.knowledge_space_id,
expected_revision=control_space.knowledge_space_revision,
)
payload = cast(KnowledgeFSDeleteCommandPayload, claimed.command_payload)
recovered = self._remote.find_by_provisioning_key(
provisioning_key=payload["provisioning_key"],
control_space_id=claimed.control_space_id,
)
if recovered is None:
return None
if recovered.namespace_id != claimed.tenant_id:
raise KnowledgeFSLifecycleRemoteError(
"KNOWLEDGE_FS_SCOPE_MISMATCH",
"KnowledgeFS recovered deletion identity crossed its namespace",
)
with self._session_maker.begin() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
control_space = repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if (
control_space is None
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
or control_space.lifecycle_operation_id != claimed.operation_id
):
raise KnowledgeFSLifecycleLeaseLostError("Delete command lost ownership during identity recovery")
if control_space.knowledge_space_id is None:
changed = repository.compare_and_set_lifecycle(
KnowledgeFSControlSpaceCASUpdate(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
expected_resource_version=control_space.resource_version,
expected_state=KnowledgeFSControlSpaceState.DELETING,
new_state=KnowledgeFSControlSpaceState.DELETING,
lifecycle_operation_id=claimed.operation_id,
knowledge_space_id=recovered.knowledge_space_id,
knowledge_space_revision=recovered.revision,
)
)
if not changed:
raise KnowledgeFSLifecycleLeaseLostError(
"Delete command lost ownership while registering its remote identity"
)
knowledge_space_id = recovered.knowledge_space_id
expected_revision = recovered.revision
else:
if control_space.knowledge_space_id != recovered.knowledge_space_id:
raise KnowledgeFSLifecycleLeaseLostError("Recovered deletion identity conflicts with registration")
knowledge_space_id = control_space.knowledge_space_id
expected_revision = control_space.knowledge_space_revision
return _resolved_deletion_request(
claimed,
knowledge_space_id=knowledge_space_id,
expected_revision=expected_revision,
)
def _complete_provision(
self,
*,
claimed: _ClaimedCommand,
remote_space: KnowledgeFSRemoteSpace,
completed_at: datetime,
) -> None:
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
control_space = control_repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if control_space is None or remote_space.namespace_id != claimed.tenant_id:
raise KnowledgeFSLifecycleLeaseLostError("Provision response did not match its Dify control-space")
already_applied = (
control_space.state is KnowledgeFSControlSpaceState.ACTIVE
and control_space.lifecycle_operation_id == claimed.operation_id
and control_space.knowledge_space_id == remote_space.knowledge_space_id
and control_space.knowledge_space_revision >= remote_space.revision
)
if not already_applied:
if control_space.lifecycle_operation_id != claimed.operation_id:
raise KnowledgeFSLifecycleLeaseLostError("Provision command no longer owns the control-space")
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
expected_resource_version=control_space.resource_version,
new_state=KnowledgeFSControlSpaceState.ACTIVE,
lifecycle_operation_id=claimed.operation_id,
knowledge_space_id=remote_space.knowledge_space_id,
knowledge_space_revision=remote_space.revision,
)
self._ack(outbox_repository, claimed=claimed, completed_at=completed_at)
def _already_deleted(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
with self._session_maker.begin() as session:
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if control_space is None or control_space.state is not KnowledgeFSControlSpaceState.DELETED:
return False
self._ack(
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
claimed=claimed,
completed_at=completed_at,
)
return True
def _settle_deletion(self, *, claimed: _ClaimedCommand, progress: object, settled_at: datetime) -> bool:
from services.knowledge_fs.lifecycle_port import KnowledgeFSDeletionProgress
if not isinstance(progress, KnowledgeFSDeletionProgress):
raise TypeError("Deletion port returned an invalid progress value")
with self._session_maker.begin() as session:
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
control_space = control_repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if control_space is None or control_space.lifecycle_operation_id != claimed.operation_id:
raise KnowledgeFSLifecycleLeaseLostError("Delete command no longer owns the control-space")
if progress.phase in {KnowledgeFSDeletionPhase.IRREVERSIBLE, KnowledgeFSDeletionPhase.COMPLETED}:
irreversible_at = progress.irreversible_at or settled_at
if control_space.deletion_irreversible_at is None:
KnowledgeFSControlSpaceLifecycleService(control_repository).mark_deletion_irreversible(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
lifecycle_operation_id=claimed.operation_id,
irreversible_at=irreversible_at,
)
control_space = control_repository.get(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
)
if control_space is None:
raise KnowledgeFSLifecycleLeaseLostError("Control-space disappeared during deletion")
if progress.phase is KnowledgeFSDeletionPhase.COMPLETED:
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
tenant_id=claimed.tenant_id,
control_space_id=claimed.control_space_id,
expected_resource_version=control_space.resource_version,
new_state=KnowledgeFSControlSpaceState.DELETED,
lifecycle_operation_id=claimed.operation_id,
knowledge_space_revision=max(control_space.knowledge_space_revision, progress.revision),
)
self._ack(outbox_repository, claimed=claimed, completed_at=settled_at)
return True
if not outbox_repository.schedule_retry(
outbox_id=claimed.outbox_id,
lease_owner=claimed.lease_owner,
expected_lease_expires_at=claimed.lease_expires_at,
next_attempt_at=settled_at + _retry_delay(claimed.attempt_count),
error_code="REMOTE_DELETION_PENDING",
error_message=f"KnowledgeFS deletion is {progress.phase}",
):
raise KnowledgeFSLifecycleLeaseLostError("Delete command lease was lost before retry scheduling")
return False
def _settle_revoke(
self,
*,
claimed: _ClaimedCommand,
acknowledgment: object,
settled_at: datetime,
) -> bool:
if not isinstance(acknowledgment, KnowledgeFSCapabilityGrantRevokeAck):
raise TypeError("Capability revoke port returned an invalid acknowledgment")
payload = cast(KnowledgeFSRevokeCommandPayload, claimed.command_payload)
if acknowledgment.state == "revoked" and acknowledgment.highest_revoke_sequence >= payload["revoke_sequence"]:
with self._session_maker.begin() as session:
self._ack(
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
claimed=claimed,
completed_at=settled_at,
)
return True
self._retry(
claimed=claimed,
now=settled_at,
error_code="REMOTE_REVOKE_WATERMARK_LAG",
error_message="KnowledgeFS grant revoke acknowledgment is below the command watermark",
)
return False
def _retry(self, *, claimed: _ClaimedCommand, now: datetime, error_code: str, error_message: str) -> None:
with self._session_maker.begin() as session:
changed = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session).schedule_retry(
outbox_id=claimed.outbox_id,
lease_owner=claimed.lease_owner,
expected_lease_expires_at=claimed.lease_expires_at,
next_attempt_at=now + _retry_delay(claimed.attempt_count),
error_code=error_code,
error_message=error_message,
)
if not changed:
raise KnowledgeFSLifecycleLeaseLostError("Lifecycle command lease was lost before retry scheduling")
@staticmethod
def _ack(
repository: SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
*,
claimed: _ClaimedCommand,
completed_at: datetime,
) -> None:
if not repository.acknowledge(
outbox_id=claimed.outbox_id,
lease_owner=claimed.lease_owner,
expected_lease_expires_at=claimed.lease_expires_at,
completed_at=completed_at,
):
raise KnowledgeFSLifecycleLeaseLostError("Lifecycle command lease was lost before ACK")
class _ClaimedCommand(NamedTuple):
outbox_id: str
tenant_id: str
control_space_id: str
operation_id: str
idempotency_key: str
operation: KnowledgeFSLifecycleOperation
command_payload: dict[str, object]
attempt_count: int
lease_owner: str
lease_expires_at: datetime
created_at: datetime
@classmethod
def from_model(cls, command: KnowledgeFSLifecycleOutbox) -> _ClaimedCommand:
if (
command.status is not KnowledgeFSLifecycleOutboxStatus.PROCESSING
or command.lease_owner is None
or command.lease_expires_at is None
):
raise KnowledgeFSLifecycleLeaseLostError("Claimed command did not carry a complete lease")
return cls(
command.id,
command.tenant_id,
command.control_space_id,
command.operation_id,
command.idempotency_key,
command.operation,
cast(dict[str, object], command.command_payload),
command.attempt_count,
command.lease_owner,
command.lease_expires_at,
command.created_at,
)
def _provision_request(command: _ClaimedCommand) -> KnowledgeFSIntegratedProvisionRequest:
payload = cast(KnowledgeFSProvisionCommandPayload, command.command_payload)
return KnowledgeFSIntegratedProvisionRequest(
namespace_id=command.tenant_id,
control_space_id=command.control_space_id,
operation_id=command.operation_id,
idempotency_key=command.idempotency_key,
provisioning_key=payload["provisioning_key"],
name=payload["name"],
slug=payload["slug"],
icon=payload["icon"],
description=payload["description"],
model_intent=payload["model_intent"],
profile_intent=payload["profile_intent"],
)
def _resolved_deletion_request(
command: _ClaimedCommand,
*,
knowledge_space_id: str,
expected_revision: int,
) -> KnowledgeFSIntegratedDeletionRequest:
payload = cast(KnowledgeFSDeleteCommandPayload, command.command_payload)
return KnowledgeFSIntegratedDeletionRequest(
namespace_id=command.tenant_id,
control_space_id=command.control_space_id,
operation_id=command.operation_id,
idempotency_key=command.idempotency_key,
knowledge_space_id=knowledge_space_id,
provisioning_key=payload["provisioning_key"],
expected_revision=expected_revision,
)
def _revoke_request(command: _ClaimedCommand) -> KnowledgeFSCapabilityGrantRevokeRequest:
payload = cast(KnowledgeFSRevokeCommandPayload, command.command_payload)
return KnowledgeFSCapabilityGrantRevokeRequest(
namespace_id=command.tenant_id,
control_space_id=command.control_space_id,
operation_id=command.operation_id,
idempotency_key=command.idempotency_key,
knowledge_space_id=payload["knowledge_space_id"],
grant_id=payload["grant_id"],
event_id=payload["event_id"],
reason_code=payload["reason_code"],
revoke_sequence=payload["revoke_sequence"],
expected_revision=payload["expected_revision"],
)
def _retry_delay(attempt_count: int) -> timedelta:
return timedelta(seconds=min(300, 2 ** min(attempt_count, 8)))
__all__ = [
"KnowledgeFSLifecycleDispatchResult",
"KnowledgeFSLifecycleLeaseLostError",
"KnowledgeFSLifecycleSagaRunner",
]
@@ -0,0 +1,250 @@
"""Transactional KnowledgeFS authorization effects of Dify Workspace membership changes."""
from __future__ import annotations
from collections.abc import Sequence
from enum import StrEnum
import sqlalchemy as sa
from sqlalchemy.orm import Session
from libs.datetime_utils import naive_utc_now
from models.knowledge_fs import (
KnowledgeFSAuthorizationRevision,
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpacePermissionStatus,
)
from services.knowledge_fs.revocation_commands import KnowledgeFSRevocationCommandProducer
_MAX_SAFE_INTEGER = 2**53 - 1
class KnowledgeFSWorkspaceMembershipChange(StrEnum):
MEMBER_ADDED = "member_added"
MEMBER_REMOVED = "member_removed"
ROLE_CHANGED = "role_changed"
class KnowledgeFSMembershipChangeInvariantError(RuntimeError):
"""Workspace membership state cannot be applied safely to KnowledgeFS authorization."""
def knowledge_fs_membership_schema_installed(session: Session) -> bool:
"""Support additive rollout and lightweight tests before the control-plane table exists."""
return sa.inspect(session.connection()).has_table(KnowledgeFSControlSpace.__tablename__)
def apply_workspace_membership_change(
*,
session: Session,
tenant_id: str,
actor_account_id: str,
account_ids: Sequence[str],
change: KnowledgeFSWorkspaceMembershipChange,
removed_account_id: str | None = None,
replacement_owner_account_id: str | None = None,
) -> None:
"""Advance every Space epoch and append revokes in the caller's membership transaction."""
normalized_tenant_id = _identifier(tenant_id, "tenant_id")
normalized_actor_id = _identifier(actor_account_id, "actor_account_id")
normalized_accounts = tuple(dict.fromkeys(_identifier(value, "account_id") for value in account_ids))
if not normalized_accounts:
raise KnowledgeFSMembershipChangeInvariantError("Membership change requires an affected account")
normalized_removed_id = _optional_identifier(removed_account_id, "removed_account_id")
normalized_replacement_id = _optional_identifier(
replacement_owner_account_id,
"replacement_owner_account_id",
)
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED:
if normalized_removed_id not in normalized_accounts or normalized_replacement_id is None:
raise KnowledgeFSMembershipChangeInvariantError(
"Member removal requires the removed account and replacement Workspace owner"
)
if normalized_removed_id == normalized_replacement_id:
raise KnowledgeFSMembershipChangeInvariantError("Removed member cannot remain the replacement owner")
elif normalized_removed_id is not None or normalized_replacement_id is not None:
raise KnowledgeFSMembershipChangeInvariantError("Owner replacement is valid only for member removal")
if not knowledge_fs_membership_schema_installed(session):
return
spaces, revisions = _lock_workspace_authorization(session, normalized_tenant_id)
if not spaces:
return
producer = KnowledgeFSRevocationCommandProducer()
for space in spaces:
revision = revisions[space.id]
if revision.membership_epoch >= _MAX_SAFE_INTEGER:
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS membership epoch is exhausted")
revision.membership_epoch += 1
acl_changed = False
if normalized_removed_id is not None and normalized_replacement_id is not None:
acl_changed = _remove_space_member(
session=session,
space=space,
removed_account_id=normalized_removed_id,
replacement_owner_account_id=normalized_replacement_id,
actor_account_id=normalized_actor_id,
)
if acl_changed:
if revision.space_acl_epoch >= _MAX_SAFE_INTEGER:
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS Space ACL epoch is exhausted")
revision.space_acl_epoch += 1
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_ADDED:
continue
reason_code = (
"workspace_membership_removed"
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED
else "workspace_role_changed"
)
for account_id in normalized_accounts:
producer.enqueue_principal_grants(
session=session,
tenant_id=normalized_tenant_id,
control_space_id=space.id,
subject=f"dify-account:{account_id}",
reason_code=reason_code,
caller_kinds=("interactive",),
)
def apply_workspace_rbac_role_change(*, session: Session, tenant_id: str) -> None:
"""Advance Workspace membership epochs and revoke every interactive grant.
Custom Enterprise RBAC role edits can affect any member bound to the role,
while Dify deliberately does not mirror those bindings locally. The safe
invalidation therefore scans durable grant provenance per control-space
instead of relying on a potentially incomplete member list.
"""
normalized_tenant_id = _identifier(tenant_id, "tenant_id")
if not knowledge_fs_membership_schema_installed(session):
return
spaces, revisions = _lock_workspace_authorization(session, normalized_tenant_id)
if not spaces:
return
producer = KnowledgeFSRevocationCommandProducer()
for space in spaces:
revision = revisions[space.id]
if revision.membership_epoch >= _MAX_SAFE_INTEGER:
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS membership epoch is exhausted")
revision.membership_epoch += 1
producer.enqueue_control_space_grants(
session=session,
tenant_id=normalized_tenant_id,
control_space_id=space.id,
reason_code="workspace_rbac_role_changed",
caller_kinds=("interactive",),
)
def _lock_workspace_authorization(
session: Session,
tenant_id: str,
) -> tuple[tuple[KnowledgeFSControlSpace, ...], dict[str, KnowledgeFSAuthorizationRevision]]:
spaces = tuple(
session.scalars(
sa.select(KnowledgeFSControlSpace)
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
.order_by(KnowledgeFSControlSpace.id)
.with_for_update()
)
)
if not spaces:
return (), {}
revisions = {
revision.control_space_id: revision
for revision in session.scalars(
sa.select(KnowledgeFSAuthorizationRevision)
.where(
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
KnowledgeFSAuthorizationRevision.control_space_id.in_(space.id for space in spaces),
)
.order_by(KnowledgeFSAuthorizationRevision.control_space_id)
.with_for_update()
)
}
if len(revisions) != len(spaces):
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS authorization revision is missing")
return spaces, revisions
def _remove_space_member(
*,
session: Session,
space: KnowledgeFSControlSpace,
removed_account_id: str,
replacement_owner_account_id: str,
actor_account_id: str,
) -> bool:
permissions = {
permission.account_id: permission
for permission in session.scalars(
sa.select(KnowledgeFSControlSpacePermission)
.where(
KnowledgeFSControlSpacePermission.tenant_id == space.tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == space.id,
KnowledgeFSControlSpacePermission.account_id.in_((removed_account_id, replacement_owner_account_id)),
)
.with_for_update()
)
}
changed = False
now = naive_utc_now()
removed_permission = permissions.get(removed_account_id)
if removed_permission is not None and (removed_permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE):
removed_permission.status = KnowledgeFSControlSpacePermissionStatus.REVOKED
removed_permission.revision += 1
removed_permission.revoked_at = now
removed_permission.revoked_by_account_id = actor_account_id
changed = True
if space.owner_account_id != removed_account_id:
return changed
space.owner_account_id = replacement_owner_account_id
space.resource_version += 1
replacement_permission = permissions.get(replacement_owner_account_id)
if replacement_permission is None:
session.add(
KnowledgeFSControlSpacePermission(
tenant_id=space.tenant_id,
control_space_id=space.id,
account_id=replacement_owner_account_id,
role=KnowledgeFSControlSpacePermissionRole.OWNER,
granted_by_account_id=actor_account_id,
)
)
elif (
replacement_permission.role is not KnowledgeFSControlSpacePermissionRole.OWNER
or replacement_permission.status is not KnowledgeFSControlSpacePermissionStatus.ACTIVE
):
replacement_permission.role = KnowledgeFSControlSpacePermissionRole.OWNER
replacement_permission.status = KnowledgeFSControlSpacePermissionStatus.ACTIVE
replacement_permission.revision += 1
replacement_permission.revoked_at = None
replacement_permission.revoked_by_account_id = None
return True
def _identifier(value: str, field: str) -> str:
normalized = value.strip()
if not normalized or normalized != value or len(normalized) > 255:
raise KnowledgeFSMembershipChangeInvariantError(f"{field} must be a normalized identifier")
return normalized
def _optional_identifier(value: str | None, field: str) -> str | None:
return None if value is None else _identifier(value, field)
__all__ = [
"KnowledgeFSMembershipChangeInvariantError",
"KnowledgeFSWorkspaceMembershipChange",
"apply_workspace_membership_change",
"apply_workspace_rbac_role_change",
"knowledge_fs_membership_schema_installed",
]
+234
View File
@@ -0,0 +1,234 @@
"""Low-cardinality operational metrics for the Dify side of KnowledgeFS.
The metric contracts intentionally exclude tenant, resource, principal, token, URL, and free-form
error values. Callers own best-effort failure isolation so telemetry can never change product state.
"""
from __future__ import annotations
from collections.abc import Callable, Iterable, Mapping
from functools import lru_cache
from typing import Literal, NamedTuple, Protocol
from opentelemetry.metrics import Observation, get_meter
class KnowledgeFSCapabilityIssuanceMetric(NamedTuple):
caller_kind: str
operation_id: str
outcome: Literal["denied", "failed", "issued"]
reason: str
class KnowledgeFSBatchStatusMetric(NamedTuple):
duration_seconds: float
missing_spaces: int
outcome: Literal["degraded", "failed", "success"]
requested_spaces: int
returned_spaces: int
class KnowledgeFSControlSpaceStateMetric(NamedTuple):
duration_seconds: float
from_state: str
to_state: str
class KnowledgeFSLifecycleTaskMetric(NamedTuple):
duration_seconds: float | None
operation: str
status: Literal["dispatch_error", "queued", "retry", "running", "succeeded"]
class KnowledgeFSOperationAdmissionMetric(NamedTuple):
operation_id: str
bucket: str
phase: Literal["commit", "refund", "reserve"]
outcome: Literal["failure", "success"]
class KnowledgeFSOperationalMetricsPort(Protocol):
def record_batch_status(self, event: KnowledgeFSBatchStatusMetric) -> None: ...
def record_capability_issuance(self, event: KnowledgeFSCapabilityIssuanceMetric) -> None: ...
def record_control_space_state(self, event: KnowledgeFSControlSpaceStateMetric) -> None: ...
def record_lifecycle_task(self, event: KnowledgeFSLifecycleTaskMetric) -> None: ...
def record_operation_admission(self, event: KnowledgeFSOperationAdmissionMetric) -> None: ...
def register_control_space_state_gauge(self, read_counts: Callable[[], Mapping[str, int]]) -> None: ...
class _CounterPort(Protocol):
def add(self, amount: int, *, attributes: dict[str, str]) -> None: ...
class _HistogramPort(Protocol):
def record(self, amount: float, *, attributes: dict[str, str]) -> None: ...
class _MeterPort(Protocol):
def create_counter(self, name: str, *, description: str, unit: str) -> _CounterPort: ...
def create_histogram(self, name: str, *, description: str, unit: str) -> _HistogramPort: ...
def create_observable_gauge(
self,
name: str,
*,
callbacks: tuple[Callable[[object], Iterable[Observation]], ...],
description: str,
unit: str,
) -> object: ...
class OpenTelemetryKnowledgeFSOperationalMetrics:
"""Map sanitized events to OpenTelemetry counters and duration histograms."""
def __init__(self, *, meter: _MeterPort | None = None) -> None:
resolved_meter = meter or get_meter("dify.knowledge_fs")
self._meter = resolved_meter
self._control_space_state_gauge: object | None = None
self._capability_issuance = resolved_meter.create_counter(
"dify.knowledge_fs.capability_issuance",
description="KnowledgeFS capability issuance outcomes",
unit="{issuance}",
)
self._batch_requests = resolved_meter.create_counter(
"dify.knowledge_fs.batch_status_requests",
description="KnowledgeFS batch status request outcomes",
unit="{request}",
)
self._batch_spaces = resolved_meter.create_counter(
"dify.knowledge_fs.batch_status_spaces",
description="KnowledgeFS batch status returned and missing spaces",
unit="{space}",
)
self._batch_latency = resolved_meter.create_histogram(
"dify.knowledge_fs.batch_status_latency",
description="KnowledgeFS batch status latency",
unit="s",
)
self._control_space_transitions = resolved_meter.create_counter(
"dify.knowledge_fs.control_space_transitions",
description="KnowledgeFS control-space lifecycle transitions",
unit="{transition}",
)
self._control_space_state_duration = resolved_meter.create_histogram(
"dify.knowledge_fs.control_space_state_duration",
description="Time spent in a KnowledgeFS control-space state before transition",
unit="s",
)
self._lifecycle_tasks = resolved_meter.create_counter(
"dify.knowledge_fs.lifecycle_tasks",
description="KnowledgeFS durable lifecycle task state observations",
unit="{task}",
)
self._lifecycle_task_latency = resolved_meter.create_histogram(
"dify.knowledge_fs.lifecycle_task_latency",
description="KnowledgeFS durable lifecycle task terminal latency",
unit="s",
)
self._revoke_latency = resolved_meter.create_histogram(
"dify.knowledge_fs.revoke_latency",
description="KnowledgeFS capability revoke enqueue-to-ack latency",
unit="s",
)
self._operation_admission = resolved_meter.create_counter(
"dify.knowledge_fs.operation_admission",
description="KnowledgeFS direct-operation reserve and finalization outcomes",
unit="{operation}",
)
def record_capability_issuance(self, event: KnowledgeFSCapabilityIssuanceMetric) -> None:
self._capability_issuance.add(
1,
attributes={
"caller_kind": event.caller_kind,
"operation_id": event.operation_id,
"outcome": event.outcome,
"reason": event.reason,
},
)
def record_batch_status(self, event: KnowledgeFSBatchStatusMetric) -> None:
outcome: dict[str, str] = {"outcome": event.outcome}
self._batch_requests.add(1, attributes=outcome)
self._batch_latency.record(event.duration_seconds, attributes=outcome)
self._batch_spaces.add(event.returned_spaces, attributes={"result": "returned"})
self._batch_spaces.add(event.missing_spaces, attributes={"result": "missing"})
def record_control_space_state(self, event: KnowledgeFSControlSpaceStateMetric) -> None:
attributes = {"from_state": event.from_state, "to_state": event.to_state}
self._control_space_transitions.add(1, attributes=attributes)
self._control_space_state_duration.record(event.duration_seconds, attributes=attributes)
def record_lifecycle_task(self, event: KnowledgeFSLifecycleTaskMetric) -> None:
attributes = {"operation": event.operation, "status": event.status}
self._lifecycle_tasks.add(1, attributes=attributes)
if event.duration_seconds is None:
return
self._lifecycle_task_latency.record(event.duration_seconds, attributes=attributes)
if event.operation == "revoke" and event.status == "succeeded":
self._revoke_latency.record(event.duration_seconds, attributes={"operation": "revoke"})
def record_operation_admission(self, event: KnowledgeFSOperationAdmissionMetric) -> None:
self._operation_admission.add(
1,
attributes={
"bucket": event.bucket,
"operation_id": event.operation_id,
"outcome": event.outcome,
"phase": event.phase,
},
)
def register_control_space_state_gauge(self, read_counts: Callable[[], Mapping[str, int]]) -> None:
"""Register one DB-backed current-state instrument per process."""
if self._control_space_state_gauge is not None:
return
def observe(_: object) -> Iterable[Observation]:
try:
counts = read_counts()
return tuple(
Observation(
max(0, counts.get(state, 0)),
attributes={"aggregation_scope": "global_database_snapshot", "state": state},
)
for state in ("provisioning", "deleting", "error")
)
except Exception:
return ()
self._control_space_state_gauge = self._meter.create_observable_gauge(
"dify.knowledge_fs.control_spaces",
callbacks=(observe,),
description=(
"Global KnowledgeFS control-space database snapshot by operational state; "
"aggregate replica series with max, never sum"
),
unit="{space}",
)
@lru_cache(maxsize=1)
def get_knowledge_fs_operational_metrics() -> KnowledgeFSOperationalMetricsPort:
"""Return one process-wide instrument set to avoid duplicate OpenTelemetry registration."""
return OpenTelemetryKnowledgeFSOperationalMetrics()
__all__ = [
"KnowledgeFSBatchStatusMetric",
"KnowledgeFSCapabilityIssuanceMetric",
"KnowledgeFSControlSpaceStateMetric",
"KnowledgeFSLifecycleTaskMetric",
"KnowledgeFSOperationAdmissionMetric",
"KnowledgeFSOperationalMetricsPort",
"OpenTelemetryKnowledgeFSOperationalMetrics",
"get_knowledge_fs_operational_metrics",
]
@@ -0,0 +1,408 @@
"""Weighted admission for BFF calls and browser-executed KnowledgeFS operations."""
from __future__ import annotations
import logging
import time
import uuid
from collections.abc import Callable
from typing import Literal, NamedTuple, Protocol, cast
from configs import dify_config
from extensions.ext_redis import redis_client
from services.billing_service import BillingService
from services.feature_service import FeatureService, KnowledgeRateLimitModel
from services.knowledge_fs.capability_broker import KnowledgeFSIssuedProductCapability
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
from services.knowledge_fs.observability import (
KnowledgeFSOperationAdmissionMetric,
KnowledgeFSOperationalMetricsPort,
get_knowledge_fs_operational_metrics,
)
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS
logger = logging.getLogger(__name__)
_BILLING_FEATURE_KEY = "knowledge_fs_operations"
_RATE_LIMIT_WINDOW_MS = 60_000
_WEIGHTED_RATE_LIMIT_SCRIPT = """
redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[2])
local current = redis.call('ZCARD', KEYS[1])
local requested = tonumber(ARGV[4])
if current + requested > tonumber(ARGV[3]) then
return 0
end
for index = 1, requested do
redis.call('ZADD', KEYS[1], ARGV[1], ARGV[5] .. ':' .. index)
end
redis.call('PEXPIRE', KEYS[1], 61000)
return 1
"""
class KnowledgeFSOperationRateLimitExceededError(RuntimeError):
"""The operation's weighted per-minute bucket is exhausted."""
class KnowledgeFSOperationQuotaExceededError(RuntimeError):
"""The billing service explicitly rejected an operation reservation."""
class KnowledgeFSOperationUsage(NamedTuple):
tenant_id: str
operation_id: str
bucket: str
billing_cost: int
rate_limit_cost: int
class KnowledgeFSOperationChargePort(Protocol):
def commit(self) -> None: ...
def refund(self) -> None: ...
class KnowledgeFSOperationRateLimitPort(Protocol):
def admit(self, usage: KnowledgeFSOperationUsage) -> None: ...
class KnowledgeFSOperationBillingPort(Protocol):
def reserve(self, usage: KnowledgeFSOperationUsage) -> KnowledgeFSOperationChargePort: ...
class KnowledgeFSRateLimitAuditPort(Protocol):
def record_rejection(self, usage: KnowledgeFSOperationUsage, *, subscription_plan: str) -> None: ...
class KnowledgeFSBillingGateway(Protocol):
def quota_reserve(self, **kwargs: object) -> dict[str, object]: ...
def quota_commit(self, **kwargs: object) -> dict[str, object]: ...
def quota_release(self, **kwargs: object) -> dict[str, object]: ...
class KnowledgeFSRedisEvalPort(Protocol):
def eval(self, *args: object) -> int: ...
class _NoopCharge:
def commit(self) -> None:
return
def refund(self) -> None:
return
class _DifyBillingCharge:
def __init__(
self,
*,
gateway: KnowledgeFSBillingGateway,
usage: KnowledgeFSOperationUsage,
reservation_id: str,
) -> None:
self._gateway = gateway
self._usage = usage
self._reservation_id = reservation_id
self._finalized = False
def commit(self) -> None:
if self._finalized:
return
self._finalized = True
try:
self._gateway.quota_commit(
tenant_id=self._usage.tenant_id,
feature_key=_BILLING_FEATURE_KEY,
bucket=self._usage.bucket,
reservation_id=self._reservation_id,
actual_amount=self._usage.billing_cost,
meta={"operation_id": self._usage.operation_id, "source": "knowledge_fs"},
)
except Exception:
logger.exception(
"KnowledgeFS billing commit failed for tenant_id=%s operation_id=%s",
self._usage.tenant_id,
self._usage.operation_id,
)
def refund(self) -> None:
if self._finalized:
return
self._finalized = True
try:
self._gateway.quota_release(
tenant_id=self._usage.tenant_id,
feature_key=_BILLING_FEATURE_KEY,
bucket=self._usage.bucket,
reservation_id=self._reservation_id,
)
except Exception:
logger.exception(
"KnowledgeFS billing release failed for tenant_id=%s operation_id=%s",
self._usage.tenant_id,
self._usage.operation_id,
)
class LoggingKnowledgeFSRateLimitAudit:
def record_rejection(self, usage: KnowledgeFSOperationUsage, *, subscription_plan: str) -> None:
logger.warning(
"KnowledgeFS weighted rate limit rejected tenant_id=%s operation_id=%s bucket=%s cost=%s plan=%s",
usage.tenant_id,
usage.operation_id,
usage.bucket,
usage.rate_limit_cost,
subscription_plan,
)
class DifyKnowledgeFSWeightedRateLimitPort:
"""Consume weighted operation units atomically from one Redis bucket."""
def __init__(
self,
*,
redis: KnowledgeFSRedisEvalPort | None = None,
audit: KnowledgeFSRateLimitAuditPort,
rate_limit_lookup: Callable[[str], KnowledgeRateLimitModel] = FeatureService.get_knowledge_rate_limit,
clock_ms: Callable[[], int] = lambda: int(time.time() * 1000),
member_id: Callable[[], str] = lambda: str(uuid.uuid4()),
) -> None:
self._redis = redis or cast(KnowledgeFSRedisEvalPort, redis_client)
self._audit = audit
self._rate_limit_lookup = rate_limit_lookup
self._clock_ms = clock_ms
self._member_id = member_id
def admit(self, usage: KnowledgeFSOperationUsage) -> None:
rate_limit = self._rate_limit_lookup(usage.tenant_id)
if not rate_limit.enabled:
return
now = self._clock_ms()
accepted = self._redis.eval(
_WEIGHTED_RATE_LIMIT_SCRIPT,
1,
f"knowledge_fs:rate_limit:{usage.tenant_id}:{usage.bucket}",
now,
now - _RATE_LIMIT_WINDOW_MS,
rate_limit.limit,
usage.rate_limit_cost,
self._member_id(),
)
if accepted == 1:
return
self._audit.record_rejection(usage, subscription_plan=rate_limit.subscription_plan)
raise KnowledgeFSOperationRateLimitExceededError("KnowledgeFS operation rate limit exceeded")
class DifyKnowledgeFSBillingPort:
"""Reserve operation-specific billing units for the caller to commit or release."""
def __init__(
self,
*,
gateway: KnowledgeFSBillingGateway | None = None,
billing_enabled: Callable[[], bool] = lambda: dify_config.BILLING_ENABLED,
request_id: Callable[[], str] = lambda: str(uuid.uuid4()),
) -> None:
self._gateway = gateway or cast(KnowledgeFSBillingGateway, BillingService)
self._billing_enabled = billing_enabled
self._request_id = request_id
def reserve(self, usage: KnowledgeFSOperationUsage) -> KnowledgeFSOperationChargePort:
if not self._billing_enabled():
return _NoopCharge()
try:
result = self._gateway.quota_reserve(
tenant_id=usage.tenant_id,
feature_key=_BILLING_FEATURE_KEY,
bucket=usage.bucket,
request_id=self._request_id(),
amount=usage.billing_cost,
meta={"operation_id": usage.operation_id, "source": "knowledge_fs"},
)
except Exception:
logger.exception(
"KnowledgeFS billing reservation unavailable for tenant_id=%s operation_id=%s; allowing request",
usage.tenant_id,
usage.operation_id,
)
return _NoopCharge()
reservation_id = result.get("reservation_id")
if not isinstance(reservation_id, str) or not reservation_id:
raise KnowledgeFSOperationQuotaExceededError("KnowledgeFS operation quota exceeded")
return _DifyBillingCharge(gateway=self._gateway, usage=usage, reservation_id=reservation_id)
class KnowledgeFSOperationAdmissionService:
def __init__(
self,
*,
rate_limit: KnowledgeFSOperationRateLimitPort,
billing: KnowledgeFSOperationBillingPort,
) -> None:
self._rate_limit = rate_limit
self._billing = billing
def reserve(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSOperationChargePort:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
usage = KnowledgeFSOperationUsage(
tenant_id=tenant_id,
operation_id=operation_id,
bucket=operation.rate_limit_bucket,
billing_cost=operation.billing_cost,
rate_limit_cost=operation.rate_limit_cost,
)
self._rate_limit.admit(usage)
return self._billing.reserve(usage)
class KnowledgeFSDirectCapabilityBrokerPort(Protocol):
def issue_interactive(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability: ...
def issue_service(
self,
*,
profile: KnowledgeFSServiceCredentialProfile,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability: ...
class KnowledgeFSDirectOperationAdmissionService:
"""Admit browser-executed operations at the Dify Capability issuance boundary.
A successful Capability issuance is the only outcome Dify can observe before the browser
talks to KnowledgeFS, so it commits the operation charge immediately. Issuance failures
release the reservation. Controllers must treat alternative endpoints for the same manifest
operation as separate entry points and must never chain them for one user action.
"""
_admission: KnowledgeFSOperationAdmissionService
_broker: KnowledgeFSDirectCapabilityBrokerPort
_metrics: KnowledgeFSOperationalMetricsPort
def __init__(
self,
*,
admission: KnowledgeFSOperationAdmissionService,
broker: KnowledgeFSDirectCapabilityBrokerPort,
metrics: KnowledgeFSOperationalMetricsPort | None = None,
) -> None:
self._admission = admission
self._broker = broker
self._metrics = metrics or get_knowledge_fs_operational_metrics()
def issue_interactive(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
return self._issue(
tenant_id=tenant_id,
operation_id=operation_id,
issue=lambda: self._broker.issue_interactive(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
operation_id=operation_id,
resource_id=resource_id,
trace_id=trace_id,
),
)
def issue_service(
self,
*,
profile: KnowledgeFSServiceCredentialProfile,
operation_id: str,
resource_id: str | None = None,
trace_id: str | None = None,
) -> KnowledgeFSIssuedProductCapability:
return self._issue(
tenant_id=profile.tenant_id,
operation_id=operation_id,
issue=lambda: self._broker.issue_service(
profile=profile,
operation_id=operation_id,
resource_id=resource_id,
trace_id=trace_id,
),
)
def _issue(
self,
*,
tenant_id: str,
operation_id: str,
issue: Callable[[], KnowledgeFSIssuedProductCapability],
) -> KnowledgeFSIssuedProductCapability:
bucket = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].rate_limit_bucket
try:
charge = self._admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
except BaseException:
self._record_metric(operation_id, bucket=bucket, phase="reserve", outcome="failure")
raise
self._record_metric(operation_id, bucket=bucket, phase="reserve", outcome="success")
try:
issued = issue()
except BaseException:
try:
charge.refund()
except BaseException:
self._record_metric(operation_id, bucket=bucket, phase="refund", outcome="failure")
raise
self._record_metric(operation_id, bucket=bucket, phase="refund", outcome="success")
raise
try:
charge.commit()
except BaseException:
self._record_metric(operation_id, bucket=bucket, phase="commit", outcome="failure")
raise
self._record_metric(operation_id, bucket=bucket, phase="commit", outcome="success")
return issued
def _record_metric(
self,
operation_id: str,
*,
bucket: str,
phase: Literal["commit", "refund", "reserve"],
outcome: Literal["failure", "success"],
) -> None:
try:
self._metrics.record_operation_admission(
KnowledgeFSOperationAdmissionMetric(operation_id, bucket, phase, outcome)
)
except Exception:
logger.warning("KnowledgeFS direct-operation admission metric export failed", exc_info=True)
__all__ = [
"DifyKnowledgeFSBillingPort",
"DifyKnowledgeFSWeightedRateLimitPort",
"KnowledgeFSDirectOperationAdmissionService",
"KnowledgeFSOperationAdmissionService",
"KnowledgeFSOperationChargePort",
"KnowledgeFSOperationQuotaExceededError",
"KnowledgeFSOperationRateLimitExceededError",
"KnowledgeFSOperationUsage",
"LoggingKnowledgeFSRateLimitAudit",
]
@@ -0,0 +1,112 @@
"""Conservative local/remote reconciliation for KnowledgeFS control-spaces."""
from __future__ import annotations
import uuid
from typing import NamedTuple
from sqlalchemy.orm import Session, sessionmaker
from models.knowledge_fs import KnowledgeFSControlSpaceState
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
SQLAlchemyKnowledgeFSControlSpaceRepository,
)
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
from services.knowledge_fs.lifecycle_port import KnowledgeFSLifecycleRemotePort
class KnowledgeFSOrphanReport(NamedTuple):
repaired_control_space_ids: tuple[str, ...]
remote_orphan_space_ids: tuple[str, ...]
local_missing_remote_ids: tuple[str, ...]
cleanup_control_space_ids: tuple[str, ...]
class KnowledgeFSOrphanReconciler:
"""Repair identities proven by provisioning_key; report ambiguous drift fail-closed."""
def __init__(self, session_maker: sessionmaker[Session], remote: KnowledgeFSLifecycleRemotePort):
self._session_maker = session_maker
self._remote = remote
def reconcile(self, *, limit: int = 500, apply_repairs: bool = True) -> KnowledgeFSOrphanReport:
with self._session_maker() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
local_spaces = repository.list_for_reconciliation(limit=limit)
control_space_by_tenant: dict[str, str] = {}
for space in local_spaces:
control_space_by_tenant.setdefault(space.tenant_id, space.id)
local_keys = {
space.provisioning_key
for tenant_id in control_space_by_tenant
for space in repository.list_for_tenant(tenant_id=tenant_id)
if space.state is not KnowledgeFSControlSpaceState.DELETED
}
remote_spaces = tuple(
space
for tenant_id, control_space_id in sorted(control_space_by_tenant.items())
for space in self._remote.list_spaces(
namespace_id=tenant_id,
control_space_id=control_space_id,
)
)
remote_by_key = {space.provisioning_key: space for space in remote_spaces}
repaired: list[str] = []
local_missing: list[str] = []
cleanup: list[str] = []
for snapshot in local_spaces:
remote_space = remote_by_key.get(snapshot.provisioning_key)
if apply_repairs and (
remote_space is not None
and remote_space.namespace_id == snapshot.tenant_id
and snapshot.state is KnowledgeFSControlSpaceState.PROVISIONING
and snapshot.deletion_irreversible_at is None
):
with self._session_maker.begin() as session:
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
current = repository.get(tenant_id=snapshot.tenant_id, control_space_id=snapshot.id)
if current is not None and current.resource_version == snapshot.resource_version:
KnowledgeFSControlSpaceLifecycleService(repository).transition(
tenant_id=snapshot.tenant_id,
control_space_id=snapshot.id,
expected_resource_version=snapshot.resource_version,
new_state=KnowledgeFSControlSpaceState.ACTIVE,
lifecycle_operation_id=snapshot.lifecycle_operation_id or "orphan-reconcile",
knowledge_space_id=remote_space.knowledge_space_id,
knowledge_space_revision=remote_space.revision,
)
repaired.append(snapshot.id)
continue
if apply_repairs and snapshot.state is KnowledgeFSControlSpaceState.DELETING:
operation_id = str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f"dify-kfs-orphan-cleanup:{snapshot.tenant_id}:{snapshot.id}:{snapshot.resource_version}",
)
)
result = KnowledgeFSControlSpaceCommandService(self._session_maker).request_deletion(
tenant_id=snapshot.tenant_id,
control_space_id=snapshot.id,
operation_id=operation_id,
idempotency_key=f"orphan-cleanup:{snapshot.tenant_id}:{snapshot.id}",
)
if result.outbox is not None:
cleanup.append(snapshot.id)
continue
if snapshot.knowledge_space_id is not None and remote_space is None:
local_missing.append(snapshot.id)
remote_orphans = tuple(
sorted(space.knowledge_space_id for space in remote_spaces if space.provisioning_key not in local_keys)
)
return KnowledgeFSOrphanReport(
repaired_control_space_ids=tuple(sorted(repaired)),
remote_orphan_space_ids=remote_orphans,
local_missing_remote_ids=tuple(sorted(local_missing)),
cleanup_control_space_ids=tuple(sorted(cleanup)),
)
__all__ = ["KnowledgeFSOrphanReconciler", "KnowledgeFSOrphanReport"]
@@ -0,0 +1,191 @@
"""Product-level lifecycle and metadata use cases for KnowledgeFS control-spaces."""
from __future__ import annotations
import uuid
from models.knowledge_fs import (
KnowledgeFSControlSpaceVisibility,
KnowledgeFSModelSelectionIntentPayload,
KnowledgeFSRerankIntentPayload,
KnowledgeFSRetrievalProfileIntentPayload,
KnowledgeFSScoreThresholdIntentPayload,
)
from services.knowledge_fs.control_plane_service import KnowledgeFSControlPlaneService
from services.knowledge_fs.control_space_commands import (
KnowledgeFSControlSpaceCommandService,
KnowledgeFSProvisionIntent,
)
from services.knowledge_fs.data_facade import KnowledgeFSDataFacade
from services.knowledge_fs.product_authorization import KnowledgeFSProductRBACPort
from services.knowledge_fs.product_dto import (
KnowledgeFSModelIntent,
KnowledgeFSRetrievalProfileIntent,
KnowledgeFSSpaceCreatePayload,
KnowledgeFSSpaceCreateResponse,
KnowledgeFSSpaceDetailResponse,
KnowledgeFSSpaceListResponse,
KnowledgeFSSpaceUpdatePayload,
)
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
from services.knowledge_fs.product_service import KnowledgeFSProductService
class KnowledgeFSProductApplicationService:
def __init__(
self,
*,
product: KnowledgeFSProductService,
control_plane: KnowledgeFSControlPlaneService,
commands: KnowledgeFSControlSpaceCommandService,
facade: KnowledgeFSDataFacade,
rbac: KnowledgeFSProductRBACPort,
) -> None:
self._product = product
self._control_plane = control_plane
self._commands = commands
self._facade = facade
self._rbac = rbac
def list_spaces(self, *, tenant_id: str, account_id: str, page: int, limit: int) -> KnowledgeFSSpaceListResponse:
return self._product.list_spaces(
tenant_id=tenant_id,
account_id=account_id,
page=page,
limit=limit,
)
def create_space(
self,
*,
tenant_id: str,
account_id: str,
payload: KnowledgeFSSpaceCreatePayload,
) -> KnowledgeFSSpaceCreateResponse:
self._product.require_product_routes(tenant_id=tenant_id)
if not self._rbac.workspace_permission_allowed(
tenant_id=tenant_id,
account_id=account_id,
permission=KnowledgeFSProductPermission.CREATE,
):
raise PermissionError("KnowledgeFS space creation is not allowed")
idempotency_key = payload.idempotency_key or str(uuid.uuid4())
operation_id = str(
uuid.uuid5(uuid.NAMESPACE_URL, f"dify-kfs-provision:{tenant_id}:{account_id}:{idempotency_key}")
)
result = self._commands.create_provision_intent(
KnowledgeFSProvisionIntent(
tenant_id=tenant_id,
owner_account_id=account_id,
provisioning_key=f"dify:{tenant_id}:{payload.slug}",
operation_id=operation_id,
idempotency_key=idempotency_key,
name=payload.name,
slug=payload.slug,
icon=payload.icon,
description=payload.description,
model_intent=_model_intent(payload.embedding),
profile_intent=_retrieval_profile_intent(payload.retrieval),
)
)
if payload.visibility is not KnowledgeFSControlSpaceVisibility.ONLY_ME:
self._control_plane.update_visibility(
tenant_id=tenant_id,
actor_account_id=account_id,
control_space_id=result.control_space.id,
visibility=payload.visibility,
)
return KnowledgeFSSpaceCreateResponse(
control_space_id=result.control_space.id,
state=result.control_space.state,
operation_id=operation_id,
)
def get_space(self, *, tenant_id: str, account_id: str, control_space_id: str) -> KnowledgeFSSpaceDetailResponse:
return self._product.get_space(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
)
def update_space(
self,
*,
tenant_id: str,
account_id: str,
control_space_id: str,
payload: KnowledgeFSSpaceUpdatePayload,
) -> KnowledgeFSSpaceDetailResponse:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.EDIT,
)
if payload.visibility is not None:
self._control_plane.update_visibility(
tenant_id=tenant_id,
actor_account_id=account_id,
control_space_id=control_space_id,
visibility=payload.visibility,
)
metadata = payload.model_copy(update={"visibility": None})
if any(value is not None for value in (metadata.name, metadata.icon, metadata.description)):
self._facade.update_space(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
payload=metadata,
)
return self.get_space(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
)
def delete_space(self, *, tenant_id: str, account_id: str, control_space_id: str) -> None:
self._product.authorize_control_space(
tenant_id=tenant_id,
account_id=account_id,
control_space_id=control_space_id,
permission=KnowledgeFSProductPermission.DELETE,
)
operation_id = str(uuid.uuid4())
self._commands.request_deletion(
tenant_id=tenant_id,
control_space_id=control_space_id,
operation_id=operation_id,
idempotency_key=f"delete:{operation_id}",
)
def _model_intent(model: KnowledgeFSModelIntent) -> KnowledgeFSModelSelectionIntentPayload:
return {
"pluginId": model.plugin_id,
"provider": model.provider,
"model": model.model,
}
def _retrieval_profile_intent(
profile: KnowledgeFSRetrievalProfileIntent,
) -> KnowledgeFSRetrievalProfileIntentPayload:
rerank: KnowledgeFSRerankIntentPayload = {"enabled": profile.rerank.enabled}
if profile.rerank.model is not None:
rerank["model"] = _model_intent(profile.rerank.model)
score_threshold: KnowledgeFSScoreThresholdIntentPayload = {
"enabled": profile.score_threshold.enabled,
"stage": profile.score_threshold.stage,
}
if profile.score_threshold.value is not None:
score_threshold["value"] = profile.score_threshold.value
return {
"defaultMode": profile.default_mode,
"reasoningModel": _model_intent(profile.reasoning_model),
"rerank": rerank,
"scoreThreshold": score_threshold,
"topK": profile.top_k,
}
__all__ = ["KnowledgeFSProductApplicationService"]
@@ -0,0 +1,260 @@
"""Dify-owned visibility, role, and enterprise RBAC authorization for KnowledgeFS."""
from __future__ import annotations
import logging
from collections.abc import Sequence
from typing import Protocol
import sqlalchemy as sa
from sqlalchemy.orm import Session
from configs import dify_config
from models.knowledge_fs import (
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpacePermissionRole,
KnowledgeFSControlSpacePermissionStatus,
KnowledgeFSControlSpaceState,
KnowledgeFSControlSpaceVisibility,
)
from services.enterprise.rbac_service import RBACService
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
logger = logging.getLogger(__name__)
class KnowledgeFSProductNotFoundError(RuntimeError):
"""A missing and an unauthorized control-space deliberately share one error."""
class KnowledgeFSProductRBACPort(Protocol):
def permission_keys_by_control_space(
self,
*,
session: Session,
tenant_id: str,
account_id: str,
control_space_ids: Sequence[str],
) -> dict[str, frozenset[KnowledgeFSProductPermission]]: ...
def filter_authorized_control_space_ids(
self,
*,
session: Session,
tenant_id: str,
account_id: str,
control_space_ids: Sequence[str],
permission: KnowledgeFSProductPermission,
) -> frozenset[str]: ...
def workspace_permission_allowed(
self,
*,
tenant_id: str,
account_id: str,
permission: KnowledgeFSProductPermission,
) -> bool: ...
class DifyKnowledgeFSProductRBACPort:
"""Use one Enterprise batch request for a page; fail closed if it is unavailable."""
def permission_keys_by_control_space(
self,
*,
session: Session,
tenant_id: str,
account_id: str,
control_space_ids: Sequence[str],
) -> dict[str, frozenset[KnowledgeFSProductPermission]]:
if not control_space_ids:
return {}
try:
permissions = RBACService.KnowledgeFSPermissions.batch_get(
tenant_id,
account_id,
list(control_space_ids),
session=session,
)
except Exception:
logger.warning("KnowledgeFS enterprise RBAC batch lookup failed for tenant_id=%s", tenant_id, exc_info=True)
return {}
return {
control_space_id: frozenset(
permission
for permission in KnowledgeFSProductPermission
if permission.value in permissions.get(control_space_id, ())
)
for control_space_id in control_space_ids
}
def filter_authorized_control_space_ids(
self,
*,
session: Session,
tenant_id: str,
account_id: str,
control_space_ids: Sequence[str],
permission: KnowledgeFSProductPermission,
) -> frozenset[str]:
permissions = self.permission_keys_by_control_space(
session=session,
tenant_id=tenant_id,
account_id=account_id,
control_space_ids=control_space_ids,
)
return frozenset(
control_space_id
for control_space_id, permission_keys in permissions.items()
if permission in permission_keys
)
def workspace_permission_allowed(
self,
*,
tenant_id: str,
account_id: str,
permission: KnowledgeFSProductPermission,
) -> bool:
if not dify_config.RBAC_ENABLED:
return True
return RBACService.CheckAccess.check(
tenant_id,
account_id,
scene=permission.value,
resource_type=None,
resource_id=None,
)
_ROLE_PERMISSIONS: dict[KnowledgeFSControlSpacePermissionRole, frozenset[KnowledgeFSProductPermission]] = {
KnowledgeFSControlSpacePermissionRole.OWNER: frozenset(KnowledgeFSProductPermission),
KnowledgeFSControlSpacePermissionRole.EDITOR: frozenset(
{
KnowledgeFSProductPermission.READ,
KnowledgeFSProductPermission.EDIT,
KnowledgeFSProductPermission.DOCUMENT_WRITE,
KnowledgeFSProductPermission.QUERY,
}
),
KnowledgeFSControlSpacePermissionRole.VIEWER: frozenset(
{KnowledgeFSProductPermission.READ, KnowledgeFSProductPermission.QUERY}
),
}
def visible_control_space_statement(*, tenant_id: str, account_id: str) -> sa.Select[tuple[KnowledgeFSControlSpace]]:
"""Build the SQL visibility filter before pagination or any KFS request."""
active_permission = sa.exists().where(
KnowledgeFSControlSpacePermission.tenant_id == KnowledgeFSControlSpace.tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == KnowledgeFSControlSpace.id,
KnowledgeFSControlSpacePermission.account_id == account_id,
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
)
return sa.select(KnowledgeFSControlSpace).where(
KnowledgeFSControlSpace.tenant_id == tenant_id,
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
sa.or_(
KnowledgeFSControlSpace.owner_account_id == account_id,
KnowledgeFSControlSpace.visibility == KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS,
sa.and_(
KnowledgeFSControlSpace.visibility == KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS,
active_permission,
),
),
)
def resolve_local_role(
session: Session,
*,
control_space: KnowledgeFSControlSpace,
account_id: str,
) -> KnowledgeFSControlSpacePermissionRole | None:
if control_space.owner_account_id == account_id:
return KnowledgeFSControlSpacePermissionRole.OWNER
permission = session.scalar(
sa.select(KnowledgeFSControlSpacePermission).where(
KnowledgeFSControlSpacePermission.tenant_id == control_space.tenant_id,
KnowledgeFSControlSpacePermission.control_space_id == control_space.id,
KnowledgeFSControlSpacePermission.account_id == account_id,
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
)
)
if permission is not None:
return permission.role
if control_space.visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS:
return KnowledgeFSControlSpacePermissionRole.VIEWER
return None
def resolve_local_roles(
session: Session,
*,
control_spaces: Sequence[KnowledgeFSControlSpace],
account_id: str,
) -> dict[str, KnowledgeFSControlSpacePermissionRole | None]:
if not control_spaces:
return {}
control_space_ids = tuple(control_space.id for control_space in control_spaces)
tenant_ids = tuple({control_space.tenant_id for control_space in control_spaces})
explicit_roles = {
(permission.tenant_id, permission.control_space_id): permission.role
for permission in session.scalars(
sa.select(KnowledgeFSControlSpacePermission).where(
KnowledgeFSControlSpacePermission.tenant_id.in_(tenant_ids),
KnowledgeFSControlSpacePermission.control_space_id.in_(control_space_ids),
KnowledgeFSControlSpacePermission.account_id == account_id,
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
)
)
}
return {
control_space.id: (
KnowledgeFSControlSpacePermissionRole.OWNER
if control_space.owner_account_id == account_id
else explicit_roles.get((control_space.tenant_id, control_space.id))
or (
KnowledgeFSControlSpacePermissionRole.VIEWER
if control_space.visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS
else None
)
)
for control_space in control_spaces
}
def local_role_allows(
role: KnowledgeFSControlSpacePermissionRole | None,
permission: KnowledgeFSProductPermission,
) -> bool:
return role is not None and permission in _ROLE_PERMISSIONS[role]
def effective_product_permissions(
role: KnowledgeFSControlSpacePermissionRole | None,
rbac_permissions: frozenset[KnowledgeFSProductPermission],
) -> tuple[KnowledgeFSProductPermission, ...]:
if role is None:
return ()
return tuple(
permission
for permission in KnowledgeFSProductPermission
if permission is not KnowledgeFSProductPermission.CREATE
and permission in _ROLE_PERMISSIONS[role]
and permission in rbac_permissions
)
__all__ = [
"DifyKnowledgeFSProductRBACPort",
"KnowledgeFSProductNotFoundError",
"KnowledgeFSProductRBACPort",
"effective_product_permissions",
"local_role_allows",
"resolve_local_role",
"resolve_local_roles",
"visible_control_space_statement",
]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,759 @@
"""Typed Dify BFF manifest and explicit gaps against the Capability v2 registry.
An operation is executable only when ``capability_operation_id`` resolves to an
exact method/path/action/resource entry in the P2 registry. Declared operations
without an executable transport remain explicit gaps and fail before token
issuance or external I/O.
"""
from __future__ import annotations
from enum import StrEnum
from types import MappingProxyType
from typing import Final, Literal, NamedTuple
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
class KnowledgeFSProductPermission(StrEnum):
READ = "knowledge_space_read"
CREATE = "knowledge_space_create"
EDIT = "knowledge_space_edit"
DELETE = "knowledge_space_delete"
ACCESS_CONFIG = "knowledge_space_access_config"
API_KEY_MANAGE = "knowledge_space_api_key_manage"
DOCUMENT_WRITE = "knowledge_space_document_write"
QUERY = "knowledge_space_query"
class KnowledgeFSProductOperation(NamedTuple):
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
capability_operation_id: str | None
permission: KnowledgeFSProductPermission
kfs_path: str | None
transport: Literal["binary", "direct", "json", "multipart", "sse", "unavailable"]
resource_resolver: Literal[
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
]
rbac_permission: KnowledgeFSProductPermission
billing_cost: int
max_request_bytes: int
max_response_bytes: int
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"]
rate_limit_bucket: Literal["direct", "import", "job", "query", "read", "write"]
rate_limit_cost: int
@property
def action(self) -> str | None:
if self.capability_operation_id is None:
return None
capability = KNOWLEDGE_FS_CAPABILITY_OPERATIONS.get(self.capability_operation_id)
return capability.action if capability is not None else None
def _operation(
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"],
capability_operation_id: str | None,
permission: KnowledgeFSProductPermission,
kfs_path: str | None,
transport: Literal["binary", "direct", "json", "multipart", "sse", "unavailable"],
*,
resource_resolver: Literal[
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
],
billing_cost: int,
max_request_bytes: int,
max_response_bytes: int,
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"],
rate_limit_bucket: Literal["direct", "import", "job", "query", "read", "write"] | None = None,
rate_limit_cost: int | None = None,
) -> KnowledgeFSProductOperation:
resolved_rate_limit_bucket = rate_limit_bucket or _default_rate_limit_bucket(
method=method,
permission=permission,
resource_resolver=resource_resolver,
stream_kind=stream_kind,
)
return KnowledgeFSProductOperation(
method=method,
capability_operation_id=capability_operation_id,
permission=permission,
kfs_path=kfs_path,
transport=transport,
resource_resolver=resource_resolver,
rbac_permission=permission,
billing_cost=billing_cost,
max_request_bytes=max_request_bytes,
max_response_bytes=max_response_bytes,
stream_kind=stream_kind,
rate_limit_bucket=resolved_rate_limit_bucket,
rate_limit_cost=rate_limit_cost if rate_limit_cost is not None else billing_cost,
)
def _default_rate_limit_bucket(
*,
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"],
permission: KnowledgeFSProductPermission,
resource_resolver: Literal[
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
],
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"],
) -> Literal["direct", "import", "job", "query", "read", "write"]:
if stream_kind in {"direct-upload", "sse"}:
return "direct"
if permission == KnowledgeFSProductPermission.QUERY:
return "query"
if resource_resolver == "job":
return "job"
if method == "GET":
return "read"
return "write"
KNOWLEDGE_FS_PRODUCT_OPERATIONS: Final[MappingProxyType[str, KnowledgeFSProductOperation]] = MappingProxyType(
{
"batchSpaceSummaries": _operation(
"POST",
"batchKnowledgeSpaceProductSummaries",
KnowledgeFSProductPermission.READ,
"/internal/knowledge-spaces/product-summaries/batch",
"json",
resource_resolver="namespace",
billing_cost=2,
max_request_bytes=64 * 1024,
max_response_bytes=1024 * 1024,
stream_kind="json",
),
"getSpace": _operation(
"GET",
"getKnowledgeSpace",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}",
"json",
resource_resolver="knowledge_space",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"updateSpace": _operation(
"PATCH",
"updateKnowledgeSpace",
KnowledgeFSProductPermission.EDIT,
"/knowledge-spaces/{id}",
"json",
resource_resolver="knowledge_space",
billing_cost=3,
max_request_bytes=32 * 1024,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"getSettings": _operation(
"GET",
"getKnowledgeSpaceProductSettings",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/product-settings",
"json",
resource_resolver="knowledge_space",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"updateSettings": _operation(
"PATCH",
"updateKnowledgeSpaceProductSettings",
KnowledgeFSProductPermission.EDIT,
"/knowledge-spaces/{id}/product-settings",
"json",
resource_resolver="knowledge_space",
billing_cost=5,
max_request_bytes=64 * 1024,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"listDocuments": _operation(
"GET",
"listDocuments",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents",
"json",
resource_resolver="knowledge_space",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"createDocument": _operation(
"POST",
"uploadDocument",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/documents",
"multipart",
resource_resolver="knowledge_space",
billing_cost=10,
max_request_bytes=0,
max_response_bytes=0,
stream_kind="buffered-multipart",
),
"getDocument": _operation(
"GET",
"getDocument",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents/{documentId}",
"json",
resource_resolver="document",
billing_cost=2,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"getDocumentOutline": _operation(
"GET",
"getDocumentOutline",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents/{documentId}/outline",
"json",
resource_resolver="document",
billing_cost=3,
max_request_bytes=0,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"listDocumentRevisions": _operation(
"GET",
"listDocumentRevisions",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents/{documentId}/revisions",
"json",
resource_resolver="document",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"updateDocumentMetadata": _operation(
"PATCH",
"patchDocumentMetadata",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/documents/{documentId}/metadata",
"json",
resource_resolver="document",
billing_cost=4,
max_request_bytes=128 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"listDocumentChunks": _operation(
"GET",
"listDocumentChunks",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks",
"json",
resource_resolver="document",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"getDocumentChunk": _operation(
"GET",
"getDocumentChunk",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}",
"json",
resource_resolver="document",
billing_cost=2,
max_request_bytes=0,
max_response_bytes=1024 * 1024,
stream_kind="json",
),
"deleteDocument": _operation(
"DELETE",
"requestDocumentDeletion",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/documents/{documentId}",
"json",
resource_resolver="document",
billing_cost=8,
max_request_bytes=32 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"bulkDeleteDocuments": _operation(
"DELETE",
"requestBulkDocumentDeletion",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/documents/bulk",
"json",
resource_resolver="knowledge_space",
billing_cost=20,
max_request_bytes=1024 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"reindexDocuments": _operation(
"POST",
"bulkReindexDocuments",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/documents/bulk/reindex",
"json",
resource_resolver="knowledge_space",
billing_cost=20,
max_request_bytes=1024 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"getCompilationJob": _operation(
"GET",
"getDocumentCompilationJob",
KnowledgeFSProductPermission.READ,
"/jobs/{id}",
"json",
resource_resolver="job",
billing_cost=1,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"cancelCompilationJob": _operation(
"DELETE",
"cancelDocumentCompilationJob",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/jobs/{id}",
"json",
resource_resolver="job",
billing_cost=5,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"retryCompilationJob": _operation(
"POST",
"retryDocumentCompilationJob",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/jobs/{id}/retry",
"json",
resource_resolver="job",
billing_cost=8,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"getBulkJob": _operation(
"GET",
"getBulkOperation",
KnowledgeFSProductPermission.READ,
"/bulk-jobs/{id}",
"json",
resource_resolver="job",
billing_cost=1,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"listSources": _operation(
"GET",
"listKnowledgeSpaceSources",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/sources",
"json",
resource_resolver="knowledge_space",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"createSource": _operation(
"POST",
"createKnowledgeSpaceSource",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources",
"json",
resource_resolver="knowledge_space",
billing_cost=8,
max_request_bytes=256 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"getSource": _operation(
"GET",
"getKnowledgeSpaceSource",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/sources/{sourceId}",
"json",
resource_resolver="source",
billing_cost=1,
max_request_bytes=0,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"updateSource": _operation(
"PATCH",
"updateKnowledgeSpaceSource",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}",
"json",
resource_resolver="source",
billing_cost=4,
max_request_bytes=256 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"deleteSource": _operation(
"DELETE",
"requestSourceDeletion",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}",
"json",
resource_resolver="source",
billing_cost=8,
max_request_bytes=32 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"testSource": _operation(
"POST",
"testKnowledgeSpaceSource",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/test",
"json",
resource_resolver="source",
billing_cost=3,
max_request_bytes=0,
max_response_bytes=256 * 1024,
stream_kind="json",
),
"crawlSource": _operation(
"POST",
"crawlKnowledgeSpaceSource",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/crawl",
"json",
resource_resolver="source",
billing_cost=20,
max_request_bytes=0,
max_response_bytes=8 * 1024 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"listSourcePages": _operation(
"GET",
"listKnowledgeSpaceSourcePages",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/sources/{sourceId}/pages",
"json",
resource_resolver="source",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"importSourcePages": _operation(
"POST",
"importKnowledgeSpaceSourcePages",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/import",
"json",
resource_resolver="source",
billing_cost=25,
max_request_bytes=1024 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"listSourceFiles": _operation(
"GET",
"listKnowledgeSpaceSourceFiles",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/sources/{sourceId}/files",
"json",
resource_resolver="source",
billing_cost=3,
max_request_bytes=32 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"importSourceFiles": _operation(
"POST",
"importKnowledgeSpaceSourceFiles",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/sources/{sourceId}/import-files",
"json",
resource_resolver="source",
billing_cost=25,
max_request_bytes=1024 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
rate_limit_bucket="import",
),
"createQuery": _operation(
"POST",
"createQuery",
KnowledgeFSProductPermission.QUERY,
"/queries",
"direct",
resource_resolver="knowledge_space",
billing_cost=20,
max_request_bytes=64 * 1024,
max_response_bytes=0,
stream_kind="sse",
),
"listResearchTasks": _operation(
"GET",
"listKnowledgeSpaceResearchTasks",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/research-tasks",
"json",
resource_resolver="knowledge_space",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"createResearchTask": _operation(
"POST",
"createResearchTask",
KnowledgeFSProductPermission.QUERY,
"/research-tasks",
"json",
resource_resolver="knowledge_space",
billing_cost=25,
max_request_bytes=64 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"planResearchTask": _operation(
"POST",
"planResearchTask",
KnowledgeFSProductPermission.QUERY,
"/research-tasks/plan",
"json",
resource_resolver="knowledge_space",
billing_cost=8,
max_request_bytes=64 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"getResearchTask": _operation(
"GET",
"getResearchTask",
KnowledgeFSProductPermission.QUERY,
"/research-tasks/{id}",
"json",
resource_resolver="research_task",
billing_cost=1,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"listResearchTaskPartials": _operation(
"GET",
"listResearchTaskPartials",
KnowledgeFSProductPermission.QUERY,
"/research-tasks/{id}/partials",
"json",
resource_resolver="research_task",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=8 * 1024 * 1024,
stream_kind="json",
),
"cancelResearchTask": _operation(
"DELETE",
"cancelResearchTask",
KnowledgeFSProductPermission.QUERY,
"/research-tasks/{id}",
"json",
resource_resolver="research_task",
billing_cost=5,
max_request_bytes=16 * 1024,
max_response_bytes=512 * 1024,
stream_kind="json",
),
"listTraces": _operation(
"GET",
"listKnowledgeSpaceQualityTraces",
KnowledgeFSProductPermission.READ,
"/knowledge-spaces/{id}/quality/traces",
"json",
resource_resolver="knowledge_space",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"getTrace": _operation(
"GET",
"getAnswerTrace",
KnowledgeFSProductPermission.QUERY,
"/queries/{traceId}",
"json",
resource_resolver="query",
billing_cost=2,
max_request_bytes=16 * 1024,
max_response_bytes=2 * 1024 * 1024,
stream_kind="json",
),
"listTraceEvidence": _operation(
"GET",
"listQueryEvidence",
KnowledgeFSProductPermission.QUERY,
"/queries/{traceId}/evidence",
"json",
resource_resolver="query",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"listTraceConflicts": _operation(
"GET",
"listQueryConflicts",
KnowledgeFSProductPermission.QUERY,
"/queries/{traceId}/conflicts",
"json",
resource_resolver="query",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"listTraceMissing": _operation(
"GET",
"listQueryMissing",
KnowledgeFSProductPermission.QUERY,
"/queries/{traceId}/missing",
"json",
resource_resolver="query",
billing_cost=3,
max_request_bytes=16 * 1024,
max_response_bytes=4 * 1024 * 1024,
stream_kind="json",
),
"createUploadSession": _operation(
"POST",
"createUploadSession",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/knowledge-spaces/{id}/upload-sessions",
"direct",
resource_resolver="knowledge_space",
billing_cost=5,
max_request_bytes=64 * 1024,
max_response_bytes=64 * 1024,
stream_kind="direct-upload",
),
"presignUploadSessionPart": _operation(
"POST",
"presignUploadSessionPart",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/upload-sessions/{id}/parts/{partNumber}/presign",
"direct",
resource_resolver="upload_session",
billing_cost=1,
max_request_bytes=32 * 1024,
max_response_bytes=64 * 1024,
stream_kind="direct-upload",
),
"uploadSmallFile": _operation(
"POST",
"uploadSmallFile",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/upload-sessions/{id}/small-file",
"binary",
resource_resolver="upload_session",
billing_cost=8,
max_request_bytes=8 * 1024 * 1024,
max_response_bytes=128 * 1024,
stream_kind="json",
),
"completeUploadSession": _operation(
"POST",
"completeUploadSession",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/upload-sessions/{id}/complete",
"direct",
resource_resolver="upload_session",
billing_cost=8,
max_request_bytes=128 * 1024,
max_response_bytes=128 * 1024,
stream_kind="direct-upload",
),
"abortUploadSession": _operation(
"POST",
"abortUploadSession",
KnowledgeFSProductPermission.DOCUMENT_WRITE,
"/upload-sessions/{id}/abort",
"direct",
resource_resolver="upload_session",
billing_cost=1,
max_request_bytes=32 * 1024,
max_response_bytes=64 * 1024,
stream_kind="direct-upload",
),
"streamResearchTask": _operation(
"GET",
"streamResearchTaskProgress",
KnowledgeFSProductPermission.READ,
"/research-tasks/{id}/events",
"direct",
resource_resolver="research_task",
billing_cost=5,
max_request_bytes=0,
max_response_bytes=0,
stream_kind="sse",
),
}
)
def is_product_operation_registered(operation_id: str) -> bool:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
if operation is None or operation.capability_operation_id is None or operation.kfs_path is None:
return False
capability = KNOWLEDGE_FS_CAPABILITY_OPERATIONS.get(operation.capability_operation_id)
return bool(
capability
and capability.method == operation.method
and capability.path == operation.kfs_path
and capability.action
and capability.resource_type == operation.resource_resolver
and operation.permission == operation.rbac_permission
)
def is_product_operation_ready(operation_id: str) -> bool:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
return bool(
operation
and operation.transport in {"binary", "direct", "json"}
and is_product_operation_registered(operation_id)
)
def knowledge_fs_product_operation_gaps() -> tuple[str, ...]:
return tuple(
operation_id for operation_id in KNOWLEDGE_FS_PRODUCT_OPERATIONS if not is_product_operation_ready(operation_id)
)
def product_operation_action(operation_id: str) -> str:
if not is_product_operation_ready(operation_id):
raise KeyError(operation_id)
capability_operation_id = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].capability_operation_id
if capability_operation_id is None:
raise KeyError(operation_id)
return KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id].action
__all__ = [
"KNOWLEDGE_FS_PRODUCT_OPERATIONS",
"KnowledgeFSProductOperation",
"KnowledgeFSProductPermission",
"is_product_operation_ready",
"is_product_operation_registered",
"knowledge_fs_product_operation_gaps",
"product_operation_action",
]
+102
View File
@@ -0,0 +1,102 @@
"""Ports for capability-authenticated, manifest-bounded KnowledgeFS BFF calls."""
from __future__ import annotations
from typing import Literal, NamedTuple, Protocol
from pydantic import JsonValue
from services.knowledge_fs.product_dto import KnowledgeFSTechnicalSummary
class KnowledgeFSProductRemoteError(RuntimeError):
"""KnowledgeFS could not provide an authoritative product response."""
class KnowledgeFSOperationUnavailableError(RuntimeError):
"""The Dify/KFS/Capability operation manifests are not yet aligned."""
class KnowledgeFSProductRequestRejectedError(RuntimeError):
"""A bounded product request was rejected locally or by authoritative KFS validation."""
def __init__(self, *, status_code: Literal[409, 413, 422]) -> None:
super().__init__(f"KnowledgeFS rejected the product request with HTTP {status_code}")
self.status_code = status_code
class KnowledgeFSRemoteJSONRequest(NamedTuple):
operation_id: str
method: str
path: str
namespace_id: str
knowledge_space_id: str
capability_token: str
trace_id: str
payload: JsonValue | None
query: tuple[tuple[str, str], ...] = ()
headers: tuple[tuple[str, str], ...] = ()
class KnowledgeFSRemoteBinaryRequest(NamedTuple):
operation_id: str
method: str
path: str
namespace_id: str
knowledge_space_id: str
capability_token: str
trace_id: str
body: bytes
query: tuple[tuple[str, str], ...]
class KnowledgeFSProductRemotePort(Protocol):
def batch_space_summaries(
self,
*,
namespace_id: str,
knowledge_space_ids: tuple[str, ...],
capability_token: str,
trace_id: str,
) -> dict[str, KnowledgeFSTechnicalSummary]:
"""Fetch exactly the explicit authorized Space IDs in one remote call."""
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
"""Execute one manifest-approved JSON request using only its operation capability."""
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
"""Execute one strictly bounded binary request using only its operation capability."""
class UnavailableKnowledgeFSProductRemote:
"""Fail-closed default until a manifest-aligned KFS transport is assembled."""
def batch_space_summaries(
self,
*,
namespace_id: str,
knowledge_space_ids: tuple[str, ...],
capability_token: str,
trace_id: str,
) -> dict[str, KnowledgeFSTechnicalSummary]:
_ = (namespace_id, knowledge_space_ids, capability_token, trace_id)
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
_ = request
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
_ = request
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
__all__ = [
"KnowledgeFSOperationUnavailableError",
"KnowledgeFSProductRemoteError",
"KnowledgeFSProductRemotePort",
"KnowledgeFSProductRequestRejectedError",
"KnowledgeFSRemoteBinaryRequest",
"KnowledgeFSRemoteJSONRequest",
"UnavailableKnowledgeFSProductRemote",
]
@@ -0,0 +1,264 @@
"""Capability-only HTTP transport for manifest-approved JSON and bounded binary calls."""
from __future__ import annotations
import json
from http import HTTPStatus
from urllib.parse import urlencode
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
from core.helper import ssrf_proxy
from core.tools.errors import ToolSSRFError
from services.knowledge_fs.product_dto import (
KnowledgeFSBatchTechnicalSummaryResponse,
KnowledgeFSTechnicalSummary,
)
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
from services.knowledge_fs.product_remote import (
KnowledgeFSOperationUnavailableError,
KnowledgeFSProductRemoteError,
KnowledgeFSProductRequestRejectedError,
KnowledgeFSRemoteBinaryRequest,
KnowledgeFSRemoteJSONRequest,
)
_JSON_ADAPTER: TypeAdapter[JsonValue] = TypeAdapter(JsonValue)
_MAX_BATCH_SUMMARIES = 100
class HTTPKnowledgeFSProductRemoteClient:
"""Build outbound headers from trusted capability input; browser headers never enter this class."""
def __init__(self, *, base_url: str, timeout_seconds: float, max_response_bytes: int = 4 * 1024 * 1024) -> None:
self._base_url = base_url
self._timeout_seconds = timeout_seconds
self._max_response_bytes = max_response_bytes
def batch_space_summaries(
self,
*,
namespace_id: str,
knowledge_space_ids: tuple[str, ...],
capability_token: str,
trace_id: str,
) -> dict[str, KnowledgeFSTechnicalSummary]:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS["batchSpaceSummaries"]
if not is_product_operation_ready("batchSpaceSummaries") or operation.kfs_path is None:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch summary operation is unavailable")
if (
not namespace_id
or not capability_token
or not trace_id
or not knowledge_space_ids
or len(knowledge_space_ids) > _MAX_BATCH_SUMMARIES
or any(not knowledge_space_id for knowledge_space_id in knowledge_space_ids)
or len(set(knowledge_space_ids)) != len(knowledge_space_ids)
):
raise KnowledgeFSProductRemoteError("KnowledgeFS batch request binding is invalid")
payload = self._request_json(
method=operation.method,
path=operation.kfs_path,
capability_token=capability_token,
trace_id=trace_id,
payload={"knowledgeSpaceIds": list(knowledge_space_ids)},
query=(),
max_request_bytes=operation.max_request_bytes,
max_response_bytes=operation.max_response_bytes,
)
try:
response = KnowledgeFSBatchTechnicalSummaryResponse.model_validate(payload)
except ValidationError as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an invalid batch summary") from exc
requested_ids = frozenset(knowledge_space_ids)
summaries: dict[str, KnowledgeFSTechnicalSummary] = {}
for summary in response.items:
if summary.knowledge_space_id not in requested_ids or summary.knowledge_space_id in summaries:
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an out-of-scope batch summary")
summaries[summary.knowledge_space_id] = summary
return summaries
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(request.operation_id)
if operation is None or operation.transport != "json" or not is_product_operation_ready(request.operation_id):
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {request.operation_id}")
if (
request.method != operation.method
or operation.kfs_path is None
or not _matches_path(operation.kfs_path, request.path)
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request does not match its operation manifest")
if (
not request.namespace_id
or not request.knowledge_space_id
or not request.capability_token
or not request.trace_id
):
raise KnowledgeFSProductRemoteError("KnowledgeFS request binding is incomplete")
return self._request_json(
method=request.method,
path=request.path,
capability_token=request.capability_token,
trace_id=request.trace_id,
payload=request.payload,
query=request.query,
extra_headers=request.headers,
max_request_bytes=operation.max_request_bytes,
max_response_bytes=operation.max_response_bytes,
)
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(request.operation_id)
if operation is None or operation.transport != "binary" or not is_product_operation_ready(request.operation_id):
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {request.operation_id}")
if (
request.method != operation.method
or operation.kfs_path is None
or not _matches_path(operation.kfs_path, request.path)
):
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request does not match its operation manifest")
if (
not request.namespace_id
or not request.knowledge_space_id
or not request.capability_token
or not request.trace_id
or request.query != (("knowledgeSpaceId", request.knowledge_space_id),)
):
raise KnowledgeFSProductRemoteError("KnowledgeFS binary request binding is incomplete")
if not isinstance(request.body, bytes) or not request.body:
raise KnowledgeFSProductRequestRejectedError(status_code=422)
if len(request.body) > operation.max_request_bytes:
raise KnowledgeFSProductRequestRejectedError(status_code=413)
response_limit = min(self._max_response_bytes, operation.max_response_bytes)
if response_limit <= 0:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS operation response limit is unavailable")
headers = {
"Accept": "application/json",
"Accept-Encoding": "identity",
"Authorization": f"Bearer {request.capability_token}",
"Content-Type": "application/octet-stream",
"X-Trace-Id": request.trace_id,
}
try:
upstream_url = httpx.URL(f"{self._base_url.rstrip('/')}/").join(request.path.lstrip("/"))
response = ssrf_proxy.make_request(
method=request.method,
url=str(upstream_url),
headers=headers,
params=request.query,
content=request.body,
timeout=self._timeout_seconds,
follow_redirects=False,
max_retries=0,
stream_response=True,
)
response = ssrf_proxy.buffer_response(response, max_response_bytes=response_limit)
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS request failed") from exc
try:
if response.status_code == 409:
raise KnowledgeFSProductRequestRejectedError(status_code=409)
if response.status_code == 413:
raise KnowledgeFSProductRequestRejectedError(status_code=413)
if response.status_code == 422:
raise KnowledgeFSProductRequestRejectedError(status_code=422)
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
if content_type != "application/json" and not content_type.endswith("+json"):
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
try:
return _JSON_ADAPTER.validate_python(response.json())
except (ValueError, ValidationError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS returned invalid JSON") from exc
finally:
response.close()
def _request_json(
self,
*,
method: str,
path: str,
capability_token: str,
trace_id: str,
payload: JsonValue | None,
query: tuple[tuple[str, str], ...],
extra_headers: tuple[tuple[str, str], ...] = (),
max_request_bytes: int,
max_response_bytes: int,
) -> JsonValue:
request_size = len(urlencode(query).encode("utf-8"))
if payload is not None:
try:
request_size += len(
json.dumps(payload, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
)
except (TypeError, ValueError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS request payload is invalid") from exc
if request_size > max_request_bytes:
raise KnowledgeFSProductRemoteError("KnowledgeFS request exceeds its operation byte limit")
response_limit = min(self._max_response_bytes, max_response_bytes)
if response_limit <= 0:
raise KnowledgeFSOperationUnavailableError("KnowledgeFS operation response limit is unavailable")
headers = {
"Accept": "application/json",
"Accept-Encoding": "identity",
"Authorization": f"Bearer {capability_token}",
"X-Trace-Id": trace_id,
}
for name, value in extra_headers:
if name.lower() != "idempotency-key" or not 8 <= len(value.strip()) <= 255:
raise KnowledgeFSProductRemoteError("KnowledgeFS request header binding is invalid")
headers["Idempotency-Key"] = value.strip()
request_kwargs: dict[str, object] = {
"headers": headers,
"params": query,
"timeout": self._timeout_seconds,
"follow_redirects": False,
}
if payload is not None:
headers["Content-Type"] = "application/json"
request_kwargs["json"] = payload
try:
upstream_url = httpx.URL(f"{self._base_url.rstrip('/')}/").join(path.lstrip("/"))
response = ssrf_proxy.make_request(
method=method,
url=str(upstream_url),
max_retries=0,
stream_response=True,
**request_kwargs,
)
response = ssrf_proxy.buffer_response(response, max_response_bytes=response_limit)
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS request failed") from exc
try:
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
if content_type != "application/json" and not content_type.endswith("+json"):
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
try:
return _JSON_ADAPTER.validate_python(response.json())
except (ValueError, ValidationError) as exc:
raise KnowledgeFSProductRemoteError("KnowledgeFS returned invalid JSON") from exc
finally:
response.close()
def _matches_path(template: str, path: str) -> bool:
template_segments = template.strip("/").split("/")
path_segments = path.strip("/").split("/")
if len(template_segments) != len(path_segments):
return False
for expected, actual in zip(template_segments, path_segments, strict=True):
if expected.startswith("{") and expected.endswith("}"):
if not actual or actual in {".", ".."} or any(character in actual for character in ("%", "?", "#", "\\")):
return False
elif expected != actual:
return False
return True
__all__ = ["HTTPKnowledgeFSProductRemoteClient"]

Some files were not shown because too many files have changed in this diff Show More