Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d74499c09d | ||
|
|
29858d0db0 | ||
|
|
77732c7bb1 | ||
|
|
dde79fc9bf | ||
|
|
4ee43b8afc |
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "npx -y block-no-verify@1.1.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@
|
||||
**/*.pyc
|
||||
**/.mypy_cache
|
||||
**/.ruff_cache
|
||||
knowledge-fs/
|
||||
.git
|
||||
.github
|
||||
*.md
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
version: 2
|
||||
|
||||
updates:
|
||||
- package-ecosystem: "npm"
|
||||
directory: "/knowledge-fs"
|
||||
open-pull-requests-limit: 10
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
groups:
|
||||
knowledge-fs-dependencies:
|
||||
patterns:
|
||||
- "*"
|
||||
- package-ecosystem: "uv"
|
||||
directory: "/api"
|
||||
open-pull-requests-limit: 10
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
name: KnowledgeFS CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
merge_group:
|
||||
branches: ["main"]
|
||||
types: [checks_requested]
|
||||
push:
|
||||
branches: ["main"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: knowledge-fs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CI: true
|
||||
DIFY_KNOWLEDGE_FS_API_IMAGE_NAME: >-
|
||||
${{ vars.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME || 'langgenius/dify-knowledge-fs-api' }}
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
name: Check KnowledgeFS changes
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
outputs:
|
||||
knowledge-fs: ${{ steps.changes.outputs.knowledge-fs }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Detect KnowledgeFS changes
|
||||
id: changes
|
||||
uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
|
||||
with:
|
||||
filters: |
|
||||
knowledge-fs:
|
||||
- 'knowledge-fs/**'
|
||||
- 'knowledge-fs/packages/api/src/dify-capability-v2.ts'
|
||||
- 'knowledge-fs/packages/api/src/knowledge-space-routes.ts'
|
||||
- 'knowledge-fs/packages/api/src/upload-session-routes.ts'
|
||||
- 'knowledge-fs/scripts/export-capability-v2-operations.mjs'
|
||||
- 'knowledge-fs/scripts/export-openapi.mjs'
|
||||
- 'api/dev/generate_knowledge_fs_contract.py'
|
||||
- 'api/dev/knowledge_fs_product_contract.py'
|
||||
- 'api/knowledge-fs-contract.lock.json'
|
||||
- 'api/knowledge-fs-product-operation-gaps.json'
|
||||
- 'api/knowledge-fs-product-operations.json'
|
||||
- 'api/**/knowledge_fs/**'
|
||||
- 'api/**/*knowledge_fs*'
|
||||
- 'api/**/*knowledge-fs*'
|
||||
- 'api/.env.example'
|
||||
- 'api/app_factory.py'
|
||||
- 'api/commands/__init__.py'
|
||||
- 'api/controllers/console/__init__.py'
|
||||
- 'api/controllers/console/workspace/rbac.py'
|
||||
- 'api/controllers/service_api/__init__.py'
|
||||
- 'api/core/agent/base_agent_runner.py'
|
||||
- 'api/core/app/apps/agent_app/runtime_request_builder.py'
|
||||
- 'api/core/rbac/entities.py'
|
||||
- 'api/core/tools/__base/tool_runtime.py'
|
||||
- 'api/core/tools/builtin_tool/_position.yaml'
|
||||
- 'api/core/workflow/node_runtime.py'
|
||||
- 'api/core/workflow/nodes/agent_v2/runtime_request_builder.py'
|
||||
- 'api/extensions/ext_celery.py'
|
||||
- 'api/extensions/ext_commands.py'
|
||||
- 'api/models/__init__.py'
|
||||
- 'api/services/account_service.py'
|
||||
- 'api/services/agent_tool_inner_service.py'
|
||||
- 'api/services/enterprise/rbac_service.py'
|
||||
- 'api/services/entities/agent_tool_inner.py'
|
||||
- 'api/services/knowledge_fs/**'
|
||||
- 'api/services/knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
|
||||
- 'api/tests/unit_tests/controllers/console/workspace/test_rbac.py'
|
||||
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
|
||||
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
|
||||
- 'api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py'
|
||||
- 'api/tests/unit_tests/core/workflow/test_node_runtime.py'
|
||||
- 'api/tests/unit_tests/services/enterprise/test_rbac_service.py'
|
||||
- 'api/tests/unit_tests/services/test_account_service.py'
|
||||
- 'api/tests/unit_tests/services/test_agent_tool_inner_service.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_capability.py'
|
||||
- 'api/tests/unit_tests/services/test_knowledge_fs_product_operations.py'
|
||||
- 'api/pyproject.toml'
|
||||
- 'api/uv.lock'
|
||||
- 'dify-agent/src/dify_agent/layers/dify_core_tools/client.py'
|
||||
- 'dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_client.py'
|
||||
- 'packages/contracts/generated/api/console/**'
|
||||
- 'packages/contracts/generated/api/service/**'
|
||||
- 'docker/.env.example'
|
||||
- 'docker/README.md'
|
||||
- 'docker/dify-env-sync.py'
|
||||
- 'docker/dify-env-sync.sh'
|
||||
- 'docker/docker-compose-template.yaml'
|
||||
- 'docker/docker-compose.yaml'
|
||||
- 'docker/envs/core-services/api.env.example'
|
||||
- 'docker/envs/core-services/knowledge-fs.env.example'
|
||||
- 'docker/generate_docker_compose'
|
||||
- 'docs/design/knowledge-fs*'
|
||||
- '.github/dependabot.yml'
|
||||
- '.github/workflows/knowledge-fs-ci.yml'
|
||||
|
||||
build:
|
||||
name: Build KnowledgeFS API production image
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USER }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract KnowledgeFS image metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: ${{ env.DIFY_KNOWLEDGE_FS_API_IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
type=ref,event=branch
|
||||
type=sha,format=long
|
||||
|
||||
- name: Build KnowledgeFS API image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: ./knowledge-fs
|
||||
file: ./knowledge-fs/apps/api/Dockerfile
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
platforms: linux/amd64
|
||||
push: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
|
||||
quality:
|
||||
name: Run KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs == 'true' || github.event_name == 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
working-directory: ./knowledge-fs
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||||
with:
|
||||
package_json_file: knowledge-fs/package.json
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
cache-dependency-path: knowledge-fs/pnpm-lock.yaml
|
||||
|
||||
- name: Install KnowledgeFS dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Scan KnowledgeFS secrets
|
||||
run: pnpm security:secrets
|
||||
|
||||
- name: Audit KnowledgeFS production dependencies
|
||||
run: pnpm security:dependencies
|
||||
|
||||
- name: Run KnowledgeFS checks
|
||||
run: pnpm check
|
||||
|
||||
- name: Build KnowledgeFS
|
||||
run: pnpm build
|
||||
|
||||
- name: Lint KnowledgeFS
|
||||
run: pnpm lint
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
|
||||
with:
|
||||
enable-cache: true
|
||||
python-version: "3.12"
|
||||
cache-dependency-glob: |
|
||||
api/uv.lock
|
||||
dify-agent/uv.lock
|
||||
|
||||
- name: Verify Dify dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install Dify contract dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project api --locked --dev
|
||||
|
||||
- name: Collect Dify KnowledgeFS gate targets
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
target_dir="${RUNNER_TEMP:?}/knowledge-fs-ci-targets"
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
production_targets=()
|
||||
add_production_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS production target is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
production_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_production_target "$path"
|
||||
fi
|
||||
done < <(
|
||||
find api \
|
||||
\( -path 'api/.venv' -o -path 'api/tests' -o -path 'api/storage' \) -prune \
|
||||
-o -type f -name '*.py' -print0
|
||||
)
|
||||
|
||||
production_touchpoints=(
|
||||
api/app_factory.py
|
||||
api/commands/__init__.py
|
||||
api/controllers/console/__init__.py
|
||||
api/controllers/console/workspace/rbac.py
|
||||
api/controllers/service_api/__init__.py
|
||||
api/core/agent/base_agent_runner.py
|
||||
api/core/app/apps/agent_app/runtime_request_builder.py
|
||||
api/core/rbac/entities.py
|
||||
api/core/tools/__base/tool_runtime.py
|
||||
api/core/workflow/node_runtime.py
|
||||
api/core/workflow/nodes/agent_v2/runtime_request_builder.py
|
||||
api/extensions/ext_celery.py
|
||||
api/extensions/ext_commands.py
|
||||
api/models/__init__.py
|
||||
api/services/account_service.py
|
||||
api/services/agent_tool_inner_service.py
|
||||
api/services/enterprise/rbac_service.py
|
||||
api/services/entities/agent_tool_inner.py
|
||||
)
|
||||
for path in "${production_touchpoints[@]}"; do
|
||||
add_production_target "$path"
|
||||
done
|
||||
printf '%s\0' "${production_touchpoints[@]}" > "$target_dir/glue-files"
|
||||
|
||||
if ((${#production_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${production_targets[@]}" > "$target_dir/production-files"
|
||||
|
||||
test_targets=()
|
||||
add_test_target() {
|
||||
local path="$1"
|
||||
if [[ ! -f "$path" ]]; then
|
||||
echo "required Dify KnowledgeFS unit test is missing: $path" >&2
|
||||
exit 1
|
||||
fi
|
||||
test_targets+=("$path")
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' path; do
|
||||
if [[ "$path" == *knowledge_fs* ]]; then
|
||||
add_test_target "$path"
|
||||
fi
|
||||
done < <(find api/tests/unit_tests -type f -name '*.py' -print0)
|
||||
|
||||
test_touchpoints=(
|
||||
api/tests/unit_tests/controllers/console/workspace/test_rbac.py
|
||||
api/tests/unit_tests/core/agent/test_base_agent_runner.py
|
||||
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
|
||||
api/tests/unit_tests/core/workflow/nodes/tool/test_tool_node_runtime.py
|
||||
api/tests/unit_tests/core/workflow/test_node_runtime.py
|
||||
api/tests/unit_tests/services/enterprise/test_rbac_service.py
|
||||
api/tests/unit_tests/services/test_account_service.py
|
||||
api/tests/unit_tests/services/test_agent_tool_inner_service.py
|
||||
)
|
||||
for path in "${test_touchpoints[@]}"; do
|
||||
add_test_target "$path"
|
||||
done
|
||||
|
||||
required_test_scopes=(
|
||||
/commands/
|
||||
/configs/
|
||||
/controllers/
|
||||
/core/agent/
|
||||
/core/app/apps/agent_app/
|
||||
/core/tools/builtin_tool/providers/knowledge_fs/
|
||||
/core/workflow/
|
||||
/dev/
|
||||
/extensions/
|
||||
/migrations/
|
||||
/models/
|
||||
/repositories/
|
||||
/services/
|
||||
/tasks/
|
||||
)
|
||||
for required_scope in "${required_test_scopes[@]}"; do
|
||||
scope_found=false
|
||||
for path in "${test_targets[@]}"; do
|
||||
if [[ "$path" == *"$required_scope"* ]]; then
|
||||
scope_found=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$scope_found" != true ]]; then
|
||||
echo "required Dify KnowledgeFS test scope is empty: $required_scope" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ((${#test_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\0' "${test_targets[@]}" > "$target_dir/unit-test-files"
|
||||
|
||||
- name: Lint Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev ruff format --check "${targets[@]}"
|
||||
uv run --project api --dev ruff check "${targets[@]}"
|
||||
|
||||
- name: Type-check Dify KnowledgeFS integration
|
||||
working-directory: .
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/production-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS production target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
PYREFLY_OUTPUT_FORMAT=github ./dev/pyrefly-check-local "${targets[@]}"
|
||||
|
||||
mypy_targets=()
|
||||
for path in "${targets[@]}"; do
|
||||
if [[ "$path" != api/migrations/* ]]; then
|
||||
mypy_targets+=("${path#api/}")
|
||||
fi
|
||||
done
|
||||
if ((${#mypy_targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS Mypy target set is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --directory api --dev mypy \
|
||||
--explicit-package-bases \
|
||||
--exclude-gitignore \
|
||||
--exclude '(^|/)conftest\.py$' \
|
||||
--exclude 'tests/' \
|
||||
--exclude 'migrations/' \
|
||||
--check-untyped-defs \
|
||||
--disable-error-code=import-untyped \
|
||||
"${mypy_targets[@]}"
|
||||
|
||||
- name: Test Dify KnowledgeFS unit surface
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets=()
|
||||
while IFS= read -r -d '' path; do
|
||||
targets+=("$path")
|
||||
done < "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/unit-test-files"
|
||||
if ((${#targets[@]} == 0)); then
|
||||
echo "Dify KnowledgeFS unit test target manifest is empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
uv run --project api --dev coverage run --branch --source=api -m pytest "${targets[@]}" --no-cov -q
|
||||
|
||||
- name: Enforce Dify KnowledgeFS focused coverage
|
||||
working-directory: .
|
||||
env:
|
||||
COVERAGE_FILE: ${{ runner.temp }}/dify-knowledge-fs.coverage
|
||||
KNOWLEDGE_FS_COVERAGE_BASE: >-
|
||||
${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
report="${RUNNER_TEMP:?}/dify-knowledge-fs-coverage.json"
|
||||
uv run --project api --dev coverage json --show-contexts -o "$report"
|
||||
uv run --project api --dev python api/dev/check_knowledge_fs_coverage.py \
|
||||
--coverage-json "$report" \
|
||||
--glue-manifest "${RUNNER_TEMP:?}/knowledge-fs-ci-targets/glue-files" \
|
||||
--base "$KNOWLEDGE_FS_COVERAGE_BASE" \
|
||||
--minimum 90 \
|
||||
--glue-minimum 90
|
||||
|
||||
- name: Verify Dify KnowledgeFS contract
|
||||
working-directory: .
|
||||
run: uv run --project api python api/dev/generate_knowledge_fs_contract.py --check
|
||||
|
||||
- name: Verify Dify Agent dependency lock
|
||||
working-directory: .
|
||||
run: uv lock --project dify-agent --check
|
||||
|
||||
- name: Install Dify Agent gate dependencies
|
||||
working-directory: .
|
||||
run: uv sync --project dify-agent --locked --dev
|
||||
|
||||
- name: Lint Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: |
|
||||
uv run --project . --dev ruff format --check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
uv run --project . --dev ruff check \
|
||||
src/dify_agent/layers/dify_core_tools/client.py \
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Type-check Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev basedpyright --level error
|
||||
src/dify_agent/layers/dify_core_tools/client.py
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
|
||||
- name: Test Dify Agent KnowledgeFS integration
|
||||
working-directory: ./dify-agent
|
||||
run: >-
|
||||
uv run --project . --dev python -m pytest
|
||||
tests/local/dify_agent/layers/dify_core_tools/test_client.py
|
||||
-q
|
||||
|
||||
skip:
|
||||
name: Skip KnowledgeFS quality and contract gates
|
||||
needs: check-changes
|
||||
if: needs.check-changes.outputs.knowledge-fs != 'true' && github.event_name != 'workflow_dispatch'
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Report skipped KnowledgeFS checks
|
||||
run: echo "No KnowledgeFS-related changes detected; skipping KnowledgeFS checks."
|
||||
|
||||
final:
|
||||
name: KnowledgeFS CI
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
- check-changes
|
||||
- build
|
||||
- quality
|
||||
- skip
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
steps:
|
||||
- name: Finalize KnowledgeFS CI status
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BUILD_RESULT: ${{ needs.build.result }}
|
||||
KNOWLEDGE_FS_CHANGED: ${{ needs.check-changes.outputs.knowledge-fs }}
|
||||
QUALITY_RESULT: ${{ needs.quality.result }}
|
||||
SKIP_RESULT: ${{ needs.skip.result }}
|
||||
run: |
|
||||
if [[ "$EVENT_NAME" == 'workflow_dispatch' || "$KNOWLEDGE_FS_CHANGED" == 'true' ]]; then
|
||||
if [[ "$BUILD_RESULT" == 'success' && "$QUALITY_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS build and checks ran successfully."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS build or checks failed: build=$BUILD_RESULT quality=$QUALITY_RESULT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$SKIP_RESULT" == 'success' ]]; then
|
||||
echo "KnowledgeFS checks were skipped because no related files changed."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "KnowledgeFS change detection or skip reporting failed with result: $SKIP_RESULT" >&2
|
||||
exit 1
|
||||
@@ -30,6 +30,11 @@ share/python-wheels/
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# KnowledgeFS is an independently rooted TypeScript workspace. Its admin `lib`
|
||||
# directory contains source files rather than Python build output.
|
||||
!/knowledge-fs/apps/admin/lib/
|
||||
!/knowledge-fs/apps/admin/lib/**
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
|
||||
+18
-1
@@ -683,11 +683,28 @@ AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
|
||||
|
||||
# KnowledgeFS (Dataset 2.0)
|
||||
KNOWLEDGE_FS_ENABLED=false
|
||||
# Production deployments require HTTPS; plain HTTP is limited to non-production or loopback.
|
||||
KNOWLEDGE_FS_BASE_URL=
|
||||
# Shared with KnowledgeFS; use at least 32 random characters.
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN=
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED=false
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY=false
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY=false
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS=15
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS=60
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE=25
|
||||
# Legacy rollback-only HMAC; Capability v2 deployments leave this blank.
|
||||
KNOWLEDGE_FS_JWT_SECRET=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED=false
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS=
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER=dify-control-plane
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE=knowledge-fs
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS=60
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS=300
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES=4194304
|
||||
|
||||
# Marketplace configuration
|
||||
MARKETPLACE_ENABLED=true
|
||||
|
||||
@@ -154,6 +154,7 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_forward_refs,
|
||||
ext_hosting_provider,
|
||||
ext_import_modules,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_logging,
|
||||
ext_login,
|
||||
ext_logstore,
|
||||
@@ -204,6 +205,7 @@ def initialize_extensions(app: DifyApp):
|
||||
ext_enterprise_telemetry,
|
||||
ext_request_logging,
|
||||
ext_session_factory,
|
||||
ext_knowledge_fs_observability,
|
||||
ext_oauth_bearer,
|
||||
]
|
||||
for ext in extensions:
|
||||
|
||||
@@ -10,6 +10,7 @@ from .data_migration import (
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
)
|
||||
from .knowledge_fs import knowledge_fs_control_space
|
||||
from .plugin import (
|
||||
backfill_plugin_auto_upgrade,
|
||||
extract_plugins,
|
||||
@@ -75,6 +76,7 @@ __all__ = [
|
||||
"import_migration_data",
|
||||
"install_plugins",
|
||||
"install_rag_pipeline_plugins",
|
||||
"knowledge_fs_control_space",
|
||||
"legacy_model_types",
|
||||
"migrate_annotation_vector_database",
|
||||
"migrate_data_for_plugin",
|
||||
|
||||
@@ -0,0 +1,604 @@
|
||||
"""Operator commands for the independent KnowledgeFS control-plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from services.knowledge_fs.cleanup import (
|
||||
CleanupApprovalInput,
|
||||
CleanupCompletionEvidenceInput,
|
||||
CleanupReadinessEvidenceInput,
|
||||
CleanupStartInput,
|
||||
KnowledgeFSCleanupError,
|
||||
KnowledgeFSCleanupService,
|
||||
)
|
||||
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleError
|
||||
from services.knowledge_fs.control_space_management import (
|
||||
KnowledgeFSControlSpaceManagementService,
|
||||
KnowledgeFSControlSpaceRegistration,
|
||||
)
|
||||
from services.knowledge_fs.cutover import (
|
||||
CutoverSmokeResultsInput,
|
||||
FinalDeltaInput,
|
||||
KnowledgeFSCutoverError,
|
||||
KnowledgeFSWorkspaceCutoverService,
|
||||
LegacyDependencyInput,
|
||||
QuarantineResolutionInput,
|
||||
ShadowAuthorizationObservationInput,
|
||||
ShadowCompletionInput,
|
||||
WorkspaceInventoryInput,
|
||||
)
|
||||
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
|
||||
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
|
||||
|
||||
|
||||
@click.group("knowledge-fs-control-space")
|
||||
def knowledge_fs_control_space() -> None:
|
||||
"""Inspect and repair Dify-owned KnowledgeFS control-space state."""
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("dry-run")
|
||||
@click.option("--tenant-id", default=None)
|
||||
def dry_run(tenant_id: str | None) -> None:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("inventory")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Create ledgers; omitted means read-only inventory.")
|
||||
def inventory(input_path: Path, apply: bool) -> None:
|
||||
"""Validate strict Workspace inventory JSONL and optionally create ledgers."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.inventory, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("register")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--owner-account-id", required=True)
|
||||
@click.option("--provisioning-key", required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def register(
|
||||
tenant_id: str,
|
||||
owner_account_id: str,
|
||||
provisioning_key: str,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space, replayed = _management_service().register(
|
||||
KnowledgeFSControlSpaceRegistration(
|
||||
tenant_id,
|
||||
owner_account_id,
|
||||
provisioning_key,
|
||||
knowledge_space_id,
|
||||
knowledge_space_revision,
|
||||
)
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "replayed": replayed}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("backfill")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist registrations; omitted means dry-run.")
|
||||
def backfill(input_path: Path, apply: bool) -> None:
|
||||
"""Backfill strict Workspace inventory JSONL; dry-run unless --apply is explicit."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, WorkspaceInventoryInput):
|
||||
report = _operator_call(partial(service.backfill, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("quarantine-resolve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist resolutions; omitted means dry-run.")
|
||||
def quarantine_resolve(input_path: Path, apply: bool) -> None:
|
||||
"""Resolve strict tenant-scoped quarantine JSONL with immutable operator evidence."""
|
||||
|
||||
service = _cutover_service()
|
||||
for payload in _read_jsonl(input_path, QuarantineResolutionInput):
|
||||
report = _operator_call(partial(service.resolve_quarantine, payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-start")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "started_at", default=None, help="Optional explicit timezone-aware shadow start.")
|
||||
def shadow_start(tenant_id: str, expected_cas_version: int, started_at: str | None) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.begin_shadow(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at) if started_at is not None else None,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-report")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist observations; omitted means dry-run.")
|
||||
def shadow_report(input_path: Path, apply: bool) -> None:
|
||||
observations = _read_jsonl(input_path, ShadowAuthorizationObservationInput)
|
||||
report = _operator_call(lambda: _cutover_service().record_shadow_report(observations, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist completion; omitted means dry-run.")
|
||||
def shadow_complete(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, ShadowCompletionInput)
|
||||
report = _operator_call(lambda: _cutover_service().complete_shadow(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def issue_approve(tenant_id: str, issue_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_issue_fail_closed(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("issue-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--issue-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def issue_resolve(tenant_id: str, issue_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_issue(
|
||||
tenant_id=tenant_id,
|
||||
issue_key=issue_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-approve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "approved_at", required=True)
|
||||
def shadow_approve(tenant_id: str, diff_key: str, account_id: str, approved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().approve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
approved_at=_parse_timestamp(approved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("shadow-resolve")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--diff-key", required=True)
|
||||
@click.option("--account-id", required=True)
|
||||
@click.option("--at", "resolved_at", required=True)
|
||||
def shadow_resolve(tenant_id: str, diff_key: str, account_id: str, resolved_at: str) -> None:
|
||||
_operator_call(
|
||||
lambda: _cutover_service().resolve_shadow_diff(
|
||||
tenant_id=tenant_id,
|
||||
diff_key=diff_key,
|
||||
account_id=account_id,
|
||||
resolved_at=_parse_timestamp(resolved_at),
|
||||
)
|
||||
)
|
||||
_echo_json(_cutover_service().status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-dashboard")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--checked-at", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), default=None)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist gate evidence; omitted means read-only report.")
|
||||
def legacy_dashboard(
|
||||
tenant_id: str,
|
||||
input_path: Path,
|
||||
checked_at: str,
|
||||
expected_cas_version: int | None,
|
||||
apply: bool,
|
||||
) -> None:
|
||||
dependencies = _read_jsonl(input_path, LegacyDependencyInput, allow_empty=True)
|
||||
report = _operator_call(
|
||||
lambda: _cutover_service().legacy_dependency_dashboard(
|
||||
tenant_id=tenant_id,
|
||||
dependencies=dependencies,
|
||||
expected_cas_version=expected_cas_version,
|
||||
checked_at=_parse_timestamp(checked_at),
|
||||
apply=apply,
|
||||
)
|
||||
)
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("legacy-check")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def legacy_check(tenant_id: str) -> None:
|
||||
status_report = _operator_call(lambda: _cutover_service().status(tenant_id=tenant_id))
|
||||
passed = (
|
||||
bool(status_report["legacy_dependency_ready"])
|
||||
and status_report["open_issues"] == 0
|
||||
and status_report["unresolved_cutover_quarantine"] == 0
|
||||
)
|
||||
_echo_json({"tenant_id": tenant_id, "passed": passed, "status": status_report})
|
||||
if not passed:
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("freeze")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "freeze_at", required=True)
|
||||
def freeze(tenant_id: str, expected_cas_version: int, freeze_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.freeze(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
freeze_at=_parse_timestamp(freeze_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("final-delta")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def final_delta(input_path: Path) -> None:
|
||||
payload = _read_one_jsonl(input_path, FinalDeltaInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(lambda: service.apply_final_delta(payload))
|
||||
_echo_json(service.status(tenant_id=str(payload.tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cutover")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "cutover_at", required=True)
|
||||
@click.option("--rollback-cutoff-at", required=True)
|
||||
def cutover(tenant_id: str, expected_cas_version: int, cutover_at: str, rollback_cutoff_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.cutover(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
cutover_at=_parse_timestamp(cutover_at),
|
||||
rollback_cutoff_at=_parse_timestamp(rollback_cutoff_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("smoke")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
def smoke(tenant_id: str, expected_cas_version: int, input_path: Path) -> None:
|
||||
results = _read_one_jsonl(input_path, CutoverSmokeResultsInput)
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.record_smoke_results(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
results=results,
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("observe")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--started-at", default=None)
|
||||
@click.option("--window-ends-at", default=None)
|
||||
@click.option("--maximum-task-expires-at", default=None)
|
||||
@click.option("--observed-at", default=None)
|
||||
def observe(
|
||||
tenant_id: str,
|
||||
expected_cas_version: int,
|
||||
started_at: str | None,
|
||||
window_ends_at: str | None,
|
||||
maximum_task_expires_at: str | None,
|
||||
observed_at: str | None,
|
||||
) -> None:
|
||||
service = _cutover_service()
|
||||
if observed_at is not None:
|
||||
if any(value is not None for value in (started_at, window_ends_at, maximum_task_expires_at)):
|
||||
raise click.UsageError("--observed-at cannot be combined with observation start options")
|
||||
_operator_call(
|
||||
lambda: service.complete_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
observed_at=_parse_timestamp(observed_at),
|
||||
)
|
||||
)
|
||||
else:
|
||||
if started_at is None or window_ends_at is None or maximum_task_expires_at is None:
|
||||
raise click.UsageError(
|
||||
"observation start requires --started-at, --window-ends-at, and --maximum-task-expires-at"
|
||||
)
|
||||
_operator_call(
|
||||
lambda: service.begin_observation(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
started_at=_parse_timestamp(started_at),
|
||||
window_ends_at=_parse_timestamp(window_ends_at),
|
||||
maximum_task_expires_at=_parse_timestamp(maximum_task_expires_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("rollback")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--expected-cas-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--at", "rolled_back_at", required=True)
|
||||
def rollback(tenant_id: str, expected_cas_version: int, rolled_back_at: str) -> None:
|
||||
service = _cutover_service()
|
||||
_operator_call(
|
||||
lambda: service.rollback(
|
||||
tenant_id=tenant_id,
|
||||
expected_cas_version=expected_cas_version,
|
||||
rolled_back_at=_parse_timestamp(rolled_back_at),
|
||||
)
|
||||
)
|
||||
_echo_json(service.status(tenant_id=tenant_id))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
def status(tenant_id: str) -> None:
|
||||
_echo_json(_operator_call(lambda: _cutover_service().status(tenant_id=tenant_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-request")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist readiness evidence; omitted means dry-run.")
|
||||
def cleanup_request(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupReadinessEvidenceInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().request_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-approve")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist four-eyes approval; omitted means dry-run.")
|
||||
def cleanup_approve(input_path: Path, apply: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupApprovalInput)
|
||||
report = _cleanup_call(lambda: _cleanup_service().approve_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-start")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist the irreversible fence; never runs deletion.")
|
||||
@click.option(
|
||||
"--acknowledge-irreversible",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms rollback will be permanently closed.",
|
||||
)
|
||||
def cleanup_start(input_path: Path, apply: bool, acknowledge_irreversible: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupStartInput)
|
||||
if apply and not acknowledge_irreversible:
|
||||
raise click.UsageError("--apply requires --acknowledge-irreversible")
|
||||
report = _cleanup_call(lambda: _cleanup_service().start_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-complete")
|
||||
@click.option("--input", "input_path", type=click.Path(path_type=Path, exists=True, dir_okay=False), required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist externally verified cleanup completion.")
|
||||
@click.option(
|
||||
"--acknowledge-executed",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply; confirms the reviewed destructive bundle already executed.",
|
||||
)
|
||||
def cleanup_complete(input_path: Path, apply: bool, acknowledge_executed: bool) -> None:
|
||||
payload = _read_one_jsonl(input_path, CleanupCompletionEvidenceInput)
|
||||
if apply and not acknowledge_executed:
|
||||
raise click.UsageError("--apply requires --acknowledge-executed")
|
||||
report = _cleanup_call(lambda: _cleanup_service().complete_cleanup(payload, apply=apply))
|
||||
_echo_json(report._asdict())
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("cleanup-status")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--request-id", required=True)
|
||||
def cleanup_status(tenant_id: str, request_id: str) -> None:
|
||||
_echo_json(_cleanup_call(lambda: _cleanup_service().status(tenant_id=tenant_id, request_id=request_id)))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("repair")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--control-space-id", required=True)
|
||||
@click.option("--expected-resource-version", type=click.IntRange(min=0), required=True)
|
||||
@click.option("--knowledge-space-id", required=True)
|
||||
@click.option("--knowledge-space-revision", type=click.IntRange(min=0), required=True)
|
||||
def repair(
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
expected_resource_version: int,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> None:
|
||||
control_space = _management_service().repair_registration(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=expected_resource_version,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
click.echo(json.dumps({"control_space_id": control_space.id, "state": control_space.state.value}, sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("orphan-report")
|
||||
@click.option("--limit", type=click.IntRange(min=1, max=10_000), default=500, show_default=True)
|
||||
def orphan_report(limit: int) -> None:
|
||||
report = KnowledgeFSOrphanReconciler(
|
||||
session_factory.get_session_maker(),
|
||||
get_knowledge_fs_lifecycle_remote(),
|
||||
).reconcile(limit=limit, apply_repairs=False)
|
||||
click.echo(json.dumps(report._asdict(), sort_keys=True))
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-request")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Persist durable deletion intents; omitted means dry-run.")
|
||||
def workspace_delete_request(tenant_id: str, apply: bool) -> None:
|
||||
"""Route every KnowledgeFS Space through the canonical lifecycle deletion path."""
|
||||
|
||||
if not apply:
|
||||
report = _management_service().dry_run(tenant_id=tenant_id)
|
||||
_echo_json(
|
||||
{
|
||||
"apply": False,
|
||||
"by_state": report.by_state,
|
||||
"tenant_id": tenant_id,
|
||||
"total": report.total,
|
||||
}
|
||||
)
|
||||
return
|
||||
results = _lifecycle_call(lambda: _lifecycle_service().request_workspace_cleanup(tenant_id=tenant_id))
|
||||
_echo_json(
|
||||
{
|
||||
"apply": True,
|
||||
"control_space_ids": [result.control_space.id for result in results],
|
||||
"operation_ids": [result.outbox.operation_id for result in results if result.outbox is not None],
|
||||
"tenant_id": tenant_id,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@knowledge_fs_control_space.command("workspace-delete-finalize")
|
||||
@click.option("--tenant-id", required=True)
|
||||
@click.option("--apply", is_flag=True, default=False, help="Purge terminal local control-plane rows.")
|
||||
@click.option(
|
||||
"--acknowledge-control-plane-purge",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Required with --apply after every remote Space has reached deleted.",
|
||||
)
|
||||
def workspace_delete_finalize(tenant_id: str, apply: bool, acknowledge_control_plane_purge: bool) -> None:
|
||||
"""Release the Workspace FK only after all remote deletions are terminal."""
|
||||
|
||||
service = _lifecycle_service()
|
||||
if not apply:
|
||||
_lifecycle_call(lambda: service.assert_workspace_deletion_allowed(tenant_id=tenant_id))
|
||||
_echo_json({"apply": False, "ready": True, "tenant_id": tenant_id})
|
||||
return
|
||||
if not acknowledge_control_plane_purge:
|
||||
raise click.UsageError("--apply requires --acknowledge-control-plane-purge")
|
||||
deleted = _lifecycle_call(lambda: service.finalize_workspace_deletion(tenant_id=tenant_id))
|
||||
_echo_json({"apply": True, "purged_control_spaces": deleted, "tenant_id": tenant_id})
|
||||
|
||||
|
||||
def _read_jsonl[InputT: BaseModel](
|
||||
input_path: Path, input_type: type[InputT], *, allow_empty: bool = False
|
||||
) -> tuple[InputT, ...]:
|
||||
records: list[InputT] = []
|
||||
for line_number, line in enumerate(input_path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
records.append(input_type.model_validate_json(line))
|
||||
except ValidationError as exc:
|
||||
raise click.ClickException(f"invalid strict JSONL at line {line_number}: {exc}") from exc
|
||||
if not records and not allow_empty:
|
||||
raise click.ClickException("strict JSONL input must contain at least one record")
|
||||
return tuple(records)
|
||||
|
||||
|
||||
def _read_one_jsonl[InputT: BaseModel](input_path: Path, input_type: type[InputT]) -> InputT:
|
||||
records = _read_jsonl(input_path, input_type)
|
||||
if len(records) != 1:
|
||||
raise click.ClickException("this command requires exactly one JSONL record")
|
||||
return records[0]
|
||||
|
||||
|
||||
def _parse_timestamp(value: str) -> datetime:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError as exc:
|
||||
raise click.ClickException(f"invalid ISO-8601 timestamp: {value}") from exc
|
||||
if parsed.tzinfo is None:
|
||||
raise click.ClickException("operator timestamps must include an explicit timezone")
|
||||
return parsed
|
||||
|
||||
|
||||
def _operator_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCutoverError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _cleanup_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSCleanupError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _lifecycle_call[ResultT](operation: Callable[[], ResultT]) -> ResultT:
|
||||
try:
|
||||
return operation()
|
||||
except KnowledgeFSControlSpaceLifecycleError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _echo_json(payload: object) -> None:
|
||||
click.echo(json.dumps(payload, default=str, sort_keys=True))
|
||||
|
||||
|
||||
def _management_service() -> KnowledgeFSControlSpaceManagementService:
|
||||
return KnowledgeFSControlSpaceManagementService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _cutover_service() -> KnowledgeFSWorkspaceCutoverService:
|
||||
return KnowledgeFSWorkspaceCutoverService(
|
||||
session_factory.get_session_maker(),
|
||||
remote_factory=get_knowledge_fs_lifecycle_remote,
|
||||
)
|
||||
|
||||
|
||||
def _cleanup_service() -> KnowledgeFSCleanupService:
|
||||
return KnowledgeFSCleanupService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
def _lifecycle_service() -> KnowledgeFSControlSpaceCommandService:
|
||||
return KnowledgeFSControlSpaceCommandService(session_factory.get_session_maker())
|
||||
|
||||
|
||||
__all__ = ["knowledge_fs_control_space"]
|
||||
@@ -1,30 +1,68 @@
|
||||
"""Configuration for the optional KnowledgeFS Console bridge."""
|
||||
"""Configuration for the optional KnowledgeFS control-plane integration."""
|
||||
|
||||
from ipaddress import ip_address
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, PositiveFloat, SecretStr, field_validator, model_validator
|
||||
from pydantic import Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class KnowledgeFSConfig(BaseSettings):
|
||||
"""Server-only settings for the KnowledgeFS production connection."""
|
||||
"""Server-only KnowledgeFS connection and rollout settings."""
|
||||
|
||||
KNOWLEDGE_FS_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable the private KnowledgeFS Console bridge.",
|
||||
)
|
||||
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
|
||||
KNOWLEDGE_FS_JWT_SECRET: SecretStr | None = Field(
|
||||
default=None,
|
||||
min_length=32,
|
||||
description="Shared secret used to sign short-lived KnowledgeFS service JWTs.",
|
||||
KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable delivery of durable KnowledgeFS lifecycle commands after every rollout gate is ready.",
|
||||
)
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS: PositiveFloat = Field(default=300.0, le=3600.0, allow_inf_nan=False)
|
||||
KNOWLEDGE_FS_INTEGRATED_PROVISION_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that the Capability-v2 integrated provision route is deployed and verified.",
|
||||
)
|
||||
KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY: bool = Field(
|
||||
default=False,
|
||||
description="Confirm that legacy KFS ACL mutation is frozen for integrated mode.",
|
||||
)
|
||||
KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS: PositiveInt = Field(default=15, le=300)
|
||||
KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS: PositiveInt = Field(default=60, le=600)
|
||||
KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE: PositiveInt = Field(default=25, le=1_000)
|
||||
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
|
||||
KNOWLEDGE_FS_DIRECT_ORIGIN: str | None = Field(
|
||||
default=None,
|
||||
description="Public KnowledgeFS origin returned with direct upload capabilities.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Prepare resource-scoped Capability v2 issuance; disabled until rollout approval.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID: str | None = Field(
|
||||
default=None,
|
||||
description="Identifier for the current asymmetric Capability v2 signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM: SecretStr | None = Field(
|
||||
default=None,
|
||||
description="Server-only PEM for the current Capability v2 RSA signing key.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS: str | None = Field(
|
||||
default=None,
|
||||
description="Optional public-only JWKS JSON retained during key rotation overlap.",
|
||||
)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_ISSUER: str = Field(default="dify-control-plane", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_AUDIENCE: str = Field(default="knowledge-fs", min_length=1)
|
||||
KNOWLEDGE_FS_CAPABILITY_V2_MAX_TTL_SECONDS: PositiveInt = Field(default=60, le=60)
|
||||
KNOWLEDGE_FS_JWKS_CACHE_MAX_AGE_SECONDS: PositiveInt = Field(default=300, le=86_400)
|
||||
KNOWLEDGE_FS_PRODUCT_MAX_RESPONSE_BYTES: PositiveInt = Field(default=4 * 1024 * 1024, le=16 * 1024 * 1024)
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
|
||||
|
||||
@field_validator(
|
||||
"KNOWLEDGE_FS_BASE_URL",
|
||||
"KNOWLEDGE_FS_JWT_SECRET",
|
||||
"KNOWLEDGE_FS_DIRECT_ORIGIN",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM",
|
||||
"KNOWLEDGE_FS_CAPABILITY_V2_PREVIOUS_PUBLIC_JWKS",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
@@ -40,25 +78,57 @@ class KnowledgeFSConfig(BaseSettings):
|
||||
@field_validator("KNOWLEDGE_FS_BASE_URL")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_BASE_URL")
|
||||
|
||||
@field_validator("KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
@classmethod
|
||||
def validate_direct_origin(cls, value: str | None) -> str | None:
|
||||
return cls._validate_origin(value, name="KNOWLEDGE_FS_DIRECT_ORIGIN")
|
||||
|
||||
@classmethod
|
||||
def _validate_origin(cls, value: str | None, *, name: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must be an absolute HTTP(S) URL")
|
||||
raise ValueError(f"{name} must be an absolute HTTP(S) URL")
|
||||
try:
|
||||
_ = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must not include credentials, query, or fragment")
|
||||
raise ValueError(f"{name} must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment or parsed.path not in {"", "/"}:
|
||||
raise ValueError(f"{name} must be an origin without credentials, path, query, or fragment")
|
||||
return value.rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
|
||||
if not self.KNOWLEDGE_FS_ENABLED:
|
||||
return self
|
||||
if bool(self.KNOWLEDGE_FS_BASE_URL) != bool(self.KNOWLEDGE_FS_JWT_SECRET):
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL and KNOWLEDGE_FS_JWT_SECRET must be configured together")
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS connection settings are required when the integration is enabled")
|
||||
if str(getattr(self, "DEPLOY_ENV", "")).strip().upper() == "PRODUCTION":
|
||||
for name, value in (
|
||||
("KNOWLEDGE_FS_BASE_URL", self.KNOWLEDGE_FS_BASE_URL),
|
||||
("KNOWLEDGE_FS_DIRECT_ORIGIN", self.KNOWLEDGE_FS_DIRECT_ORIGIN),
|
||||
):
|
||||
if value and not self._is_secure_or_loopback_origin(value):
|
||||
raise ValueError(f"{name} must use HTTPS in production unless it targets loopback")
|
||||
if self.KNOWLEDGE_FS_ENABLED:
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS base URL is required when the integration is enabled")
|
||||
if not self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED:
|
||||
raise ValueError("KnowledgeFS product routes require Capability v2 when enabled")
|
||||
if self.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED and not (
|
||||
self.KNOWLEDGE_FS_CAPABILITY_V2_SIGNING_KID and self.KNOWLEDGE_FS_CAPABILITY_V2_PRIVATE_KEY_PEM
|
||||
):
|
||||
raise ValueError("Capability v2 signing kid and private key are required when issuance is enabled")
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def _is_secure_or_loopback_origin(value: str) -> bool:
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme == "https":
|
||||
return True
|
||||
hostname = (parsed.hostname or "").rstrip(".").lower()
|
||||
if hostname == "localhost":
|
||||
return True
|
||||
try:
|
||||
return ip_address(hostname).is_loopback
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@@ -38,7 +38,6 @@ from . import (
|
||||
feature,
|
||||
human_input_form,
|
||||
init_validate,
|
||||
knowledge_fs_proxy,
|
||||
notification,
|
||||
ping,
|
||||
setup,
|
||||
@@ -126,6 +125,7 @@ from .explore import (
|
||||
saved_message,
|
||||
trial,
|
||||
)
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .snippets import snippet_workflow, snippet_workflow_draft_variable
|
||||
from .socketio import workflow as socketio_workflow
|
||||
|
||||
@@ -196,7 +196,7 @@ __all__ = [
|
||||
"human_input_form",
|
||||
"init_validate",
|
||||
"installed_app",
|
||||
"knowledge_fs_proxy",
|
||||
"knowledge_fs_resources",
|
||||
"load_balancing_config",
|
||||
"login",
|
||||
"mcp_server",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Typed Dify-owned KnowledgeFS Console product API."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Stable, non-enumerating Console error contract for KnowledgeFS."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSSpaceNotFoundHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_space_not_found"
|
||||
description = "KnowledgeFS space was not found."
|
||||
code = 404
|
||||
|
||||
|
||||
class KnowledgeFSOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSAccessDeniedHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_access_denied"
|
||||
description = "KnowledgeFS operation is not allowed."
|
||||
code = 403
|
||||
|
||||
|
||||
class KnowledgeFSRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSAccessDeniedHTTPError",
|
||||
"KnowledgeFSInvalidRequestHTTPError",
|
||||
"KnowledgeFSOperationUnavailableHTTPError",
|
||||
"KnowledgeFSQuotaExceededHTTPError",
|
||||
"KnowledgeFSRateLimitHTTPError",
|
||||
"KnowledgeFSSpaceNotFoundHTTPError",
|
||||
"KnowledgeFSUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,332 +0,0 @@
|
||||
"""Authenticated transport adapter for the Console-to-KnowledgeFS proxy.
|
||||
|
||||
These raw Blueprint routes deliberately stay outside Dify's OpenAPI surface:
|
||||
KnowledgeFS owns the wire contract consumed by the frontend. The catch-all path
|
||||
avoids resource-specific Dify controllers, while the forwarding module consumes
|
||||
only the operations explicitly enabled by Dify's product registry. The registry
|
||||
can be validated explicitly against the pinned KnowledgeFS contract during development.
|
||||
Console auth and contract-specific dataset RBAC run before forwarding. Request
|
||||
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
|
||||
SSE responses remain streaming with a bounded idle read timeout, and only safe
|
||||
response headers are exposed. Upstream 401 responses become 502 so they cannot
|
||||
trigger Dify browser-session recovery; resource-level 403 responses remain 403.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Iterator
|
||||
from functools import wraps
|
||||
from http import HTTPStatus
|
||||
from typing import NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from flask import Response, request, stream_with_context
|
||||
from flask.typing import ResponseReturnValue
|
||||
from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
GatewayTimeout,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
)
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.console import api, bp
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
cloud_edition_billing_rate_limit_check,
|
||||
setup_required,
|
||||
)
|
||||
from core.helper import ssrf_proxy
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
|
||||
_RESPONSE_HEADER_ALLOWLIST = (
|
||||
"Cache-Control",
|
||||
"Content-Disposition",
|
||||
"Content-Type",
|
||||
"Retry-After",
|
||||
"X-Trace-Id",
|
||||
)
|
||||
_RESPONSE_HEADER_DENYLIST = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"connection",
|
||||
"cookie",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"set-cookie",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _console_api_errors[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Route raw Blueprint exceptions through the Console API JSON handlers."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
try:
|
||||
return view(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
return api.handle_error(exc)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _knowledge_fs_enabled[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Hide the complete KnowledgeFS route surface while the bridge is disabled."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
return view(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
|
||||
"""Map forwarding failures to the stable Console HTTP error surface."""
|
||||
if isinstance(exc, KnowledgeFSRouteNotAllowedError):
|
||||
raise NotFound() from exc
|
||||
if isinstance(exc, KnowledgeFSAccessDeniedError):
|
||||
raise Forbidden() from exc
|
||||
if isinstance(exc, KnowledgeFSConfigurationError):
|
||||
logger.error("KnowledgeFS request was blocked by invalid configuration for tenant_id=%s", tenant_id)
|
||||
raise ServiceUnavailable("KnowledgeFS integration is misconfigured") from exc
|
||||
if isinstance(exc, KnowledgeFSTimeoutError):
|
||||
raise GatewayTimeout("KnowledgeFS request timed out") from exc
|
||||
if isinstance(exc, KnowledgeFSTransportError):
|
||||
logger.warning("KnowledgeFS transport request failed for tenant_id=%s", tenant_id)
|
||||
raise BadGateway("KnowledgeFS is unavailable") from exc
|
||||
raise exc
|
||||
|
||||
|
||||
def _knowledge_fs_operation_access_required(
|
||||
view: Callable[[KnowledgeFSMethod, str], ResponseReturnValue],
|
||||
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
|
||||
"""Authorize one declared operation before billing and request-body work."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
|
||||
try:
|
||||
operation = get_knowledge_fs_operation(method, upstream_path)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
authorize_knowledge_fs_request(
|
||||
account=current_user,
|
||||
tenant_id=tenant_id,
|
||||
operation=operation,
|
||||
)
|
||||
except KnowledgeFSAccessDeniedError as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return view(method, upstream_path)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _request_body() -> bytes:
|
||||
"""Read the raw body up to the proxy limit or raise RequestEntityTooLarge."""
|
||||
body = request.stream.read(_MAX_PROXY_BODY_BYTES + 1)
|
||||
if len(body) > _MAX_PROXY_BODY_BYTES:
|
||||
raise RequestEntityTooLarge("KnowledgeFS proxy request body is too large")
|
||||
return body
|
||||
|
||||
|
||||
def _stream_response_body(
|
||||
upstream: httpx.Response,
|
||||
*,
|
||||
tenant_id: str,
|
||||
max_response_bytes: int,
|
||||
) -> Iterator[bytes]:
|
||||
"""Yield one bounded SSE response and always release its pooled connection."""
|
||||
total_bytes = 0
|
||||
try:
|
||||
for chunk in upstream.iter_bytes():
|
||||
total_bytes += len(chunk)
|
||||
if total_bytes > max_response_bytes:
|
||||
logger.warning("KnowledgeFS stream exceeded the proxy limit for tenant_id=%s", tenant_id)
|
||||
raise ssrf_proxy.ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
|
||||
yield chunk
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
|
||||
def _proxy_response(
|
||||
upstream_result: KnowledgeFSUpstreamResponse,
|
||||
*,
|
||||
tenant_id: str,
|
||||
contract_response_headers: tuple[str, ...],
|
||||
max_response_bytes: int,
|
||||
) -> Response:
|
||||
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
BadGateway: KnowledgeFS rejects the configured server credential.
|
||||
Forbidden: KnowledgeFS denies the account access to the requested resource.
|
||||
"""
|
||||
upstream = upstream_result.response
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
upstream.close()
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
raise BadGateway("KnowledgeFS authentication failed")
|
||||
if upstream.status_code == HTTPStatus.FORBIDDEN:
|
||||
upstream.close()
|
||||
raise Forbidden()
|
||||
|
||||
allowed_header_names = dict.fromkeys(
|
||||
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
|
||||
)
|
||||
headers = {
|
||||
name: value
|
||||
for name in allowed_header_names
|
||||
if name not in _RESPONSE_HEADER_DENYLIST
|
||||
if (value := upstream.headers.get(name)) is not None
|
||||
}
|
||||
if upstream_result.response_kind == "stream":
|
||||
response = Response(
|
||||
stream_with_context( # pyrefly: ignore[no-matching-overload]
|
||||
_stream_response_body(
|
||||
upstream,
|
||||
tenant_id=tenant_id,
|
||||
max_response_bytes=max_response_bytes,
|
||||
)
|
||||
),
|
||||
status=upstream.status_code,
|
||||
headers=headers,
|
||||
)
|
||||
response.call_on_close(upstream.close)
|
||||
return response
|
||||
|
||||
try:
|
||||
content = upstream.content
|
||||
finally:
|
||||
upstream.close()
|
||||
return Response(content, status=upstream.status_code, headers=headers)
|
||||
|
||||
|
||||
def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
|
||||
"""Forward the current raw request and return its filtered upstream response.
|
||||
|
||||
The call performs one outbound KnowledgeFS request. Integration failures are
|
||||
converted to Console HTTP exceptions for the outer JSON error adapter.
|
||||
"""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
proxy_result = proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=request.headers.get("Accept"),
|
||||
content_type=request.content_type,
|
||||
query=request.query_string or None,
|
||||
body=_request_body() if method != "GET" else None,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
except (
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
) as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return _proxy_response(
|
||||
proxy_result,
|
||||
tenant_id=tenant_id,
|
||||
contract_response_headers=proxy_result.operation.response_headers,
|
||||
max_response_bytes=proxy_result.operation.max_response_bytes,
|
||||
)
|
||||
|
||||
|
||||
@_knowledge_fs_enabled
|
||||
@_knowledge_fs_operation_access_required
|
||||
@cloud_edition_billing_rate_limit_check("knowledge")
|
||||
def _proxy_knowledge_fs_non_get(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
) -> ResponseReturnValue:
|
||||
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
|
||||
return _proxy_request(method, upstream_path)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["GET", "OPTIONS"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_get(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated, dataset-readable GET request.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
if request.method != "GET":
|
||||
raise NotFound()
|
||||
return _proxy_request("GET", upstream_path)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["DELETE", "PATCH", "POST", "PUT"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_write(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated non-GET request under its contract access policy.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
method = cast(KnowledgeFSMethod, request.method)
|
||||
return _proxy_knowledge_fs_non_get(method, upstream_path)
|
||||
@@ -346,7 +346,13 @@ class RBACRoleItemApi(Resource):
|
||||
def put(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_RoleUpsertRequest)
|
||||
role = svc.RBACService.Roles.update(tenant_id, account_id, str(role_id), request.to_mutation())
|
||||
role = svc.RBACService.KnowledgeFSRoleMutations.update_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
request.to_mutation(),
|
||||
session=db.session(),
|
||||
)
|
||||
return _dump(role)
|
||||
|
||||
@login_required
|
||||
@@ -356,7 +362,12 @@ class RBACRoleItemApi(Resource):
|
||||
@console_ns.response(200, "Success", console_ns.models[svc.RBACRole.__name__])
|
||||
def delete(self, role_id):
|
||||
tenant_id, account_id = _current_ids()
|
||||
svc.RBACService.Roles.delete(tenant_id, account_id, str(role_id))
|
||||
svc.RBACService.KnowledgeFSRoleMutations.delete_role(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(role_id),
|
||||
session=db.session(),
|
||||
)
|
||||
return {"result": "success"}
|
||||
|
||||
|
||||
@@ -915,7 +926,7 @@ class RBACMemberRolesApi(Resource):
|
||||
tenant_id, account_id = _current_ids()
|
||||
request = _payload(_ReplaceMemberRolesRequest)
|
||||
return _dump(
|
||||
svc.RBACService.MemberRoles.replace(
|
||||
svc.RBACService.KnowledgeFSRoleMutations.replace_member_roles(
|
||||
tenant_id,
|
||||
account_id,
|
||||
str(member_id),
|
||||
|
||||
@@ -8,6 +8,7 @@ from controllers.inner_api.plugin.wraps import get_user_tenant, plugin_data
|
||||
from controllers.inner_api.wraps import plugin_inner_api_only
|
||||
from core.plugin.backwards_invocation.app import PluginAppBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocationResponse
|
||||
from core.plugin.backwards_invocation.datasource import PluginDatasourceBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.encrypt import PluginEncrypter
|
||||
from core.plugin.backwards_invocation.model import PluginModelBackwardsInvocation
|
||||
from core.plugin.backwards_invocation.node import PluginNodeBackwardsInvocation
|
||||
@@ -15,10 +16,12 @@ from core.plugin.backwards_invocation.tool import PluginToolBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
RequestFetchAppInfo,
|
||||
RequestInvokeApp,
|
||||
RequestInvokeDatasource,
|
||||
RequestInvokeEncrypt,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeParameterExtractorNode,
|
||||
RequestInvokeQuestionClassifierNode,
|
||||
RequestInvokeRerank,
|
||||
@@ -27,6 +30,7 @@ from core.plugin.entities.request import (
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTool,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
RequestRequestDownloadFile,
|
||||
RequestRequestUploadFile,
|
||||
)
|
||||
@@ -118,6 +122,36 @@ class PluginInvokeTextEmbeddingApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/multimodal-embedding")
|
||||
class PluginInvokeMultimodalEmbeddingApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeMultimodalEmbedding)
|
||||
@inner_api_ns.doc("plugin_invoke_multimodal_embedding")
|
||||
@inner_api_ns.doc(description="Invoke multimodal embedding models through Dify model management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Multimodal embedding successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestInvokeMultimodalEmbedding):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.invoke_multimodal_embedding(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/rerank")
|
||||
class PluginInvokeRerankApi(Resource):
|
||||
@get_user_tenant
|
||||
@@ -144,6 +178,70 @@ class PluginInvokeRerankApi(Resource):
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/model-catalog")
|
||||
class PluginModelCatalogApi(Resource):
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestListModels)
|
||||
@inner_api_ns.doc("plugin_model_catalog")
|
||||
@inner_api_ns.doc(description="List tenant-active models managed by Dify")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Model catalog lookup successful",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Service not available",
|
||||
}
|
||||
)
|
||||
def post(self, user_model: Account | EndUser, tenant_model: Tenant, payload: RequestListModels):
|
||||
try:
|
||||
return jsonable_encoder(
|
||||
BaseBackwardsInvocationResponse(
|
||||
data=PluginModelBackwardsInvocation.list_models(
|
||||
tenant_id=tenant_model.id,
|
||||
user_id=user_model.id,
|
||||
payload=payload,
|
||||
)
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
return jsonable_encoder(BaseBackwardsInvocationResponse(error=str(e)))
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/datasource")
|
||||
class PluginInvokeDatasourceApi(Resource):
|
||||
"""Invoke an installed datasource with credentials resolved inside Dify."""
|
||||
|
||||
@get_user_tenant
|
||||
@setup_required
|
||||
@plugin_inner_api_only
|
||||
@plugin_data(payload_type=RequestInvokeDatasource)
|
||||
@inner_api_ns.doc("plugin_invoke_datasource")
|
||||
@inner_api_ns.doc(description="Invoke datasource plugins through Dify credential management")
|
||||
@inner_api_ns.doc(
|
||||
responses={
|
||||
200: "Datasource invocation successful (streaming response)",
|
||||
401: "Unauthorized - invalid API key",
|
||||
404: "Datasource provider, datasource, or credential not found",
|
||||
}
|
||||
)
|
||||
def post(
|
||||
self,
|
||||
user_model: Account | EndUser,
|
||||
tenant_model: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
):
|
||||
response = PluginDatasourceBackwardsInvocation.invoke(
|
||||
user_id=user_model.id,
|
||||
tenant=tenant_model,
|
||||
payload=payload,
|
||||
)
|
||||
return length_prefixed_response(
|
||||
0xF,
|
||||
PluginDatasourceBackwardsInvocation.convert_to_event_stream(response),
|
||||
)
|
||||
|
||||
|
||||
@inner_api_ns.route("/invoke/tts")
|
||||
class PluginInvokeTTSApi(Resource):
|
||||
@get_user_tenant
|
||||
|
||||
@@ -38,6 +38,7 @@ from .dataset import (
|
||||
)
|
||||
from .dataset.rag_pipeline import rag_pipeline_workflow
|
||||
from .end_user import end_user
|
||||
from .knowledge_fs import resources as knowledge_fs_resources
|
||||
from .workspace import models
|
||||
|
||||
__all__ = [
|
||||
@@ -54,6 +55,7 @@ __all__ = [
|
||||
"hit_testing",
|
||||
"human_input_form",
|
||||
"index",
|
||||
"knowledge_fs_resources",
|
||||
"message",
|
||||
"metadata",
|
||||
"models",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""KnowledgeFS-specific Service API authenticated by resource credentials."""
|
||||
|
||||
from . import resources
|
||||
|
||||
__all__ = ["resources"]
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Stable KnowledgeFS Service API error contract."""
|
||||
|
||||
from libs.exception import BaseHTTPException
|
||||
|
||||
|
||||
class KnowledgeFSInvalidCredentialHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_credential"
|
||||
description = "Invalid KnowledgeFS service credential."
|
||||
code = 401
|
||||
|
||||
|
||||
class KnowledgeFSServiceOperationUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_operation_unavailable"
|
||||
description = "KnowledgeFS operation is not available."
|
||||
code = 503
|
||||
|
||||
|
||||
class KnowledgeFSServiceUpstreamUnavailableHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_upstream_unavailable"
|
||||
description = "KnowledgeFS is unavailable."
|
||||
code = 502
|
||||
|
||||
|
||||
class KnowledgeFSServiceInvalidRequestHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_invalid_request"
|
||||
description = "KnowledgeFS request is invalid."
|
||||
code = 400
|
||||
|
||||
|
||||
class KnowledgeFSServiceRateLimitHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_rate_limit_exceeded"
|
||||
description = "KnowledgeFS operation rate limit exceeded."
|
||||
code = 429
|
||||
|
||||
|
||||
class KnowledgeFSServiceQuotaExceededHTTPError(BaseHTTPException):
|
||||
error_code = "knowledge_fs_quota_exceeded"
|
||||
description = "KnowledgeFS operation quota exceeded."
|
||||
code = 403
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSInvalidCredentialHTTPError",
|
||||
"KnowledgeFSServiceInvalidRequestHTTPError",
|
||||
"KnowledgeFSServiceOperationUnavailableHTTPError",
|
||||
"KnowledgeFSServiceQuotaExceededHTTPError",
|
||||
"KnowledgeFSServiceRateLimitHTTPError",
|
||||
"KnowledgeFSServiceUpstreamUnavailableHTTPError",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,10 @@ from core.app.apps.base_app_queue_manager import AppQueueManager
|
||||
from core.app.apps.base_app_runner import AppRunner
|
||||
from core.app.entities.app_invoke_entities import (
|
||||
AgentChatAppGenerateEntity,
|
||||
DifyRunContext,
|
||||
InvokeFrom,
|
||||
ModelConfigWithCredentialsEntity,
|
||||
UserFrom,
|
||||
)
|
||||
from core.app.file_access import DatabaseFileAccessController
|
||||
from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler
|
||||
@@ -148,6 +151,15 @@ class BaseAgentRunner(AppRunner):
|
||||
user_id=self.user_id,
|
||||
invoke_from=self.application_generate_entity.invoke_from,
|
||||
)
|
||||
invoke_from = self.application_generate_entity.invoke_from
|
||||
if isinstance(invoke_from, InvokeFrom):
|
||||
tool_entity.runtime.dify_run_context = DifyRunContext(
|
||||
tenant_id=self.tenant_id,
|
||||
app_id=self.app_config.app_id,
|
||||
user_id=self.user_id,
|
||||
user_from=(UserFrom.ACCOUNT if invoke_from.runs_as_account() else UserFrom.END_USER),
|
||||
invoke_from=invoke_from,
|
||||
)
|
||||
assert tool_entity.entity.description
|
||||
message_tool = PromptMessageTool(
|
||||
name=tool.tool_name,
|
||||
|
||||
@@ -159,6 +159,7 @@ class AgentAppRuntimeRequestBuilder:
|
||||
user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
agent_mode="agent_app",
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
# ENG-616: expand slash-menu mention tokens to canonical names so
|
||||
# no frontend-internal {{#…#}} marker ever reaches the model.
|
||||
|
||||
@@ -38,6 +38,11 @@ class DatasourcePluginProviderController(ABC):
|
||||
):
|
||||
raise ToolProviderCredentialValidationError("Invalid credentials")
|
||||
|
||||
def validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
"""Validate credential shape and value against this installed provider declaration."""
|
||||
self.validate_credentials_format(credentials)
|
||||
self._validate_credentials(user_id, credentials)
|
||||
|
||||
@property
|
||||
def provider_type(self) -> DatasourceProviderType:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Datasource backward invocation through Dify's tenant-bound runtime.
|
||||
|
||||
This module is the internal service boundary used by trusted callers such as
|
||||
KnowledgeFS. It resolves installed provider declarations and Dify-owned
|
||||
credential references before reaching ``PluginDatasourceManager``; callers
|
||||
must never provide raw datasource credentials or a plugin-daemon API key.
|
||||
"""
|
||||
|
||||
from collections.abc import Generator
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.datasource.datasource_manager import DatasourceManager
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
OnlineDriveBrowseFilesRequest,
|
||||
OnlineDriveDownloadFileRequest,
|
||||
)
|
||||
from core.datasource.online_document.online_document_plugin import OnlineDocumentDatasourcePlugin
|
||||
from core.datasource.online_drive.online_drive_plugin import OnlineDriveDatasourcePlugin
|
||||
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import RequestInvokeDatasource
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import DatasourceProviderID
|
||||
from services.datasource_provider_service import DatasourceProviderService
|
||||
|
||||
|
||||
class PluginDatasourceBackwardsInvocation(BaseBackwardsInvocation):
|
||||
"""Resolve and invoke a datasource without exposing credential material to the caller."""
|
||||
|
||||
@classmethod
|
||||
def invoke(
|
||||
cls,
|
||||
*,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeDatasource,
|
||||
) -> Generator[BaseModel | dict[str, Any], None, None]:
|
||||
"""Yield datasource messages for one validated inner-runtime request."""
|
||||
provider_id = DatasourceProviderID(payload.provider)
|
||||
canonical_provider_id = str(provider_id)
|
||||
controller = DatasourceManager.get_datasource_plugin_provider(
|
||||
provider_id=canonical_provider_id,
|
||||
tenant_id=tenant.id,
|
||||
datasource_type=payload.datasource_type,
|
||||
)
|
||||
if controller.entity.provider_type != payload.datasource_type:
|
||||
raise ValueError("Datasource provider type mismatch")
|
||||
|
||||
# Resolving the datasource from the installed declaration prevents a caller
|
||||
# from dispatching an arbitrary datasource name under a valid plugin ID.
|
||||
runtime = controller.get_datasource(payload.datasource)
|
||||
credentials = DatasourceProviderService().get_datasource_credentials(
|
||||
tenant_id=tenant.id,
|
||||
provider=provider_id.provider_name,
|
||||
plugin_id=provider_id.plugin_id,
|
||||
credential_id=payload.credential_id,
|
||||
)
|
||||
if controller.need_credentials and not credentials:
|
||||
raise ValueError("Datasource credential not found")
|
||||
|
||||
if payload.operation == "validate_credentials":
|
||||
controller.validate_credentials(user_id=user_id, credentials=credentials)
|
||||
yield {"result": True}
|
||||
return
|
||||
|
||||
runtime.runtime.credentials = credentials
|
||||
provider_type = runtime.datasource_provider_type()
|
||||
|
||||
match payload.operation:
|
||||
case "get_website_crawl":
|
||||
website = cast(WebsiteCrawlDatasourcePlugin, runtime)
|
||||
yield from website.get_website_crawl(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_pages":
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_pages(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.datasource_parameters,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "get_online_document_page_content":
|
||||
if payload.page is None:
|
||||
raise ValueError("Online-document page input is required")
|
||||
document = cast(OnlineDocumentDatasourcePlugin, runtime)
|
||||
yield from document.get_online_document_page_content(
|
||||
user_id=user_id,
|
||||
datasource_parameters=payload.page,
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_browse_files":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_browse_files(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveBrowseFilesRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case "online_drive_download_file":
|
||||
drive = cast(OnlineDriveDatasourcePlugin, runtime)
|
||||
yield from drive.online_drive_download_file(
|
||||
user_id=user_id,
|
||||
request=OnlineDriveDownloadFileRequest.model_validate(payload.request),
|
||||
provider_type=provider_type,
|
||||
)
|
||||
case _:
|
||||
raise ValueError(f"Unsupported datasource operation: {payload.operation}")
|
||||
@@ -1,22 +1,31 @@
|
||||
import tempfile
|
||||
from binascii import hexlify, unhexlify
|
||||
from collections.abc import Generator
|
||||
from collections.abc import Generator, Mapping
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.app.llm import deduct_llm_quota
|
||||
from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
|
||||
from core.model_manager import ModelManager
|
||||
from core.plugin.backwards_invocation.base import BaseBackwardsInvocation
|
||||
from core.plugin.entities.request import (
|
||||
InvokableModelCatalogItem,
|
||||
InvokableModelCatalogPage,
|
||||
RequestInvokeLLM,
|
||||
RequestInvokeLLMWithStructuredOutput,
|
||||
RequestInvokeModeration,
|
||||
RequestInvokeMultimodalEmbedding,
|
||||
RequestInvokeRerank,
|
||||
RequestInvokeSpeech2Text,
|
||||
RequestInvokeSummary,
|
||||
RequestInvokeTextEmbedding,
|
||||
RequestInvokeTTS,
|
||||
RequestListModels,
|
||||
)
|
||||
from core.plugin.impl.model_runtime_factory import create_plugin_provider_manager
|
||||
from core.plugin.plugin_service import PluginService
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
from core.tools.utils.model_invocation_utils import ModelInvocationUtils
|
||||
from graphon.model_runtime.entities.llm_entities import (
|
||||
@@ -33,6 +42,20 @@ from graphon.model_runtime.entities.message_entities import (
|
||||
)
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from models.account import Tenant
|
||||
from models.provider_ids import ModelProviderID
|
||||
|
||||
|
||||
def _json_compatible(value: Any) -> Any:
|
||||
"""Convert model-runtime metadata into stable JSON-compatible values."""
|
||||
if isinstance(value, BaseModel):
|
||||
return value.model_dump(mode="json")
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if isinstance(value, Mapping):
|
||||
return {str(_json_compatible(key)): _json_compatible(child) for key, child in value.items()}
|
||||
if isinstance(value, list | tuple | set):
|
||||
return [_json_compatible(child) for child in value]
|
||||
return value
|
||||
|
||||
|
||||
class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
@@ -183,7 +206,30 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
)
|
||||
|
||||
# invoke model
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts)
|
||||
response = model_instance.invoke_text_embedding(texts=payload.texts, input_type=payload.input_type)
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def invoke_multimodal_embedding(
|
||||
cls,
|
||||
user_id: str,
|
||||
tenant: Tenant,
|
||||
payload: RequestInvokeMultimodalEmbedding,
|
||||
):
|
||||
"""Invoke multimodal embedding through the tenant-bound model instance."""
|
||||
model_instance = cls._get_bound_model_instance(
|
||||
tenant_id=tenant.id,
|
||||
user_id=user_id,
|
||||
provider=payload.provider,
|
||||
model_type=payload.model_type,
|
||||
model=payload.model,
|
||||
)
|
||||
|
||||
response = model_instance.invoke_multimodal_embedding(
|
||||
multimodel_documents=[document.model_dump(exclude_none=True) for document in payload.documents],
|
||||
input_type=payload.input_type,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
@@ -210,6 +256,67 @@ class PluginModelBackwardsInvocation(BaseBackwardsInvocation):
|
||||
|
||||
return response
|
||||
|
||||
@classmethod
|
||||
def list_models(
|
||||
cls,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
payload: RequestListModels,
|
||||
) -> InvokableModelCatalogPage:
|
||||
"""List only models that are active for the tenant's Dify configuration."""
|
||||
provider_manager = create_plugin_provider_manager(tenant_id=tenant_id, user_id=user_id)
|
||||
active_models = provider_manager.get_configurations(tenant_id).get_models(
|
||||
model_type=payload.model_type,
|
||||
only_active=True,
|
||||
)
|
||||
|
||||
installed_identities: dict[str, str] = {}
|
||||
for plugin in PluginService.list(tenant_id):
|
||||
existing = installed_identities.get(plugin.plugin_id)
|
||||
if existing is not None and existing != plugin.plugin_unique_identifier:
|
||||
raise ValueError(f"Ambiguous installed identity for model plugin {plugin.plugin_id}")
|
||||
installed_identities[plugin.plugin_id] = plugin.plugin_unique_identifier
|
||||
|
||||
requested_provider = str(ModelProviderID(payload.provider)) if payload.provider else None
|
||||
matched_models = [
|
||||
model
|
||||
for model in active_models
|
||||
if (requested_provider is None or model.provider.provider == requested_provider)
|
||||
and (payload.model is None or model.model == payload.model)
|
||||
]
|
||||
matched_models.sort(key=lambda model: (model.provider.provider, model.model))
|
||||
|
||||
page_models = matched_models[payload.offset : payload.offset + payload.limit]
|
||||
items: list[InvokableModelCatalogItem] = []
|
||||
for model in page_models:
|
||||
provider_id = ModelProviderID(model.provider.provider)
|
||||
unique_identifier = installed_identities.get(provider_id.plugin_id)
|
||||
if unique_identifier is None:
|
||||
raise ValueError(f"Installed identity not found for active model plugin {provider_id.plugin_id}")
|
||||
items.append(
|
||||
InvokableModelCatalogItem(
|
||||
plugin_id=provider_id.plugin_id,
|
||||
plugin_unique_identifier=unique_identifier,
|
||||
provider=provider_id.provider_name,
|
||||
model=model.model,
|
||||
model_type=model.model_type,
|
||||
capabilities={
|
||||
"deprecated": model.deprecated,
|
||||
"features": _json_compatible(model.features or []),
|
||||
"fetchFrom": _json_compatible(model.fetch_from),
|
||||
"modelProperties": _json_compatible(model.model_properties),
|
||||
"modelType": model.model_type.value,
|
||||
"status": _json_compatible(model.status),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
next_offset = payload.offset + len(page_models)
|
||||
return InvokableModelCatalogPage(
|
||||
items=items,
|
||||
next_offset=next_offset if next_offset < len(matched_models) else None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def invoke_tts(cls, user_id: str, tenant: Tenant, payload: RequestInvokeTTS):
|
||||
"""
|
||||
|
||||
@@ -6,6 +6,11 @@ from typing import Any, Literal
|
||||
from flask import Response
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from core.datasource.entities.datasource_entities import (
|
||||
DatasourceProviderType,
|
||||
GetOnlineDocumentPageContentRequest,
|
||||
)
|
||||
from core.entities.embedding_type import EmbeddingInputType
|
||||
from core.entities.provider_entities import BasicProviderConfig
|
||||
from core.plugin.utils.http_parser import deserialize_response
|
||||
from core.workflow.file_reference import is_canonical_file_reference
|
||||
@@ -54,6 +59,61 @@ class RequestInvokeTool(BaseModel):
|
||||
credential_id: str | None = None
|
||||
|
||||
|
||||
DatasourceInvocationOperation = Literal[
|
||||
"get_online_document_page_content",
|
||||
"get_online_document_pages",
|
||||
"get_website_crawl",
|
||||
"online_drive_browse_files",
|
||||
"online_drive_download_file",
|
||||
"validate_credentials",
|
||||
]
|
||||
|
||||
|
||||
class RequestInvokeDatasource(BaseModel):
|
||||
"""Invoke one installed datasource using a Dify-owned credential reference.
|
||||
|
||||
Raw credentials are intentionally not part of this contract. ``tenant_id`` and
|
||||
``user_id`` are consumed by the inner-API request context, while the remaining
|
||||
fields select an installed provider declaration and an operation-specific input.
|
||||
"""
|
||||
|
||||
tenant_id: str = Field(min_length=1, max_length=512)
|
||||
user_id: str = Field(min_length=1, max_length=512)
|
||||
provider: str = Field(min_length=1, max_length=768)
|
||||
datasource: str = Field(min_length=1, max_length=256)
|
||||
datasource_type: DatasourceProviderType
|
||||
credential_id: str = Field(min_length=1, max_length=512)
|
||||
operation: DatasourceInvocationOperation
|
||||
datasource_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
page: GetOnlineDocumentPageContentRequest | None = None
|
||||
request: dict[str, Any] | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_operation_payload(self) -> "RequestInvokeDatasource":
|
||||
expected_type = {
|
||||
"get_online_document_page_content": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_online_document_pages": DatasourceProviderType.ONLINE_DOCUMENT,
|
||||
"get_website_crawl": DatasourceProviderType.WEBSITE_CRAWL,
|
||||
"online_drive_browse_files": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"online_drive_download_file": DatasourceProviderType.ONLINE_DRIVE,
|
||||
"validate_credentials": self.datasource_type,
|
||||
}[self.operation]
|
||||
if self.datasource_type != expected_type:
|
||||
raise ValueError(f"{self.operation} requires datasource_type {expected_type.value}")
|
||||
|
||||
page_required = self.operation == "get_online_document_page_content"
|
||||
if page_required != (self.page is not None):
|
||||
raise ValueError("page is required only for get_online_document_page_content")
|
||||
|
||||
request_required = self.operation in {"online_drive_browse_files", "online_drive_download_file"}
|
||||
if request_required != (self.request is not None):
|
||||
raise ValueError("request is required only for online-drive operations")
|
||||
|
||||
return self
|
||||
|
||||
|
||||
class BaseRequestInvokeModel(BaseModel):
|
||||
provider: str
|
||||
model: str
|
||||
@@ -115,6 +175,25 @@ class RequestInvokeTextEmbedding(BaseRequestInvokeModel):
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
texts: list[str]
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class MultimodalEmbeddingDocument(BaseModel):
|
||||
"""A document accepted by a multimodal text-embedding model."""
|
||||
|
||||
content: str
|
||||
content_type: str
|
||||
file_id: str | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class RequestInvokeMultimodalEmbedding(BaseRequestInvokeModel):
|
||||
"""Request to invoke a multimodal text-embedding model."""
|
||||
|
||||
model_type: ModelType = ModelType.TEXT_EMBEDDING
|
||||
documents: list[MultimodalEmbeddingDocument] = Field(min_length=1)
|
||||
input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT
|
||||
|
||||
|
||||
class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
@@ -125,8 +204,40 @@ class RequestInvokeRerank(BaseRequestInvokeModel):
|
||||
model_type: ModelType = ModelType.RERANK
|
||||
query: str
|
||||
docs: list[str]
|
||||
score_threshold: float
|
||||
top_n: int
|
||||
score_threshold: float | None = None
|
||||
top_n: int | None = None
|
||||
|
||||
|
||||
class RequestListModels(BaseModel):
|
||||
"""Tenant-scoped query for models that Dify can invoke."""
|
||||
|
||||
model_type: Literal[ModelType.LLM, ModelType.TEXT_EMBEDDING, ModelType.RERANK]
|
||||
provider: str | None = None
|
||||
model: str | None = None
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=50, ge=1, le=100)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogItem(BaseModel):
|
||||
"""Installed identity and active Dify capability metadata for one model."""
|
||||
|
||||
plugin_id: str
|
||||
plugin_unique_identifier: str
|
||||
provider: str
|
||||
model: str
|
||||
model_type: ModelType
|
||||
capabilities: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
||||
|
||||
class InvokableModelCatalogPage(BaseModel):
|
||||
"""Offset page returned by the internal model catalog endpoint."""
|
||||
|
||||
items: list[InvokableModelCatalogItem] = Field(default_factory=list)
|
||||
next_offset: int | None = None
|
||||
|
||||
|
||||
class RequestInvokeTTS(BaseRequestInvokeModel):
|
||||
|
||||
@@ -10,6 +10,7 @@ class RBACResourceScope(StrEnum):
|
||||
|
||||
APP = "app"
|
||||
DATASET = "dataset"
|
||||
KNOWLEDGE_FS = "knowledge_space"
|
||||
WORKSPACE = "workspace"
|
||||
|
||||
|
||||
@@ -57,6 +58,15 @@ class RBACPermission(StrEnum):
|
||||
DATASET_EXTERNAL_CONNECT = "dataset_external_connect"
|
||||
DATASET_IMPORT_EXPORT_DSL = "dataset_import_export_dsl"
|
||||
|
||||
KNOWLEDGE_FS_READ = "knowledge_space_read"
|
||||
KNOWLEDGE_FS_CREATE = "knowledge_space_create"
|
||||
KNOWLEDGE_FS_EDIT = "knowledge_space_edit"
|
||||
KNOWLEDGE_FS_DELETE = "knowledge_space_delete"
|
||||
KNOWLEDGE_FS_ACCESS_CONFIG = "knowledge_space_access_config"
|
||||
KNOWLEDGE_FS_API_KEY_MANAGE = "knowledge_space_api_key_manage"
|
||||
KNOWLEDGE_FS_DOCUMENT_WRITE = "knowledge_space_document_write"
|
||||
KNOWLEDGE_FS_QUERY = "knowledge_space_query"
|
||||
|
||||
WORKSPACE_MEMBER_MANAGE = "workspace_member_manage"
|
||||
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
|
||||
API_EXTENSION_MANAGE = "api_extension_manage"
|
||||
|
||||
@@ -2,7 +2,7 @@ from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
|
||||
from core.plugin.entities.plugin_daemon import CredentialType
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom
|
||||
|
||||
@@ -20,6 +20,7 @@ class ToolRuntime(BaseModel):
|
||||
tool_id: str | None = None
|
||||
invoke_from: InvokeFrom | None = None
|
||||
tool_invoke_from: ToolInvokeFrom | None = None
|
||||
dify_run_context: DifyRunContext | None = Field(default=None, exclude=True, repr=False)
|
||||
credentials: dict[str, Any] = Field(default_factory=dict)
|
||||
credential_type: CredentialType = Field(default=CredentialType.API_KEY)
|
||||
runtime_parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
- audio
|
||||
- code
|
||||
- knowledge_fs
|
||||
- time
|
||||
- webscraper
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" fill="none">
|
||||
<rect width="64" height="64" rx="14" fill="#155EEF"/>
|
||||
<path d="M17 17h20c5.5 0 10 4.5 10 10v20H27c-5.5 0-10-4.5-10-10V17Z" fill="white" fill-opacity=".96"/>
|
||||
<path d="M26 27h12M26 34h12M26 41h7" stroke="#155EEF" stroke-width="4" stroke-linecap="round"/>
|
||||
</svg>
|
||||
|
||||
|
After Width: | Height: | Size: 340 B |
@@ -0,0 +1,9 @@
|
||||
from typing import Any, override
|
||||
|
||||
from core.tools.builtin_tool.provider import BuiltinToolProviderController
|
||||
|
||||
|
||||
class KnowledgeFSProvider(BuiltinToolProviderController):
|
||||
@override
|
||||
def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
|
||||
_ = (user_id, credentials)
|
||||
@@ -0,0 +1,13 @@
|
||||
identity:
|
||||
author: Dify
|
||||
name: knowledge_fs
|
||||
label:
|
||||
en_US: KnowledgeFS
|
||||
zh_Hans: KnowledgeFS
|
||||
description:
|
||||
en_US: Run explicitly bound KnowledgeFS operations from an Agent or Workflow.
|
||||
zh_Hans: 从 Agent 或 Workflow 执行显式绑定的 KnowledgeFS 操作。
|
||||
icon: icon.svg
|
||||
tags:
|
||||
- rag
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from collections.abc import Generator
|
||||
from typing import Any, override
|
||||
|
||||
from pydantic import ValidationError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.tools.builtin_tool.tool import BuiltinTool
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolInvokeMessage
|
||||
from core.tools.errors import ToolInvokeError
|
||||
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
|
||||
from services.knowledge_fs.app_execution_capability import KnowledgeResourceRef
|
||||
from services.knowledge_fs.product_dto import KnowledgeFSResearchTaskCreatePayload
|
||||
from services.knowledge_fs.runtime import create_knowledge_fs_runtime
|
||||
|
||||
|
||||
class KnowledgeFSCreateResearchTaskTool(BuiltinTool):
|
||||
@override
|
||||
def _invoke(
|
||||
self,
|
||||
session: Session,
|
||||
user_id: str,
|
||||
tool_parameters: dict[str, Any],
|
||||
conversation_id: str | None = None,
|
||||
app_id: str | None = None,
|
||||
message_id: str | None = None,
|
||||
) -> Generator[ToolInvokeMessage, None, None]:
|
||||
_ = (user_id, conversation_id, app_id, message_id)
|
||||
run_context = self.runtime.dify_run_context
|
||||
if run_context is None or self.runtime.tenant_id != run_context.tenant_id:
|
||||
raise ToolInvokeError("KnowledgeFS requires a trusted Dify run context")
|
||||
match self.runtime.tool_invoke_from:
|
||||
case ToolInvokeFrom.AGENT:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.AGENT
|
||||
case ToolInvokeFrom.WORKFLOW:
|
||||
caller_kind = KnowledgeFSAppSpaceJoinType.WORKFLOW
|
||||
case _:
|
||||
raise ToolInvokeError("KnowledgeFS is only available to Agent and Workflow callers")
|
||||
|
||||
try:
|
||||
resource = KnowledgeResourceRef.model_validate(tool_parameters.get("resource"))
|
||||
payload_data: dict[str, object] = {
|
||||
"query": tool_parameters.get("query"),
|
||||
}
|
||||
mode = tool_parameters.get("mode")
|
||||
if mode:
|
||||
payload_data["mode"] = mode
|
||||
payload = KnowledgeFSResearchTaskCreatePayload.model_validate(payload_data)
|
||||
runtime = create_knowledge_fs_runtime(sessionmaker(bind=session.get_bind(), expire_on_commit=False))
|
||||
response = runtime.app_capabilities.create_research_task(
|
||||
run_context=run_context,
|
||||
caller_kind=caller_kind,
|
||||
resource=resource,
|
||||
payload=payload,
|
||||
)
|
||||
except ToolInvokeError:
|
||||
raise
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
raise ToolInvokeError(str(exc)) from exc
|
||||
|
||||
yield self.create_json_message(response.model_dump(mode="json", by_alias=True))
|
||||
@@ -0,0 +1,75 @@
|
||||
identity:
|
||||
name: create_research_task
|
||||
author: Dify
|
||||
label:
|
||||
en_US: Create KnowledgeFS Research Task
|
||||
zh_Hans: 创建 KnowledgeFS Research 任务
|
||||
description:
|
||||
human:
|
||||
en_US: Create a Research task in an explicitly bound KnowledgeFS space.
|
||||
zh_Hans: 在显式绑定的 KnowledgeFS 空间中创建 Research 任务。
|
||||
llm: Create a durable research task using an explicitly configured KnowledgeFS resource.
|
||||
parameters:
|
||||
- name: resource
|
||||
type: object
|
||||
required: true
|
||||
label:
|
||||
en_US: KnowledgeFS resource
|
||||
zh_Hans: KnowledgeFS 资源
|
||||
human_description:
|
||||
en_US: A typed KnowledgeFS control-space reference configured by the app author.
|
||||
zh_Hans: 由应用作者配置的类型化 KnowledgeFS control-space 引用。
|
||||
form: form
|
||||
input_schema:
|
||||
type: object
|
||||
additionalProperties: false
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
const: knowledge_fs
|
||||
control_space_id:
|
||||
type: string
|
||||
minLength: 1
|
||||
required:
|
||||
- kind
|
||||
- control_space_id
|
||||
- name: query
|
||||
type: string
|
||||
required: true
|
||||
label:
|
||||
en_US: Research query
|
||||
zh_Hans: Research 查询
|
||||
human_description:
|
||||
en_US: The question the Research task should investigate.
|
||||
zh_Hans: Research 任务需要调查的问题。
|
||||
llm_description: The question to investigate with KnowledgeFS.
|
||||
form: llm
|
||||
- name: mode
|
||||
type: select
|
||||
required: false
|
||||
label:
|
||||
en_US: Mode
|
||||
zh_Hans: 模式
|
||||
human_description:
|
||||
en_US: Optional KnowledgeFS retrieval mode.
|
||||
zh_Hans: 可选的 KnowledgeFS 检索模式。
|
||||
llm_description: Optional retrieval mode. Use auto unless the task needs a specific mode.
|
||||
form: llm
|
||||
options:
|
||||
- value: auto
|
||||
label:
|
||||
en_US: Auto
|
||||
zh_Hans: 自动
|
||||
- value: fast
|
||||
label:
|
||||
en_US: Fast
|
||||
zh_Hans: 快速
|
||||
- value: deep
|
||||
label:
|
||||
en_US: Deep
|
||||
zh_Hans: 深度
|
||||
- value: research
|
||||
label:
|
||||
en_US: Research
|
||||
zh_Hans: Research
|
||||
|
||||
@@ -493,6 +493,7 @@ class DifyToolNodeRuntime(ToolNodeRuntimeProtocol):
|
||||
self._run_context.invoke_from,
|
||||
variable_pool,
|
||||
)
|
||||
tool_runtime.runtime.dify_run_context = self._run_context
|
||||
except ToolNodeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
|
||||
@@ -250,6 +250,7 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
agent_config_version_kind="snapshot",
|
||||
agent_mode=self._agent_backend_agent_mode(context.dify_context.invoke_from),
|
||||
invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
|
||||
trace_id=context.dify_context.trace_session_id,
|
||||
),
|
||||
agent_soul_prompt=soul_prompt or None,
|
||||
workflow_node_job_prompt=workflow_job_prompt,
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
"""Enforce focused KnowledgeFS coverage without hiding integration-critical glue.
|
||||
|
||||
The primary threshold aggregates statement and branch coverage for every Dify
|
||||
module owned by the KnowledgeFS integration. Large pre-existing Dify modules
|
||||
that only contain narrow integration hooks are checked with changed-line
|
||||
coverage instead, so unrelated legacy code cannot dilute or inflate the gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TypedDict, cast
|
||||
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parents[2]
|
||||
NON_CORE_COVERAGE_ALLOWLIST = frozenset(
|
||||
{
|
||||
"api/dev/check_knowledge_fs_coverage.py",
|
||||
"api/dev/generate_knowledge_fs_contract.py",
|
||||
"api/dev/knowledge_fs_product_contract.py",
|
||||
"api/migrations/versions/2026_07_21_1200-a4e7c2f91b30_add_knowledge_fs_control_plane.py",
|
||||
"api/migrations/versions/2026_07_21_1300-b7f2a9d41c60_add_knowledge_fs_cutover.py",
|
||||
"api/migrations/versions/2026_07_21_1400-c8e31b7d52a4_add_knowledge_fs_cleanup_authorization.py",
|
||||
}
|
||||
)
|
||||
HUNK_HEADER = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CoverageSummary(TypedDict):
|
||||
"""Coverage.py counts required by the aggregate gate."""
|
||||
|
||||
covered_lines: int
|
||||
num_statements: int
|
||||
covered_branches: int
|
||||
num_branches: int
|
||||
|
||||
|
||||
class CoverageFile(TypedDict):
|
||||
"""Per-file coverage data emitted by ``coverage json``."""
|
||||
|
||||
executed_lines: list[int]
|
||||
missing_lines: list[int]
|
||||
summary: CoverageSummary
|
||||
|
||||
|
||||
class CoverageReport(TypedDict):
|
||||
"""Relevant top-level shape of a coverage.py JSON report."""
|
||||
|
||||
files: dict[str, CoverageFile]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CoverageTotals:
|
||||
"""Covered and measurable units for one gate surface."""
|
||||
|
||||
covered: int
|
||||
total: int
|
||||
|
||||
@property
|
||||
def percent(self) -> float:
|
||||
return 100.0 if self.total == 0 else self.covered * 100 / self.total
|
||||
|
||||
|
||||
class CoverageGateError(RuntimeError):
|
||||
"""Raised when coverage input is incomplete or below its threshold."""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Validate focused module coverage and changed integration glue."""
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--coverage-json", type=Path, required=True)
|
||||
parser.add_argument("--glue-manifest", type=Path, required=True)
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
parser.add_argument("--base", default="")
|
||||
parser.add_argument("--minimum", type=float, default=90.0)
|
||||
parser.add_argument("--glue-minimum", type=float, default=90.0)
|
||||
args = parser.parse_args()
|
||||
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
report = load_coverage_report(args.coverage_json)
|
||||
core_totals = validate_core_coverage(report, workspace_root=workspace_root, minimum=args.minimum)
|
||||
base = resolve_diff_base(workspace_root, args.base)
|
||||
glue_paths = load_glue_coverage_paths(args.glue_manifest, workspace_root=workspace_root)
|
||||
changed_lines = collect_changed_glue_lines(workspace_root, base, glue_paths=glue_paths)
|
||||
glue_totals = validate_changed_glue_coverage(
|
||||
report,
|
||||
changed_lines=changed_lines,
|
||||
minimum=args.glue_minimum,
|
||||
)
|
||||
logger.info(
|
||||
"Dify KnowledgeFS coverage passed: core lines+branches %.2f%% (%d/%d); changed glue lines %.2f%% (%d/%d)",
|
||||
core_totals.percent,
|
||||
core_totals.covered,
|
||||
core_totals.total,
|
||||
glue_totals.percent,
|
||||
glue_totals.covered,
|
||||
glue_totals.total,
|
||||
)
|
||||
|
||||
|
||||
def load_coverage_report(path: Path) -> CoverageReport:
|
||||
"""Load the detailed JSON report used by both coverage checks."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"coverage JSON does not exist: {path}")
|
||||
document = json.loads(path.read_text())
|
||||
if not isinstance(document, dict) or not isinstance(document.get("files"), dict):
|
||||
raise CoverageGateError(f"coverage JSON has no files object: {path}")
|
||||
return cast(CoverageReport, document)
|
||||
|
||||
|
||||
def load_glue_coverage_paths(path: Path, *, workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Load the workflow's authoritative NUL-delimited integration touchpoints."""
|
||||
if not path.is_file():
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest does not exist: {path}")
|
||||
try:
|
||||
paths = tuple(item.decode() for item in path.read_bytes().split(b"\0") if item)
|
||||
except UnicodeDecodeError as error:
|
||||
raise CoverageGateError(f"KnowledgeFS glue manifest is not UTF-8: {path}") from error
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage target set is empty")
|
||||
if len(paths) != len(set(paths)):
|
||||
raise CoverageGateError("Dify KnowledgeFS glue coverage manifest contains duplicate paths")
|
||||
invalid_paths = [
|
||||
candidate
|
||||
for candidate in paths
|
||||
if not candidate.startswith("api/")
|
||||
or not candidate.endswith(".py")
|
||||
or not (workspace_root / candidate).is_file()
|
||||
]
|
||||
if invalid_paths:
|
||||
raise CoverageGateError(f"Dify KnowledgeFS glue coverage paths are invalid: {', '.join(invalid_paths)}")
|
||||
return paths
|
||||
|
||||
|
||||
def is_core_coverage_path(path: str) -> bool:
|
||||
"""Return whether a repository-relative path belongs to the focused aggregate."""
|
||||
if not path.endswith(".py"):
|
||||
return False
|
||||
if path in {
|
||||
"api/commands/knowledge_fs.py",
|
||||
"api/configs/extra/knowledge_fs_config.py",
|
||||
"api/extensions/ext_knowledge_fs_observability.py",
|
||||
"api/services/knowledge_fs_capability.py",
|
||||
}:
|
||||
return True
|
||||
if path.startswith(
|
||||
(
|
||||
"api/controllers/console/knowledge_fs/",
|
||||
"api/controllers/service_api/knowledge_fs/",
|
||||
"api/core/tools/builtin_tool/providers/knowledge_fs/",
|
||||
"api/services/knowledge_fs/",
|
||||
)
|
||||
):
|
||||
return True
|
||||
filename = path.rsplit("/", maxsplit=1)[-1]
|
||||
return (
|
||||
path.startswith("api/models/")
|
||||
and filename.startswith("knowledge_fs")
|
||||
or path.startswith("api/repositories/")
|
||||
and "knowledge_fs" in filename
|
||||
or path.startswith("api/tasks/")
|
||||
and "knowledge_fs" in filename
|
||||
)
|
||||
|
||||
|
||||
def discover_core_coverage_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Classify every KnowledgeFS-named production file or fail closed."""
|
||||
named_paths = discover_knowledge_fs_production_paths(workspace_root)
|
||||
core_paths = {path for path in named_paths if is_core_coverage_path(path)}
|
||||
unclassified_paths = set(named_paths) - core_paths - NON_CORE_COVERAGE_ALLOWLIST
|
||||
if unclassified_paths:
|
||||
raise CoverageGateError(
|
||||
"unclassified Dify KnowledgeFS production files must join the core coverage scope or explicit allowlist: "
|
||||
+ ", ".join(sorted(unclassified_paths))
|
||||
)
|
||||
if not core_paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage target set is empty")
|
||||
return tuple(sorted(core_paths))
|
||||
|
||||
|
||||
def discover_knowledge_fs_production_paths(workspace_root: Path) -> tuple[str, ...]:
|
||||
"""Mirror the workflow's dynamic KnowledgeFS filename discovery."""
|
||||
api_root = workspace_root / "api"
|
||||
if not api_root.is_dir():
|
||||
raise CoverageGateError(f"Dify API directory does not exist: {api_root}")
|
||||
paths: set[str] = set()
|
||||
for directory, child_directories, filenames in os.walk(api_root):
|
||||
current_directory = Path(directory)
|
||||
if current_directory == api_root:
|
||||
child_directories[:] = [name for name in child_directories if name not in {".venv", "storage", "tests"}]
|
||||
child_directories[:] = [name for name in child_directories if name != "__pycache__"]
|
||||
for filename in filenames:
|
||||
path = (current_directory / filename).relative_to(workspace_root).as_posix()
|
||||
if filename.endswith(".py") and "knowledge_fs" in path:
|
||||
paths.add(path)
|
||||
if not paths:
|
||||
raise CoverageGateError("Dify KnowledgeFS production target set is empty")
|
||||
return tuple(sorted(paths))
|
||||
|
||||
|
||||
def validate_core_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
workspace_root: Path,
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require the exact combined line-and-branch percentage for all core files."""
|
||||
paths = discover_core_coverage_paths(workspace_root)
|
||||
missing_paths = [path for path in paths if path not in report["files"]]
|
||||
if missing_paths:
|
||||
raise CoverageGateError(f"coverage report is missing core files: {', '.join(missing_paths)}")
|
||||
|
||||
covered = 0
|
||||
total = 0
|
||||
for path in paths:
|
||||
summary = report["files"][path]["summary"]
|
||||
covered += summary["covered_lines"] + summary["covered_branches"]
|
||||
total += summary["num_statements"] + summary["num_branches"]
|
||||
if total == 0:
|
||||
raise CoverageGateError("Dify KnowledgeFS core coverage has no measurable statements or branches")
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS core line-and-branch coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def resolve_diff_base(workspace_root: Path, preferred: str) -> str:
|
||||
"""Resolve an explicit event base, falling back to the previous commit for manual runs."""
|
||||
base = preferred.strip()
|
||||
if not base or set(base) == {"0"}:
|
||||
base = "HEAD^"
|
||||
result = subprocess.run(
|
||||
["git", "cat-file", "-e", f"{base}^{{commit}}"],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "commit is unavailable"
|
||||
raise CoverageGateError(f"cannot resolve coverage diff base {base}: {detail}")
|
||||
return base
|
||||
|
||||
|
||||
def collect_changed_glue_lines(
|
||||
workspace_root: Path,
|
||||
base: str,
|
||||
*,
|
||||
glue_paths: tuple[str, ...],
|
||||
) -> dict[str, set[int]]:
|
||||
"""Return added line numbers in the narrow Dify modules touched by this integration."""
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-color",
|
||||
"--unified=0",
|
||||
base,
|
||||
"--",
|
||||
*glue_paths,
|
||||
],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
detail = result.stderr.strip() or "git diff failed"
|
||||
raise CoverageGateError(f"cannot collect KnowledgeFS glue diff from {base}: {detail}")
|
||||
return parse_added_lines(result.stdout)
|
||||
|
||||
|
||||
def parse_added_lines(diff: str) -> dict[str, set[int]]:
|
||||
"""Parse repository paths and added-side line numbers from a zero-context Git diff."""
|
||||
changed_lines: dict[str, set[int]] = {}
|
||||
current_path: str | None = None
|
||||
current_line: int | None = None
|
||||
for raw_line in diff.splitlines():
|
||||
if raw_line.startswith("diff --git "):
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("+++ "):
|
||||
candidate = raw_line[4:]
|
||||
current_path = candidate[2:] if candidate.startswith("b/") else None
|
||||
if current_path is not None:
|
||||
changed_lines.setdefault(current_path, set())
|
||||
current_line = None
|
||||
continue
|
||||
if raw_line.startswith("@@ "):
|
||||
match = HUNK_HEADER.match(raw_line)
|
||||
current_line = int(match.group(1)) if match is not None else None
|
||||
continue
|
||||
if current_path is None or current_line is None:
|
||||
continue
|
||||
if raw_line.startswith("+"):
|
||||
changed_lines[current_path].add(current_line)
|
||||
current_line += 1
|
||||
elif raw_line.startswith("-") or raw_line.startswith("\\"):
|
||||
continue
|
||||
else:
|
||||
current_line += 1
|
||||
return changed_lines
|
||||
|
||||
|
||||
def validate_changed_glue_coverage(
|
||||
report: CoverageReport,
|
||||
*,
|
||||
changed_lines: dict[str, set[int]],
|
||||
minimum: float,
|
||||
) -> CoverageTotals:
|
||||
"""Require added executable glue lines to be exercised by the focused unit suite."""
|
||||
covered = 0
|
||||
total = 0
|
||||
for path, lines in sorted(changed_lines.items()):
|
||||
if not lines:
|
||||
continue
|
||||
file_coverage = report["files"].get(path)
|
||||
if file_coverage is None:
|
||||
raise CoverageGateError(f"coverage report is missing changed glue file: {path}")
|
||||
executed_lines = set(file_coverage["executed_lines"])
|
||||
executable_lines = executed_lines | set(file_coverage["missing_lines"])
|
||||
changed_executable_lines = lines & executable_lines
|
||||
covered += len(changed_executable_lines & executed_lines)
|
||||
total += len(changed_executable_lines)
|
||||
|
||||
totals = CoverageTotals(covered=covered, total=total)
|
||||
_require_minimum(totals, minimum=minimum, label="Dify KnowledgeFS changed-glue line coverage")
|
||||
return totals
|
||||
|
||||
|
||||
def _require_minimum(totals: CoverageTotals, *, minimum: float, label: str) -> None:
|
||||
if totals.percent + 1e-12 < minimum:
|
||||
raise CoverageGateError(f"{label} is {totals.percent:.2f}%; minimum {minimum:.2f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,7 +1,14 @@
|
||||
"""Validate Dify Console KnowledgeFS declarations against a pinned OpenAPI document.
|
||||
"""Pin the in-repository KnowledgeFS contract and validate every Dify product operation.
|
||||
|
||||
The OpenAPI document is exported only during explicit development validation. Runtime declarations live with Dify
|
||||
product policy; this module validates their transport metadata without generating a complete operation catalog.
|
||||
The lock is intentionally independent of the enclosing Dify commit: it records the staged ``knowledge-fs/`` tree,
|
||||
the complete generated OpenAPI document, both explicit product-operation manifests, and the active Capability v2
|
||||
profile and deterministic public-key vector. The full OpenAPI hash
|
||||
covers request/response schemas, status codes, security, deprecation, and stream metadata. Field-level validation
|
||||
cross-checks the Dify product registry, Python Capability issuer, TypeScript request guard, and exported OpenAPI;
|
||||
each product operation must be ready or an explicit gap, and KFS-only activation remains explicitly internal.
|
||||
|
||||
Contract export reads the working tree only after proving it matches the staged KnowledgeFS index. This keeps the
|
||||
OpenAPI bytes and auth manifest aligned with the exact subtree tree ID that will be reviewed and committed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -13,17 +20,36 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict
|
||||
from typing import Any, Literal, TypedDict, cast
|
||||
|
||||
import jwt
|
||||
from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey
|
||||
from jwt.algorithms import RSAAlgorithm
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
from dev.knowledge_fs_product_contract import (
|
||||
capability_operation_runtime_contracts,
|
||||
parse_capability_operation_policy,
|
||||
parse_product_operation_gap_manifest,
|
||||
parse_product_operation_manifest,
|
||||
product_operation_runtime_contracts,
|
||||
validate_product_operation_contracts,
|
||||
)
|
||||
|
||||
WORKSPACE_ROOT = API_ROOT.parent
|
||||
LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
|
||||
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
|
||||
KNOWLEDGE_FS_DIRECTORY = "knowledge-fs"
|
||||
CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH = Path("contracts/dify-capability-v2-auth-profile.json")
|
||||
CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH = Path("contracts/dify-capability-v2-test-vector.json")
|
||||
UPSTREAM_PROVENANCE_RELATIVE_PATH = Path("upstream-provenance.json")
|
||||
LOCK_RELATIVE_PATH = Path("api/knowledge-fs-contract.lock.json")
|
||||
PRODUCT_OPERATIONS_RELATIVE_PATH = Path("api/knowledge-fs-product-operations.json")
|
||||
PRODUCT_OPERATION_GAPS_RELATIVE_PATH = Path("api/knowledge-fs-product-operation-gaps.json")
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
LOCK_SCHEMA_VERSION = 5
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
@@ -40,6 +66,18 @@ class ContractDeclaration(TypedDict):
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
class ContractLock(TypedDict):
|
||||
"""Content-addressed contract inputs that must move together."""
|
||||
|
||||
schemaVersion: int
|
||||
subtreeTree: str
|
||||
openapiSha256: str
|
||||
capabilityV2AuthManifestSha256: str
|
||||
capabilityV2AuthTestVectorSha256: str
|
||||
productOperationManifestSha256: str
|
||||
productOperationGapManifestSha256: str
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
"method",
|
||||
"path",
|
||||
@@ -64,59 +102,507 @@ DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Update or verify the pin and validate Console declarations against its OpenAPI document."""
|
||||
"""Update or verify the monorepo pin and validate Dify product declarations."""
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--update-lock", action="store_true")
|
||||
parser.add_argument("--repository", type=Path, default=DEFAULT_REPOSITORY)
|
||||
parser.add_argument("--workspace-root", type=Path, default=WORKSPACE_ROOT)
|
||||
args = parser.parse_args()
|
||||
|
||||
repository = args.repository.resolve()
|
||||
lock = json.loads(LOCK_PATH.read_text())
|
||||
tracked_changes = run("git", "status", "--porcelain", "--untracked-files=no", cwd=repository).strip()
|
||||
if tracked_changes:
|
||||
raise RuntimeError("KnowledgeFS checkout must not contain tracked changes during contract export")
|
||||
workspace_root = args.workspace_root.resolve()
|
||||
knowledge_fs_root = workspace_root / KNOWLEDGE_FS_DIRECTORY
|
||||
lock_path = workspace_root / LOCK_RELATIVE_PATH
|
||||
capability_v2_auth_manifest_path = knowledge_fs_root / CAPABILITY_V2_AUTH_MANIFEST_RELATIVE_PATH
|
||||
capability_v2_auth_test_vector_path = knowledge_fs_root / CAPABILITY_V2_AUTH_TEST_VECTOR_RELATIVE_PATH
|
||||
product_operations_path = workspace_root / PRODUCT_OPERATIONS_RELATIVE_PATH
|
||||
product_operation_gaps_path = workspace_root / PRODUCT_OPERATION_GAPS_RELATIVE_PATH
|
||||
upstream_provenance_path = knowledge_fs_root / UPSTREAM_PROVENANCE_RELATIVE_PATH
|
||||
ensure_clean_knowledge_fs_worktree(workspace_root)
|
||||
ensure_contract_inputs_exist(
|
||||
knowledge_fs_root=knowledge_fs_root,
|
||||
capability_v2_auth_manifest_path=capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path=capability_v2_auth_test_vector_path,
|
||||
product_operations_path=product_operations_path,
|
||||
product_operation_gaps_path=product_operation_gaps_path,
|
||||
upstream_provenance_path=upstream_provenance_path,
|
||||
)
|
||||
|
||||
commit = run("git", "rev-parse", "HEAD", cwd=repository).strip()
|
||||
if not args.update_lock and commit != lock["commit"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS checkout mismatch: expected {lock['commit']}, received {commit}. "
|
||||
"Use the pinned commit or pass --update-lock intentionally."
|
||||
)
|
||||
subtree_tree = staged_subtree_tree(workspace_root)
|
||||
capability_v2_auth_manifest_content = capability_v2_auth_manifest_path.read_bytes()
|
||||
capability_v2_auth_test_vector_content = capability_v2_auth_test_vector_path.read_bytes()
|
||||
product_operation_manifest_content = product_operations_path.read_bytes()
|
||||
product_operation_gap_manifest_content = product_operation_gaps_path.read_bytes()
|
||||
capability_v2_auth_manifest = load_json_object(capability_v2_auth_manifest_path)
|
||||
validate_capability_v2_auth_manifest(capability_v2_auth_manifest)
|
||||
validate_capability_v2_auth_test_vector(
|
||||
load_json_object(capability_v2_auth_test_vector_path),
|
||||
capability_v2_auth_manifest,
|
||||
)
|
||||
validate_upstream_provenance(load_json_object(upstream_provenance_path))
|
||||
product_manifest = parse_product_operation_manifest(load_json_object(product_operations_path))
|
||||
product_gap_manifest = parse_product_operation_gap_manifest(load_json_object(product_operation_gaps_path))
|
||||
product_runtime_operations = product_operation_runtime_contracts()
|
||||
capability_runtime_operations = capability_operation_runtime_contracts()
|
||||
declarations = console_contract_declarations()
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
|
||||
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
|
||||
capability_policy_path = Path(directory) / "dify-capability-v2-operations.json"
|
||||
subprocess.run(
|
||||
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
|
||||
cwd=repository,
|
||||
cwd=knowledge_fs_root,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["pnpm", "capability:export", "--", "--output", str(capability_policy_path)],
|
||||
cwd=knowledge_fs_root,
|
||||
check=True,
|
||||
)
|
||||
openapi_content = openapi_path.read_bytes()
|
||||
|
||||
openapi_sha256 = sha256(openapi_content)
|
||||
if not args.update_lock and openapi_sha256 != lock["openapiSha256"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS OpenAPI hash mismatch: expected {lock['openapiSha256']}, received {openapi_sha256}"
|
||||
)
|
||||
capability_policy = parse_capability_operation_policy(load_json_object(capability_policy_path))
|
||||
|
||||
document: dict[str, Any] = json.loads(openapi_content)
|
||||
validate_declarations(document, console_contract_declarations())
|
||||
validate_product_operation_contracts(
|
||||
capability_operations=capability_runtime_operations,
|
||||
capability_policy=capability_policy,
|
||||
document=document,
|
||||
gap_manifest=product_gap_manifest,
|
||||
manifest=product_manifest,
|
||||
product_operations=product_runtime_operations,
|
||||
)
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
expected_lock: ContractLock = {
|
||||
"schemaVersion": LOCK_SCHEMA_VERSION,
|
||||
"subtreeTree": subtree_tree,
|
||||
"openapiSha256": sha256(openapi_content),
|
||||
"capabilityV2AuthManifestSha256": sha256(capability_v2_auth_manifest_content),
|
||||
"capabilityV2AuthTestVectorSha256": sha256(capability_v2_auth_test_vector_content),
|
||||
"productOperationManifestSha256": sha256(product_operation_manifest_content),
|
||||
"productOperationGapManifestSha256": sha256(product_operation_gap_manifest_content),
|
||||
}
|
||||
|
||||
if args.update_lock:
|
||||
LOCK_PATH.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"commit": commit,
|
||||
"openapiSha256": openapi_sha256,
|
||||
"repository": lock["repository"],
|
||||
},
|
||||
indent=2,
|
||||
lock_path.write_text(json.dumps(expected_lock, indent=2) + "\n")
|
||||
return
|
||||
|
||||
received_lock = parse_contract_lock(load_json_object(lock_path))
|
||||
lock_fields = (
|
||||
(
|
||||
"capabilityV2AuthManifestSha256",
|
||||
received_lock["capabilityV2AuthManifestSha256"],
|
||||
expected_lock["capabilityV2AuthManifestSha256"],
|
||||
),
|
||||
(
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
received_lock["capabilityV2AuthTestVectorSha256"],
|
||||
expected_lock["capabilityV2AuthTestVectorSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationManifestSha256",
|
||||
received_lock["productOperationManifestSha256"],
|
||||
expected_lock["productOperationManifestSha256"],
|
||||
),
|
||||
(
|
||||
"productOperationGapManifestSha256",
|
||||
received_lock["productOperationGapManifestSha256"],
|
||||
expected_lock["productOperationGapManifestSha256"],
|
||||
),
|
||||
("schemaVersion", received_lock["schemaVersion"], expected_lock["schemaVersion"]),
|
||||
("subtreeTree", received_lock["subtreeTree"], expected_lock["subtreeTree"]),
|
||||
("openapiSha256", received_lock["openapiSha256"], expected_lock["openapiSha256"]),
|
||||
)
|
||||
for field, received_value, expected_value in lock_fields:
|
||||
if received_value != expected_value:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS contract lock field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}. "
|
||||
"Run --update-lock intentionally after reviewing the staged subtree and contract changes."
|
||||
)
|
||||
+ "\n"
|
||||
|
||||
|
||||
def ensure_contract_inputs_exist(
|
||||
*,
|
||||
knowledge_fs_root: Path,
|
||||
capability_v2_auth_manifest_path: Path,
|
||||
capability_v2_auth_test_vector_path: Path,
|
||||
product_operations_path: Path,
|
||||
product_operation_gaps_path: Path,
|
||||
upstream_provenance_path: Path,
|
||||
) -> None:
|
||||
"""Fail with a stable error before invoking package tooling when a contract input is absent."""
|
||||
required_paths = (
|
||||
knowledge_fs_root / "package.json",
|
||||
capability_v2_auth_manifest_path,
|
||||
capability_v2_auth_test_vector_path,
|
||||
product_operations_path,
|
||||
product_operation_gaps_path,
|
||||
upstream_provenance_path,
|
||||
)
|
||||
missing_paths = [path for path in required_paths if not path.is_file()]
|
||||
if missing_paths:
|
||||
missing = ", ".join(str(path) for path in missing_paths)
|
||||
raise RuntimeError(f"KnowledgeFS contract input is missing: {missing}")
|
||||
|
||||
|
||||
def ensure_clean_knowledge_fs_worktree(workspace_root: Path) -> None:
|
||||
"""Require exported KnowledgeFS files to exactly match the staged index tree.
|
||||
|
||||
Staged changes are expected during intentional lock updates. Unstaged tracked changes and untracked files are
|
||||
rejected because the export process reads the working tree while the tree ID is calculated from the index.
|
||||
"""
|
||||
subtree_path = f"{KNOWLEDGE_FS_DIRECTORY}/"
|
||||
unstaged = subprocess.run(
|
||||
["git", "diff", "--quiet", "--", subtree_path],
|
||||
cwd=workspace_root,
|
||||
check=False,
|
||||
)
|
||||
if unstaged.returncode > 1:
|
||||
raise RuntimeError("git diff failed while validating the staged KnowledgeFS subtree")
|
||||
untracked = run(
|
||||
"git",
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"--",
|
||||
subtree_path,
|
||||
cwd=workspace_root,
|
||||
).strip()
|
||||
if unstaged.returncode != 0 or untracked:
|
||||
raise RuntimeError(
|
||||
"knowledge-fs/ contains unstaged or untracked changes; stage or remove them before contract export"
|
||||
)
|
||||
|
||||
|
||||
def staged_subtree_tree(workspace_root: Path) -> str:
|
||||
"""Return the Git tree object for the staged ``knowledge-fs/`` subtree."""
|
||||
return run("git", "write-tree", f"--prefix={KNOWLEDGE_FS_DIRECTORY}/", cwd=workspace_root).strip()
|
||||
|
||||
|
||||
def load_json_object(path: Path) -> dict[str, Any]:
|
||||
"""Load a JSON object and reject arrays/scalars at contract boundaries."""
|
||||
value = json.loads(path.read_text())
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f"KnowledgeFS contract file must contain a JSON object: {path}")
|
||||
return cast(dict[str, Any], value)
|
||||
|
||||
|
||||
def parse_contract_lock(value: dict[str, Any]) -> ContractLock:
|
||||
"""Validate the compact, non-self-referential contract lock schema."""
|
||||
expected_fields = {
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"schemaVersion",
|
||||
"subtreeTree",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS contract lock fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != LOCK_SCHEMA_VERSION:
|
||||
raise ValueError(f"KnowledgeFS contract lock schemaVersion must be {LOCK_SCHEMA_VERSION}")
|
||||
for field in (
|
||||
"capabilityV2AuthManifestSha256",
|
||||
"capabilityV2AuthTestVectorSha256",
|
||||
"openapiSha256",
|
||||
"productOperationGapManifestSha256",
|
||||
"productOperationManifestSha256",
|
||||
"subtreeTree",
|
||||
):
|
||||
field_value = value.get(field)
|
||||
expected_length = 40 if field == "subtreeTree" else 64
|
||||
if (
|
||||
not isinstance(field_value, str)
|
||||
or len(field_value) != expected_length
|
||||
or any(character not in "0123456789abcdef" for character in field_value)
|
||||
):
|
||||
raise ValueError(f"KnowledgeFS contract lock field {field} has an invalid digest")
|
||||
return cast(ContractLock, value)
|
||||
|
||||
|
||||
def validate_required_product_operations(document: dict[str, Any], required_operation_ids: list[str]) -> None:
|
||||
"""Fail when the pinned KFS OpenAPI omits an operation required by the Dify product."""
|
||||
available_operation_ids = [
|
||||
operation_id
|
||||
for path_item in document.get("paths", {}).values()
|
||||
for method in OPENAPI_METHODS
|
||||
if isinstance(path_item, dict)
|
||||
for operation in (path_item.get(method),)
|
||||
if isinstance(operation, dict)
|
||||
for operation_id in (operation.get("operationId"),)
|
||||
if isinstance(operation_id, str) and operation_id
|
||||
]
|
||||
for operation_id in required_operation_ids:
|
||||
count = available_operation_ids.count(operation_id)
|
||||
if count != 1:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS OpenAPI required product operation {operation_id} must occur exactly once; found {count}"
|
||||
)
|
||||
|
||||
|
||||
def validate_capability_v2_auth_manifest(value: dict[str, Any]) -> None:
|
||||
"""Validate the active production RS256 profile consumed by both Dify and KnowledgeFS."""
|
||||
expected_fields = {
|
||||
"active",
|
||||
"audience",
|
||||
"callerProfiles",
|
||||
"claimBindings",
|
||||
"issuer",
|
||||
"lifecycle",
|
||||
"maxTtlSeconds",
|
||||
"productionReady",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"requiredClaims",
|
||||
"resourceContract",
|
||||
"runtimeAssembly",
|
||||
"schemaVersion",
|
||||
"signatureAlgorithms",
|
||||
"tokenKind",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest fields must be exactly {sorted(expected_fields)}")
|
||||
fixed_values = {
|
||||
"active": True,
|
||||
"audience": "knowledge-fs",
|
||||
"issuer": "dify-control-plane",
|
||||
"lifecycle": "active",
|
||||
"maxTtlSeconds": 60,
|
||||
"productionReady": True,
|
||||
"profileId": "dify-capability-v2",
|
||||
"schemaVersion": 3,
|
||||
"signatureAlgorithms": ["RS256"],
|
||||
"tokenKind": "jwt",
|
||||
}
|
||||
for field, expected in fixed_values.items():
|
||||
if value.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 auth manifest field {field} must be {expected!r}")
|
||||
if value.get("protectedHeader") != {
|
||||
"algorithm": "RS256",
|
||||
"keyIdClaim": "kid",
|
||||
"keyIdRequired": True,
|
||||
"type": "JWT",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 protected header contract is invalid")
|
||||
required_claims = [
|
||||
"action",
|
||||
"actor",
|
||||
"aud",
|
||||
"authz_revision",
|
||||
"azp",
|
||||
"caller_kind",
|
||||
"cap_ver",
|
||||
"content_policy_revision",
|
||||
"content_scope_ids",
|
||||
"control_space_id",
|
||||
"exp",
|
||||
"grant_id",
|
||||
"iat",
|
||||
"iss",
|
||||
"jti",
|
||||
"namespace_id",
|
||||
"nbf",
|
||||
"resource",
|
||||
"sub",
|
||||
"trace_id",
|
||||
]
|
||||
if value.get("requiredClaims") != required_claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 required claims are invalid")
|
||||
if value.get("claimBindings") != {
|
||||
"action": "action",
|
||||
"callerKind": "caller_kind",
|
||||
"controlSpace": "control_space_id",
|
||||
"namespace": "namespace_id",
|
||||
"resource": "resource",
|
||||
"resourceParent": "resource.parent_id",
|
||||
"subject": "sub",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 claim bindings are invalid")
|
||||
if value.get("resourceContract") != {
|
||||
"fields": ["id", "parent_id", "type"],
|
||||
"parentForbiddenFor": ["namespace", "knowledge_space"],
|
||||
"parentRequiredFor": ["document", "job", "query", "research_task", "source", "upload_session"],
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 resource contract is invalid")
|
||||
if value.get("callerProfiles") != {
|
||||
"agent": {"authorizedParty": "dify-agent", "subjectPrefix": "dify-app:"},
|
||||
"interactive": {"authorizedParty": "dify-console", "subjectPrefix": "dify-account:"},
|
||||
"internal_worker": {"authorizedParty": "dify-worker", "subjectPrefix": "dify-worker:"},
|
||||
"mcp": {"authorizedParty": "dify-mcp", "subjectPrefix": "dify-mcp-session:"},
|
||||
"service": {"authorizedParty": "dify-service-api", "subjectPrefix": "dify-kfs-credential:"},
|
||||
"workflow": {"authorizedParty": "dify-workflow", "subjectPrefix": "dify-app:"},
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 caller profiles are invalid")
|
||||
if value.get("runtimeAssembly") != {
|
||||
"failClosed": True,
|
||||
"keySelection": "kid",
|
||||
"maximumPublishedKeys": 3,
|
||||
"verificationKeySource": "jwks",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS Capability v2 runtime assembly is invalid")
|
||||
|
||||
|
||||
def validate_capability_v2_auth_test_vector(
|
||||
value: dict[str, Any],
|
||||
manifest: dict[str, Any],
|
||||
) -> None:
|
||||
"""Verify the deterministic public-key vector and every security-sensitive binding."""
|
||||
expected_fields = {
|
||||
"algorithm",
|
||||
"audience",
|
||||
"expectedClaims",
|
||||
"expectedPrincipal",
|
||||
"issuer",
|
||||
"operation",
|
||||
"profileId",
|
||||
"protectedHeader",
|
||||
"publicJwk",
|
||||
"schemaVersion",
|
||||
"testOnly",
|
||||
"token",
|
||||
"ttlSeconds",
|
||||
}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector fields must be exactly {sorted(expected_fields)}")
|
||||
if (
|
||||
value.get("schemaVersion") != 2
|
||||
or value.get("profileId") != manifest.get("profileId")
|
||||
or value.get("testOnly") is not True
|
||||
or value.get("algorithm") != "RS256"
|
||||
or value.get("issuer") != manifest.get("issuer")
|
||||
or value.get("audience") != manifest.get("audience")
|
||||
or value.get("ttlSeconds") != manifest.get("maxTtlSeconds")
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector does not match the active profile")
|
||||
protected_header = value.get("protectedHeader")
|
||||
if not isinstance(protected_header, dict) or set(protected_header) != {"alg", "kid", "typ"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
kid = protected_header.get("kid")
|
||||
if protected_header.get("alg") != "RS256" or protected_header.get("typ") != "JWT" or not _is_non_blank(kid):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector protected header is invalid")
|
||||
public_jwk = value.get("publicJwk")
|
||||
if not isinstance(public_jwk, dict) or set(public_jwk) != {"alg", "e", "kid", "kty", "n", "use"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
if (
|
||||
public_jwk.get("alg") != "RS256"
|
||||
or public_jwk.get("kid") != kid
|
||||
or public_jwk.get("kty") != "RSA"
|
||||
or public_jwk.get("use") != "sig"
|
||||
or not _is_non_blank(public_jwk.get("e"))
|
||||
or not _is_non_blank(public_jwk.get("n"))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector public JWK is invalid")
|
||||
|
||||
claims = value.get("expectedClaims")
|
||||
required_claims = manifest.get("requiredClaims")
|
||||
if not isinstance(claims, dict) or not isinstance(required_claims, list) or set(claims) != set(required_claims):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector claims do not match the active profile")
|
||||
operation = value.get("operation")
|
||||
if not isinstance(operation, dict) or set(operation) != {"action", "method", "operationId", "requestPath"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation is invalid")
|
||||
resource = claims.get("resource")
|
||||
if not isinstance(resource, dict) or set(resource) != {"id", "parent_id", "type"}:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector resource is invalid")
|
||||
expected_operation = {
|
||||
"action": "documents.read",
|
||||
"method": "GET",
|
||||
"operationId": "getDocument",
|
||||
"requestPath": "/knowledge-spaces/space-contract-vector/documents/document-contract-vector",
|
||||
}
|
||||
if operation != expected_operation:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector operation binding is invalid")
|
||||
exact_claims = {
|
||||
"action": operation["action"],
|
||||
"aud": value["audience"],
|
||||
"caller_kind": "interactive",
|
||||
"cap_ver": 2,
|
||||
"control_space_id": "control-space-contract-vector",
|
||||
"iss": value["issuer"],
|
||||
"namespace_id": "workspace-contract-vector",
|
||||
"resource": {
|
||||
"id": "document-contract-vector",
|
||||
"parent_id": "space-contract-vector",
|
||||
"type": "document",
|
||||
},
|
||||
"sub": "dify-account:account-contract-vector",
|
||||
}
|
||||
for field, expected in exact_claims.items():
|
||||
if claims.get(field) != expected:
|
||||
raise ValueError(f"KnowledgeFS Capability v2 test vector claim {field} is invalid")
|
||||
if claims.get("actor") != claims["sub"] or claims.get("azp") != "dify-console":
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector caller binding is invalid")
|
||||
issued_at = claims.get("iat")
|
||||
not_before = claims.get("nbf")
|
||||
expires_at = claims.get("exp")
|
||||
if (
|
||||
not isinstance(issued_at, int)
|
||||
or isinstance(issued_at, bool)
|
||||
or not_before != issued_at
|
||||
or not isinstance(expires_at, int)
|
||||
or isinstance(expires_at, bool)
|
||||
or expires_at - issued_at != value["ttlSeconds"]
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector TTL is invalid")
|
||||
expected_principal = {
|
||||
"callerKind": claims["caller_kind"],
|
||||
"subject": {
|
||||
"scopes": ["knowledge-spaces:read"],
|
||||
"subjectId": claims["sub"],
|
||||
"tenantId": claims["namespace_id"],
|
||||
},
|
||||
}
|
||||
if value.get("expectedPrincipal") != expected_principal:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector principal is invalid")
|
||||
|
||||
token = value.get("token")
|
||||
if not isinstance(token, str) or not _is_non_blank(token):
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token is invalid")
|
||||
try:
|
||||
verification_key = RSAAlgorithm.from_jwk(public_jwk)
|
||||
if not isinstance(verification_key, RSAPublicKey):
|
||||
raise ValueError("Capability vector verification key is not RSA public material")
|
||||
header = jwt.get_unverified_header(token)
|
||||
decoded_claims = jwt.decode(
|
||||
token,
|
||||
verification_key,
|
||||
algorithms=["RS256"],
|
||||
audience=cast(str, value["audience"]),
|
||||
issuer=cast(str, value["issuer"]),
|
||||
options={"verify_exp": False, "verify_iat": False, "verify_nbf": False},
|
||||
)
|
||||
except (jwt.PyJWTError, TypeError, ValueError) as exc:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector signature is invalid") from exc
|
||||
if header != protected_header or decoded_claims != claims:
|
||||
raise ValueError("KnowledgeFS Capability v2 test vector token content drifted")
|
||||
|
||||
|
||||
def _is_non_blank(value: object) -> bool:
|
||||
return isinstance(value, str) and bool(value.strip()) and value == value.strip()
|
||||
|
||||
|
||||
def validate_upstream_provenance(value: dict[str, Any]) -> None:
|
||||
"""Validate the imported-source provenance that is itself covered by the subtree tree ID."""
|
||||
expected_fields = {"commit", "release", "repository", "schemaVersion"}
|
||||
if set(value) != expected_fields:
|
||||
raise ValueError(f"KnowledgeFS upstream provenance fields must be exactly {sorted(expected_fields)}")
|
||||
if value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS upstream provenance must use schemaVersion 1")
|
||||
repository = value.get("repository")
|
||||
commit = value.get("commit")
|
||||
if not isinstance(repository, str) or not repository.startswith("https://"):
|
||||
raise ValueError("KnowledgeFS upstream provenance repository must be an HTTPS URL")
|
||||
if (
|
||||
not isinstance(commit, str)
|
||||
or len(commit) != 40
|
||||
or any(character not in "0123456789abcdef" for character in commit)
|
||||
):
|
||||
raise ValueError("KnowledgeFS upstream provenance commit must be a lowercase full Git SHA")
|
||||
if value.get("release") is not None and not isinstance(value["release"], str):
|
||||
raise ValueError("KnowledgeFS upstream provenance release must be null or a string")
|
||||
|
||||
|
||||
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
|
||||
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
|
||||
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
|
||||
@@ -151,7 +637,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
"operation_id": operation_id,
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(operation),
|
||||
"required_scope": required_scope(document, operation),
|
||||
"response_kind": response_kind(operation),
|
||||
"max_response_bytes": required_max_response_bytes(operation),
|
||||
"request_headers": request_header_names(path_item, operation),
|
||||
@@ -169,23 +655,9 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""Return transport declarations from the runtime Console operation registry."""
|
||||
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
"""The P9 backend exposes only typed product controllers; the raw Console proxy is removed."""
|
||||
|
||||
return tuple(
|
||||
{
|
||||
"operation_id": operation.operation_id,
|
||||
"method": operation.method,
|
||||
"path": operation.path,
|
||||
"required_scope": operation.required_scope,
|
||||
"response_kind": operation.response_kind,
|
||||
"max_response_bytes": operation.max_response_bytes,
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
}
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
)
|
||||
return ()
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
@@ -205,12 +677,17 @@ def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(sorted(media_types))
|
||||
|
||||
|
||||
def required_scope(operation: dict[str, Any]) -> str | None:
|
||||
def required_scope(document: dict[str, Any], operation: dict[str, Any]) -> str | None:
|
||||
scope = operation.get("x-knowledge-fs-required-scope")
|
||||
security = operation["security"] if "security" in operation else document.get("security")
|
||||
if security == []:
|
||||
if scope is not None:
|
||||
raise ValueError(f"KnowledgeFS public operation must not declare a required scope: {scope}")
|
||||
return None
|
||||
if security != [{"bearerAuth": []}]:
|
||||
raise ValueError(f"KnowledgeFS operation effective security must be exactly bearerAuth: {security}")
|
||||
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
|
||||
return scope
|
||||
if operation.get("security") == []:
|
||||
return None
|
||||
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,494 @@
|
||||
"""Cross-service KnowledgeFS product-operation contract validation.
|
||||
|
||||
The checked product manifest is the reviewable boundary, while the Dify product registry, Dify
|
||||
Capability issuer registry, exported KnowledgeFS OpenAPI, and exported TypeScript request-guard
|
||||
policy remain executable sources of truth. A product operation must appear exactly once as ready
|
||||
or as an explicit gap; internal KFS-only operations require a named exclusion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, Literal, NamedTuple, TypedDict, cast
|
||||
|
||||
|
||||
class ProductOperationLimits(TypedDict):
|
||||
kfsMaxResponseBytes: int
|
||||
productMaxRequestBytes: int
|
||||
productMaxResponseBytes: int
|
||||
|
||||
|
||||
class ProductOperationStream(TypedDict):
|
||||
kfsResponseKind: str
|
||||
productKind: str
|
||||
|
||||
|
||||
class ProductOperationManifestEntry(TypedDict):
|
||||
action: str
|
||||
kfsOperationId: str
|
||||
limits: ProductOperationLimits
|
||||
method: str
|
||||
path: str
|
||||
productOperationId: str
|
||||
resource: str
|
||||
stream: ProductOperationStream
|
||||
transport: str
|
||||
|
||||
|
||||
type ProductOperationManifestField = Literal[
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
]
|
||||
|
||||
PRODUCT_OPERATION_MANIFEST_FIELDS: tuple[ProductOperationManifestField, ...] = (
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
)
|
||||
|
||||
|
||||
class ProductOperationManifest(TypedDict):
|
||||
operations: list[ProductOperationManifestEntry]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationGapEntry(ProductOperationManifestEntry):
|
||||
reason: str
|
||||
reasonCode: str
|
||||
replacementProductOperationIds: list[str]
|
||||
|
||||
|
||||
class InternalKfsOperationExclusion(TypedDict):
|
||||
kfsOperationId: str
|
||||
reason: str
|
||||
reasonCode: str
|
||||
|
||||
|
||||
class ProductOperationGapManifest(TypedDict):
|
||||
gaps: list[ProductOperationGapEntry]
|
||||
internalKfsOperationExclusions: list[InternalKfsOperationExclusion]
|
||||
schemaVersion: int
|
||||
|
||||
|
||||
class ProductOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
kfs_operation_id: str
|
||||
max_request_bytes: int
|
||||
max_response_bytes: int
|
||||
method: str
|
||||
path: str
|
||||
product_operation_id: str
|
||||
ready: bool
|
||||
resource: str
|
||||
stream_kind: str
|
||||
transport: str
|
||||
|
||||
|
||||
class CapabilityOperationRuntimeContract(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
path: str
|
||||
resource: str
|
||||
|
||||
|
||||
class CapabilityOperationPolicy(NamedTuple):
|
||||
action: str
|
||||
allowed_caller_kinds: tuple[str, ...]
|
||||
method: str
|
||||
operation_id: str
|
||||
parent_resource_binding: dict[str, object] | None
|
||||
path: str
|
||||
resource: str
|
||||
resource_binding: dict[str, object]
|
||||
|
||||
|
||||
def product_operation_runtime_contracts() -> tuple[ProductOperationRuntimeContract, ...]:
|
||||
"""Read the complete typed Dify product registry without normalizing away gaps."""
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
|
||||
contracts: list[ProductOperationRuntimeContract] = []
|
||||
for product_operation_id, operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
|
||||
if operation.capability_operation_id is None or operation.kfs_path is None or operation.action is None:
|
||||
raise ValueError(f"Dify product operation {product_operation_id} has no Capability contract")
|
||||
contracts.append(
|
||||
ProductOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
kfs_operation_id=operation.capability_operation_id,
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
product_operation_id=product_operation_id,
|
||||
ready=is_product_operation_ready(product_operation_id),
|
||||
resource=operation.resource_resolver,
|
||||
stream_kind=operation.stream_kind,
|
||||
transport=operation.transport,
|
||||
)
|
||||
)
|
||||
return tuple(contracts)
|
||||
|
||||
|
||||
def capability_operation_runtime_contracts() -> tuple[CapabilityOperationRuntimeContract, ...]:
|
||||
"""Read the Python issuer policy used to mint each operation Capability."""
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
CapabilityOperationRuntimeContract(
|
||||
action=operation.action,
|
||||
allowed_caller_kinds=operation.allowed_caller_kinds,
|
||||
method=operation.method,
|
||||
operation_id=operation_id,
|
||||
path=operation.path,
|
||||
resource=operation.resource_type,
|
||||
)
|
||||
for operation_id, operation in KNOWLEDGE_FS_CAPABILITY_OPERATIONS.items()
|
||||
)
|
||||
|
||||
|
||||
def parse_product_operation_manifest(value: dict[str, Any]) -> ProductOperationManifest:
|
||||
"""Parse the ready product-operation inventory with a closed schema."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation manifest must use schemaVersion 2")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS product operation manifest operations must be a non-empty list")
|
||||
return {
|
||||
"operations": [_parse_product_entry(entry, gap=False) for entry in operations],
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_product_operation_gap_manifest(value: dict[str, Any]) -> ProductOperationGapManifest:
|
||||
"""Parse explicit product gaps and KFS-only operation exclusions with a closed schema."""
|
||||
expected_fields = {"gaps", "internalKfsOperationExclusions", "schemaVersion"}
|
||||
if set(value) != expected_fields or value.get("schemaVersion") != 2:
|
||||
raise ValueError("KnowledgeFS product operation gap manifest must use schemaVersion 2")
|
||||
gaps = value.get("gaps")
|
||||
exclusions = value.get("internalKfsOperationExclusions")
|
||||
if not isinstance(gaps, list) or not gaps or not isinstance(exclusions, list):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest lists are invalid")
|
||||
parsed_exclusions: list[InternalKfsOperationExclusion] = []
|
||||
for raw in exclusions:
|
||||
if not isinstance(raw, dict) or set(raw) != {"kfsOperationId", "reason", "reasonCode"}:
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusion fields are invalid")
|
||||
for field in ("kfsOperationId", "reason", "reasonCode"):
|
||||
_required_string(raw.get(field), f"internal exclusion {field}")
|
||||
parsed_exclusions.append(cast(InternalKfsOperationExclusion, raw))
|
||||
return {
|
||||
"gaps": [cast(ProductOperationGapEntry, _parse_product_entry(entry, gap=True)) for entry in gaps],
|
||||
"internalKfsOperationExclusions": parsed_exclusions,
|
||||
"schemaVersion": 2,
|
||||
}
|
||||
|
||||
|
||||
def parse_capability_operation_policy(value: dict[str, Any]) -> tuple[CapabilityOperationPolicy, ...]:
|
||||
"""Parse the runtime-exported TypeScript request-guard registry."""
|
||||
if set(value) != {"operations", "schemaVersion"} or value.get("schemaVersion") != 1:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must use schemaVersion 1")
|
||||
operations = value.get("operations")
|
||||
if not isinstance(operations, list) or not operations:
|
||||
raise ValueError("KnowledgeFS Capability operation policy must contain operations")
|
||||
parsed: list[CapabilityOperationPolicy] = []
|
||||
for raw in operations:
|
||||
expected_fields = {
|
||||
"action",
|
||||
"allowedCallerKinds",
|
||||
"method",
|
||||
"operationId",
|
||||
"parentResourceBinding",
|
||||
"path",
|
||||
"resourceBinding",
|
||||
"resourceType",
|
||||
}
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS Capability operation policy fields are invalid")
|
||||
callers = raw.get("allowedCallerKinds")
|
||||
resource_binding = raw.get("resourceBinding")
|
||||
parent_binding = raw.get("parentResourceBinding")
|
||||
if (
|
||||
not isinstance(callers, list)
|
||||
or not callers
|
||||
or not all(isinstance(caller, str) and caller for caller in callers)
|
||||
or not isinstance(resource_binding, dict)
|
||||
or not resource_binding
|
||||
or (parent_binding is not None and not isinstance(parent_binding, dict))
|
||||
):
|
||||
raise ValueError("KnowledgeFS Capability operation request binding is invalid")
|
||||
for field in ("action", "method", "operationId", "path", "resourceType"):
|
||||
_required_string(raw.get(field), f"Capability operation {field}")
|
||||
parsed.append(
|
||||
CapabilityOperationPolicy(
|
||||
action=cast(str, raw["action"]),
|
||||
allowed_caller_kinds=tuple(callers),
|
||||
method=cast(str, raw["method"]),
|
||||
operation_id=cast(str, raw["operationId"]),
|
||||
parent_resource_binding=cast(dict[str, object] | None, parent_binding),
|
||||
path=cast(str, raw["path"]),
|
||||
resource=cast(str, raw["resourceType"]),
|
||||
resource_binding=cast(dict[str, object], resource_binding),
|
||||
)
|
||||
)
|
||||
_unique((operation.operation_id for operation in parsed), "Capability operation policy")
|
||||
return tuple(parsed)
|
||||
|
||||
|
||||
def validate_product_operation_contracts(
|
||||
*,
|
||||
capability_operations: tuple[CapabilityOperationRuntimeContract, ...],
|
||||
capability_policy: tuple[CapabilityOperationPolicy, ...],
|
||||
document: dict[str, Any],
|
||||
gap_manifest: ProductOperationGapManifest,
|
||||
manifest: ProductOperationManifest,
|
||||
product_operations: tuple[ProductOperationRuntimeContract, ...],
|
||||
) -> None:
|
||||
"""Require exact agreement across manifests, Dify registries, KFS guard policy, and OpenAPI."""
|
||||
ready_entries = {entry["productOperationId"]: entry for entry in manifest["operations"]}
|
||||
gap_entries = {entry["productOperationId"]: entry for entry in gap_manifest["gaps"]}
|
||||
if len(ready_entries) != len(manifest["operations"]):
|
||||
raise ValueError("KnowledgeFS product operation manifest contains duplicate product ids")
|
||||
if len(gap_entries) != len(gap_manifest["gaps"]):
|
||||
raise ValueError("KnowledgeFS product operation gap manifest contains duplicate product ids")
|
||||
if set(ready_entries) & set(gap_entries):
|
||||
raise ValueError("KnowledgeFS product operation cannot be both ready and a gap")
|
||||
runtime_by_product = {operation.product_operation_id: operation for operation in product_operations}
|
||||
if set(ready_entries) | set(gap_entries) != set(runtime_by_product):
|
||||
missing = sorted(set(runtime_by_product) - set(ready_entries) - set(gap_entries))
|
||||
extra = sorted((set(ready_entries) | set(gap_entries)) - set(runtime_by_product))
|
||||
raise ValueError(f"KnowledgeFS product operation completeness drifted: missing={missing}, extra={extra}")
|
||||
|
||||
python_capabilities = {operation.operation_id: operation for operation in capability_operations}
|
||||
guard_capabilities = {operation.operation_id: operation for operation in capability_policy}
|
||||
openapi_operations = _openapi_operations(document)
|
||||
for product_operation_id, runtime in runtime_by_product.items():
|
||||
entry = ready_entries.get(product_operation_id) or gap_entries[product_operation_id]
|
||||
if runtime.ready != (product_operation_id in ready_entries):
|
||||
state = "ready" if runtime.ready else "gap"
|
||||
raise ValueError(f"Dify product operation {product_operation_id} must be declared as {state}")
|
||||
_validate_product_entry(entry, runtime, openapi_operations, python_capabilities, guard_capabilities)
|
||||
if product_operation_id in gap_entries:
|
||||
_required_string(gap_entries[product_operation_id]["reason"], "product gap reason")
|
||||
_required_string(gap_entries[product_operation_id]["reasonCode"], "product gap reasonCode")
|
||||
replacements = gap_entries[product_operation_id]["replacementProductOperationIds"]
|
||||
if not replacements or any(replacement not in ready_entries for replacement in replacements):
|
||||
raise ValueError(f"KnowledgeFS product gap {product_operation_id} has invalid replacements")
|
||||
|
||||
exclusions = gap_manifest["internalKfsOperationExclusions"]
|
||||
exclusion_by_id = {entry["kfsOperationId"]: entry for entry in exclusions}
|
||||
if len(exclusion_by_id) != len(exclusions):
|
||||
raise ValueError("KnowledgeFS internal KFS operation exclusions contain duplicate ids")
|
||||
lifecycle_operation_ids = {
|
||||
"activateDifyWorkspaceIntegration",
|
||||
"freezeDifyWorkspaceIntegration",
|
||||
}
|
||||
if set(exclusion_by_id) != lifecycle_operation_ids:
|
||||
raise ValueError("KnowledgeFS Dify integration lifecycle operations must be explicit internal exclusions")
|
||||
product_capability_ids = {operation.kfs_operation_id for operation in product_operations}
|
||||
for operation_id in exclusion_by_id:
|
||||
if operation_id in product_capability_ids:
|
||||
raise ValueError(f"Internal KFS operation {operation_id} cannot also be a product operation")
|
||||
_validate_capability_alignment(
|
||||
operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
python_operation = python_capabilities[operation_id]
|
||||
if python_operation.allowed_caller_kinds != ("internal_worker",) or python_operation.resource != "namespace":
|
||||
raise ValueError("Dify integration lifecycle operations must remain internal-worker namespace-only")
|
||||
|
||||
|
||||
def _parse_product_entry(raw: object, *, gap: bool) -> ProductOperationManifestEntry:
|
||||
base_fields = {
|
||||
"action",
|
||||
"kfsOperationId",
|
||||
"limits",
|
||||
"method",
|
||||
"path",
|
||||
"productOperationId",
|
||||
"resource",
|
||||
"stream",
|
||||
"transport",
|
||||
}
|
||||
expected_fields = base_fields | ({"reason", "reasonCode", "replacementProductOperationIds"} if gap else set())
|
||||
if not isinstance(raw, dict) or set(raw) != expected_fields:
|
||||
raise ValueError("KnowledgeFS product operation entry fields are invalid")
|
||||
for field in ("action", "kfsOperationId", "method", "path", "productOperationId", "resource", "transport"):
|
||||
_required_string(raw.get(field), f"product operation {field}")
|
||||
limits = raw.get("limits")
|
||||
stream = raw.get("stream")
|
||||
if not isinstance(limits, dict) or set(limits) != {
|
||||
"kfsMaxResponseBytes",
|
||||
"productMaxRequestBytes",
|
||||
"productMaxResponseBytes",
|
||||
}:
|
||||
raise ValueError("KnowledgeFS product operation limits are invalid")
|
||||
if not all(isinstance(limit, int) and not isinstance(limit, bool) and limit >= 0 for limit in limits.values()):
|
||||
raise ValueError("KnowledgeFS product operation limits must be nonnegative integers")
|
||||
if not isinstance(stream, dict) or set(stream) != {"kfsResponseKind", "productKind"}:
|
||||
raise ValueError("KnowledgeFS product operation stream contract is invalid")
|
||||
_required_string(stream.get("kfsResponseKind"), "KFS response kind")
|
||||
_required_string(stream.get("productKind"), "product stream kind")
|
||||
if gap:
|
||||
_required_string(raw.get("reason"), "product gap reason")
|
||||
_required_string(raw.get("reasonCode"), "product gap reasonCode")
|
||||
replacements = raw.get("replacementProductOperationIds")
|
||||
if not isinstance(replacements, list) or not all(isinstance(item, str) and item for item in replacements):
|
||||
raise ValueError("KnowledgeFS product operation gap replacements are invalid")
|
||||
return cast(ProductOperationManifestEntry, raw)
|
||||
|
||||
|
||||
def _validate_product_entry(
|
||||
entry: ProductOperationManifestEntry,
|
||||
runtime: ProductOperationRuntimeContract,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
_validate_capability_alignment(
|
||||
runtime.kfs_operation_id,
|
||||
openapi_operations,
|
||||
python_capabilities,
|
||||
guard_capabilities,
|
||||
)
|
||||
_, _, openapi_operation = openapi_operations[runtime.kfs_operation_id]
|
||||
expected: ProductOperationManifestEntry = {
|
||||
"action": runtime.action,
|
||||
"kfsOperationId": runtime.kfs_operation_id,
|
||||
"limits": {
|
||||
"kfsMaxResponseBytes": _openapi_max_response_bytes(openapi_operation),
|
||||
"productMaxRequestBytes": runtime.max_request_bytes,
|
||||
"productMaxResponseBytes": runtime.max_response_bytes,
|
||||
},
|
||||
"method": runtime.method,
|
||||
"path": runtime.path,
|
||||
"productOperationId": runtime.product_operation_id,
|
||||
"resource": runtime.resource,
|
||||
"stream": {
|
||||
"kfsResponseKind": _openapi_response_kind(openapi_operation),
|
||||
"productKind": runtime.stream_kind,
|
||||
},
|
||||
"transport": runtime.transport,
|
||||
}
|
||||
for field in PRODUCT_OPERATION_MANIFEST_FIELDS:
|
||||
expected_value = expected[field]
|
||||
if entry[field] != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS product operation {runtime.product_operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {entry[field]!r}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_capability_alignment(
|
||||
operation_id: str,
|
||||
openapi_operations: Mapping[str, tuple[str, str, dict[str, Any]]],
|
||||
python_capabilities: Mapping[str, CapabilityOperationRuntimeContract],
|
||||
guard_capabilities: Mapping[str, CapabilityOperationPolicy],
|
||||
) -> None:
|
||||
python_operation = python_capabilities.get(operation_id)
|
||||
guard_operation = guard_capabilities.get(operation_id)
|
||||
openapi_operation = openapi_operations.get(operation_id)
|
||||
if python_operation is None or guard_operation is None or openapi_operation is None:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} must exist in Python issuer, TypeScript guard, and OpenAPI"
|
||||
)
|
||||
method, path, _ = openapi_operation
|
||||
expected = (
|
||||
python_operation.action,
|
||||
python_operation.method,
|
||||
python_operation.path,
|
||||
python_operation.resource,
|
||||
)
|
||||
received_guard = (guard_operation.action, guard_operation.method, guard_operation.path, guard_operation.resource)
|
||||
if received_guard != expected:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} TypeScript guard drifted")
|
||||
if (method, path) != (python_operation.method, python_operation.path):
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} OpenAPI method/path drifted")
|
||||
if guard_operation.allowed_caller_kinds != python_operation.allowed_caller_kinds:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} caller policy drifted")
|
||||
|
||||
|
||||
def _openapi_operations(document: dict[str, Any]) -> dict[str, tuple[str, str, dict[str, Any]]]:
|
||||
operations: dict[str, tuple[str, str, dict[str, Any]]] = {}
|
||||
duplicates: set[str] = set()
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
if not isinstance(path, str) or not isinstance(path_item, dict):
|
||||
continue
|
||||
for method in ("delete", "get", "patch", "post", "put"):
|
||||
operation = path_item.get(method)
|
||||
if not isinstance(operation, dict):
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if not isinstance(operation_id, str) or not operation_id:
|
||||
continue
|
||||
if operation_id in operations:
|
||||
duplicates.add(operation_id)
|
||||
operations[operation_id] = (method.upper(), path, operation)
|
||||
if duplicates:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI contains duplicate operation ids: {sorted(duplicates)}")
|
||||
return operations
|
||||
|
||||
|
||||
def _openapi_response_kind(operation: dict[str, Any]) -> Literal["binary", "buffered", "stream"]:
|
||||
media_types = {
|
||||
media_type
|
||||
for status, response in operation.get("responses", {}).items()
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit())
|
||||
for media_type in response.get("content", {})
|
||||
}
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def _openapi_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError("KnowledgeFS product OpenAPI operation has no positive response byte limit")
|
||||
return value
|
||||
|
||||
|
||||
def _required_string(value: object, field: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip() or value != value.strip():
|
||||
raise ValueError(f"KnowledgeFS {field} must be a non-empty trimmed string")
|
||||
return value
|
||||
|
||||
|
||||
def _unique(values: Iterable[str], label: str) -> None:
|
||||
sequence = tuple(values)
|
||||
if len(sequence) != len(set(sequence)):
|
||||
raise ValueError(f"KnowledgeFS {label} contains duplicate ids")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CapabilityOperationPolicy",
|
||||
"CapabilityOperationRuntimeContract",
|
||||
"ProductOperationGapManifest",
|
||||
"ProductOperationManifest",
|
||||
"ProductOperationRuntimeContract",
|
||||
"capability_operation_runtime_contracts",
|
||||
"parse_capability_operation_policy",
|
||||
"parse_product_operation_gap_manifest",
|
||||
"parse_product_operation_manifest",
|
||||
"product_operation_runtime_contracts",
|
||||
"validate_product_operation_contracts",
|
||||
]
|
||||
@@ -164,6 +164,14 @@ def init_app(app: DifyApp) -> Celery:
|
||||
|
||||
# if you add a new task, please add the switch to CeleryScheduleTasksConfig
|
||||
beat_schedule: dict[str, CeleryBeatScheduleEntry] = {}
|
||||
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
|
||||
|
||||
if get_configured_knowledge_fs_lifecycle_worker_readiness().ready:
|
||||
imports.append("tasks.knowledge_fs_lifecycle_tasks")
|
||||
beat_schedule["knowledge_fs_lifecycle_worker"] = {
|
||||
"task": "tasks.knowledge_fs_lifecycle_tasks.run_knowledge_fs_lifecycle_worker",
|
||||
"schedule": timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_POLL_INTERVAL_SECONDS),
|
||||
}
|
||||
if dify_config.ENABLE_CLEAN_EMBEDDING_CACHE_TASK:
|
||||
imports.append("schedule.clean_embedding_cache_task")
|
||||
beat_schedule["clean_embedding_cache_task"] = {
|
||||
|
||||
@@ -27,6 +27,7 @@ def init_app(app: DifyApp):
|
||||
import_migration_data,
|
||||
install_plugins,
|
||||
install_rag_pipeline_plugins,
|
||||
knowledge_fs_control_space,
|
||||
migrate_data_for_plugin,
|
||||
migrate_dataset_permissions_to_rbac,
|
||||
migrate_member_roles_to_rbac,
|
||||
@@ -88,6 +89,7 @@ def init_app(app: DifyApp):
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
knowledge_fs_control_space,
|
||||
]
|
||||
for cmd in cmds_to_register:
|
||||
app.cli.add_command(cmd)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Production assembly for KnowledgeFS database-backed metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from core.db.session_factory import session_factory
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_state_metrics import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceStateCountReader,
|
||||
)
|
||||
from services.knowledge_fs.observability import get_knowledge_fs_operational_metrics
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_app(app: Flask) -> None:
|
||||
del app
|
||||
try:
|
||||
reader = SQLAlchemyKnowledgeFSControlSpaceStateCountReader(session_factory.get_session_maker())
|
||||
get_knowledge_fs_operational_metrics().register_control_space_state_gauge(reader)
|
||||
except Exception:
|
||||
logger.exception("KnowledgeFS operational metric registration failed; continuing application startup")
|
||||
@@ -1,5 +1,9 @@
|
||||
{
|
||||
"commit": "4310e2d582d25e7de58183f27720afab01e123cf",
|
||||
"openapiSha256": "5827ca930ce38462bfd1b2bef387efbf37eb7ffcaedde4558af2fbaeccbfbc4b",
|
||||
"repository": "https://github.com/langgenius/knowledge-fs"
|
||||
"schemaVersion": 5,
|
||||
"subtreeTree": "04f9a6189c215f4fcccdccbb3ae7fc810b927463",
|
||||
"openapiSha256": "df702b15769f79c561e34d72472a2ec8dddcdefe4bd4c82858f06a98cd6eae8c",
|
||||
"capabilityV2AuthManifestSha256": "fc0a47e23cce12544882f0298522b4933002e892b84ce1815df7e81d36a7a0c7",
|
||||
"capabilityV2AuthTestVectorSha256": "ae0de37b1ff05c40f905cf17a7b410d8971acacf64db07d5ee3d6fecfa559ce3",
|
||||
"productOperationManifestSha256": "5ceef85419dfba9c803732f78f31c803a0836e883972bb67f26ae43b0c78d341",
|
||||
"productOperationGapManifestSha256": "ccbae37fe658177a77529822211a2cf02e72b5815cbc9b7e7e65e6817ba5e0a9"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"gaps": [
|
||||
{
|
||||
"productOperationId": "createDocument",
|
||||
"kfsOperationId": "uploadDocument",
|
||||
"method": "POST",
|
||||
"path": "/knowledge-spaces/{id}/documents",
|
||||
"action": "documents.create",
|
||||
"resource": "knowledge_space",
|
||||
"transport": "multipart",
|
||||
"stream": {
|
||||
"productKind": "buffered-multipart",
|
||||
"kfsResponseKind": "buffered"
|
||||
},
|
||||
"limits": {
|
||||
"productMaxRequestBytes": 0,
|
||||
"productMaxResponseBytes": 0,
|
||||
"kfsMaxResponseBytes": 1048576
|
||||
},
|
||||
"reasonCode": "NON_HOMOMORPHIC_MULTIPART_TRANSPORT",
|
||||
"reason": "Legacy buffered multipart document creation has no homomorphic Dify product transport; use Capability v2 direct-upload operations.",
|
||||
"replacementProductOperationIds": [
|
||||
"createUploadSession",
|
||||
"presignUploadSessionPart",
|
||||
"uploadSmallFile",
|
||||
"completeUploadSession",
|
||||
"abortUploadSession"
|
||||
]
|
||||
}
|
||||
],
|
||||
"internalKfsOperationExclusions": [
|
||||
{
|
||||
"kfsOperationId": "activateDifyWorkspaceIntegration",
|
||||
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
|
||||
"reason": "Workspace activation is an internal cutover control-plane operation and is never exposed as a product operation."
|
||||
},
|
||||
{
|
||||
"kfsOperationId": "freezeDifyWorkspaceIntegration",
|
||||
"reasonCode": "INTERNAL_CONTROL_PLANE_ONLY",
|
||||
"reason": "Workspace maintenance freeze is an internal cutover control-plane operation and is never exposed as a product operation."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"schemaVersion": 2,
|
||||
"operations": [
|
||||
{"productOperationId":"batchSpaceSummaries","kfsOperationId":"batchKnowledgeSpaceProductSummaries","method":"POST","path":"/internal/knowledge-spaces/product-summaries/batch","action":"knowledge_spaces.status.batch","resource":"namespace","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSpace","kfsOperationId":"getKnowledgeSpace","method":"GET","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSpace","kfsOperationId":"updateKnowledgeSpace","method":"PATCH","path":"/knowledge-spaces/{id}","action":"knowledge_spaces.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSettings","kfsOperationId":"getKnowledgeSpaceProductSettings","method":"GET","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.read","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSettings","kfsOperationId":"updateKnowledgeSpaceProductSettings","method":"PATCH","path":"/knowledge-spaces/{id}/product-settings","action":"knowledge_spaces.settings.update","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocuments","kfsOperationId":"listDocuments","method":"GET","path":"/knowledge-spaces/{id}/documents","action":"documents.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocument","kfsOperationId":"getDocument","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocumentOutline","kfsOperationId":"getDocumentOutline","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/outline","action":"documents.outline.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocumentRevisions","kfsOperationId":"listDocumentRevisions","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions","action":"documents.revisions.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateDocumentMetadata","kfsOperationId":"patchDocumentMetadata","method":"PATCH","path":"/knowledge-spaces/{id}/documents/{documentId}/metadata","action":"documents.metadata.update","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listDocumentChunks","kfsOperationId":"listDocumentChunks","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks","action":"documents.chunks.list","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getDocumentChunk","kfsOperationId":"getDocumentChunk","method":"GET","path":"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}","action":"documents.chunks.read","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":1048576,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"deleteDocument","kfsOperationId":"requestDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/{documentId}","action":"documents.delete","resource":"document","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"bulkDeleteDocuments","kfsOperationId":"requestBulkDocumentDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/documents/bulk","action":"documents.bulk.delete","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"reindexDocuments","kfsOperationId":"bulkReindexDocuments","method":"POST","path":"/knowledge-spaces/{id}/documents/bulk/reindex","action":"documents.bulk.reindex","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getCompilationJob","kfsOperationId":"getDocumentCompilationJob","method":"GET","path":"/jobs/{id}","action":"document_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"cancelCompilationJob","kfsOperationId":"cancelDocumentCompilationJob","method":"DELETE","path":"/jobs/{id}","action":"document_jobs.cancel","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"retryCompilationJob","kfsOperationId":"retryDocumentCompilationJob","method":"POST","path":"/jobs/{id}/retry","action":"document_jobs.retry","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getBulkJob","kfsOperationId":"getBulkOperation","method":"GET","path":"/bulk-jobs/{id}","action":"bulk_jobs.read","resource":"job","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSources","kfsOperationId":"listKnowledgeSpaceSources","method":"GET","path":"/knowledge-spaces/{id}/sources","action":"sources.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createSource","kfsOperationId":"createKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources","action":"sources.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getSource","kfsOperationId":"getKnowledgeSpaceSource","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.read","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"updateSource","kfsOperationId":"updateKnowledgeSpaceSource","method":"PATCH","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.update","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":262144,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"deleteSource","kfsOperationId":"requestSourceDeletion","method":"DELETE","path":"/knowledge-spaces/{id}/sources/{sourceId}","action":"sources.delete","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"testSource","kfsOperationId":"testKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/test","action":"sources.test","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":262144,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"crawlSource","kfsOperationId":"crawlKnowledgeSpaceSource","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/crawl","action":"sources.crawl","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSourcePages","kfsOperationId":"listKnowledgeSpaceSourcePages","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/pages","action":"sources.pages.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"importSourcePages","kfsOperationId":"importKnowledgeSpaceSourcePages","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import","action":"sources.pages.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listSourceFiles","kfsOperationId":"listKnowledgeSpaceSourceFiles","method":"GET","path":"/knowledge-spaces/{id}/sources/{sourceId}/files","action":"sources.files.list","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"importSourceFiles","kfsOperationId":"importKnowledgeSpaceSourceFiles","method":"POST","path":"/knowledge-spaces/{id}/sources/{sourceId}/import-files","action":"sources.files.import","resource":"source","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":1048576,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createQuery","kfsOperationId":"createQuery","method":"POST","path":"/queries","action":"queries.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}},
|
||||
{"productOperationId":"listResearchTasks","kfsOperationId":"listKnowledgeSpaceResearchTasks","method":"GET","path":"/knowledge-spaces/{id}/research-tasks","action":"research_tasks.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createResearchTask","kfsOperationId":"createResearchTask","method":"POST","path":"/research-tasks","action":"research_tasks.create","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"planResearchTask","kfsOperationId":"planResearchTask","method":"POST","path":"/research-tasks/plan","action":"research_tasks.plan","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getResearchTask","kfsOperationId":"getResearchTask","method":"GET","path":"/research-tasks/{id}","action":"research_tasks.read","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listResearchTaskPartials","kfsOperationId":"listResearchTaskPartials","method":"GET","path":"/research-tasks/{id}/partials","action":"research_tasks.partials.list","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":8388608,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"cancelResearchTask","kfsOperationId":"cancelResearchTask","method":"DELETE","path":"/research-tasks/{id}","action":"research_tasks.cancel","resource":"research_task","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":524288,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraces","kfsOperationId":"listKnowledgeSpaceQualityTraces","method":"GET","path":"/knowledge-spaces/{id}/quality/traces","action":"quality.traces.list","resource":"knowledge_space","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"getTrace","kfsOperationId":"getAnswerTrace","method":"GET","path":"/queries/{traceId}","action":"queries.read","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":2097152,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceEvidence","kfsOperationId":"listQueryEvidence","method":"GET","path":"/queries/{traceId}/evidence","action":"queries.evidence.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceConflicts","kfsOperationId":"listQueryConflicts","method":"GET","path":"/queries/{traceId}/conflicts","action":"queries.conflicts.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"listTraceMissing","kfsOperationId":"listQueryMissing","method":"GET","path":"/queries/{traceId}/missing","action":"queries.missing.list","resource":"query","transport":"json","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":16384,"productMaxResponseBytes":4194304,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"createUploadSession","kfsOperationId":"createUploadSession","method":"POST","path":"/knowledge-spaces/{id}/upload-sessions","action":"upload_sessions.create","resource":"knowledge_space","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":65536,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"presignUploadSessionPart","kfsOperationId":"presignUploadSessionPart","method":"POST","path":"/upload-sessions/{id}/parts/{partNumber}/presign","action":"upload_sessions.write","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"uploadSmallFile","kfsOperationId":"uploadSmallFile","method":"POST","path":"/upload-sessions/{id}/small-file","action":"upload_sessions.write","resource":"upload_session","transport":"binary","stream":{"productKind":"json","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":8388608,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"completeUploadSession","kfsOperationId":"completeUploadSession","method":"POST","path":"/upload-sessions/{id}/complete","action":"upload_sessions.complete","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":131072,"productMaxResponseBytes":131072,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"abortUploadSession","kfsOperationId":"abortUploadSession","method":"POST","path":"/upload-sessions/{id}/abort","action":"upload_sessions.abort","resource":"upload_session","transport":"direct","stream":{"productKind":"direct-upload","kfsResponseKind":"buffered"},"limits":{"productMaxRequestBytes":32768,"productMaxResponseBytes":65536,"kfsMaxResponseBytes":1048576}},
|
||||
{"productOperationId":"streamResearchTask","kfsOperationId":"streamResearchTaskProgress","method":"GET","path":"/research-tasks/{id}/events","action":"research_tasks.stream","resource":"research_task","transport":"direct","stream":{"productKind":"sse","kfsResponseKind":"stream"},"limits":{"productMaxRequestBytes":0,"productMaxResponseBytes":0,"kfsMaxResponseBytes":67108864}}
|
||||
]
|
||||
}
|
||||
+387
@@ -0,0 +1,387 @@
|
||||
"""add independent KnowledgeFS control-plane tables
|
||||
|
||||
Revision ID: a4e7c2f91b30
|
||||
Revises: 3c9f8e2a1d7b
|
||||
Create Date: 2026-07-21 12:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "a4e7c2f91b30"
|
||||
down_revision = "3c9f8e2a1d7b"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def _timestamps() -> tuple[sa.Column, sa.Column]:
|
||||
return (
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_control_spaces",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("knowledge_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("knowledge_space_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("owner_account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("visibility", sa.String(length=32), server_default=sa.text("'only_me'"), nullable=False),
|
||||
sa.Column("provisioning_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("lifecycle_operation_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("state", sa.String(length=32), server_default=sa.text("'provisioning'"), nullable=False),
|
||||
sa.Column("resource_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(length=255), nullable=True),
|
||||
sa.Column("last_error_message", models.types.LongText(), nullable=True),
|
||||
sa.Column("last_synced_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("deletion_irreversible_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
|
||||
sa.UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id"],
|
||||
["tenants.id"],
|
||||
name="kfs_control_space_workspace_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("resource_version >= 0", name="kfs_control_space_resource_version_ck"),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="kfs_control_space_attempt_count_ck"),
|
||||
sa.CheckConstraint(
|
||||
"state != 'active' OR knowledge_space_id IS NOT NULL",
|
||||
name="kfs_control_space_active_registration_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
|
||||
name="kfs_control_space_irreversible_state_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"knowledge_space_revision >= 0",
|
||||
name="kfs_control_space_remote_revision_ck",
|
||||
),
|
||||
)
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "knowledge_space_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
|
||||
)
|
||||
else:
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "knowledge_space_id"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_state_updated_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["state", "updated_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_state_updated_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "state", "updated_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_tenant_owner_state_idx",
|
||||
"knowledge_fs_control_spaces",
|
||||
["tenant_id", "owner_account_id", "state"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_control_space_permissions",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("role", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("granted_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"account_id",
|
||||
name="kfs_control_space_permission_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_control_space_permission_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_control_space_permission_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_control_space_permission_account_idx",
|
||||
"knowledge_fs_control_space_permissions",
|
||||
["tenant_id", "account_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_external_access_policies",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("service_api_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("agent_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("workflow_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("mcp_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("updated_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_external_access_policy_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_external_access_policy_revision_ck"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_api_credentials",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("credential_hash", sa.String(length=255), nullable=False),
|
||||
sa.Column("credential_prefix", sa.String(length=32), nullable=False),
|
||||
sa.Column("credential_last4", sa.String(length=4), nullable=False),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("allowed_actions", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_used_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoke_reason", sa.String(length=255), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
|
||||
sa.UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_api_credential_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="kfs_api_credential_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_api_credential_tenant_space_status_idx",
|
||||
"knowledge_fs_api_credentials",
|
||||
["tenant_id", "control_space_id", "status"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_api_credential_tenant_prefix_idx",
|
||||
"knowledge_fs_api_credentials",
|
||||
["tenant_id", "credential_prefix"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"app_knowledge_fs_space_joins",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("app_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("join_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'active'"), nullable=False),
|
||||
sa.Column("revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("revoked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("revoked_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"app_id",
|
||||
"control_space_id",
|
||||
"join_type",
|
||||
name="app_kfs_space_join_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="app_kfs_space_join_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name="app_kfs_space_join_revision_ck"),
|
||||
)
|
||||
op.create_index(
|
||||
"app_kfs_space_join_app_status_idx",
|
||||
"app_knowledge_fs_space_joins",
|
||||
["tenant_id", "app_id", "status"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_authorization_revisions",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("membership_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("space_acl_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("external_access_epoch", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("content_policy_revision", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("revoke_sequence", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_authorization_revision_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("membership_epoch >= 0", name="kfs_authorization_membership_epoch_ck"),
|
||||
sa.CheckConstraint("space_acl_epoch >= 0", name="kfs_authorization_space_acl_epoch_ck"),
|
||||
sa.CheckConstraint("external_access_epoch >= 0", name="kfs_authorization_external_access_epoch_ck"),
|
||||
sa.CheckConstraint("content_policy_revision >= 0", name="kfs_authorization_content_policy_revision_ck"),
|
||||
sa.CheckConstraint("revoke_sequence >= 0", name="kfs_authorization_revoke_sequence_ck"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("trace_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("jti_hash", sa.String(length=80), nullable=False),
|
||||
sa.Column("claims_summary", sa.JSON(), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
|
||||
sa.UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_audit_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
["tenant_id", "control_space_id", "created_at"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_audit_trace_idx",
|
||||
"knowledge_fs_capability_issuance_audits",
|
||||
["tenant_id", "trace_id"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("operation_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("operation", sa.String(length=32), nullable=False),
|
||||
sa.Column("command_payload", sa.JSON(), nullable=False),
|
||||
sa.Column("expected_control_space_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("expected_knowledge_space_revision", sa.BigInteger(), nullable=True),
|
||||
sa.Column("command_schema_version", sa.Integer(), server_default=sa.text("1"), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'pending'"), nullable=False),
|
||||
sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("next_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_attempt_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(length=255), nullable=True),
|
||||
sa.Column("lease_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("last_error_code", sa.String(length=255), nullable=True),
|
||||
sa.Column("last_error_message", models.types.LongText(), nullable=True),
|
||||
sa.Column("completed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("retain_until", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
|
||||
sa.UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_lifecycle_outbox_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("command_schema_version >= 1", name="kfs_lifecycle_outbox_schema_version_ck"),
|
||||
sa.CheckConstraint("expected_control_space_version >= 0", name="kfs_lifecycle_outbox_expected_version_ck"),
|
||||
sa.CheckConstraint("attempt_count >= 0", name="kfs_lifecycle_outbox_attempt_count_ck"),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
|
||||
name="kfs_lifecycle_outbox_status_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
|
||||
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
|
||||
name="kfs_lifecycle_outbox_lease_state_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
|
||||
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
|
||||
name="kfs_lifecycle_outbox_terminal_state_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_lifecycle_outbox_dispatch_idx",
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
["status", "next_attempt_at", "id"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_lifecycle_outbox_space_created_idx",
|
||||
"knowledge_fs_lifecycle_outbox",
|
||||
["tenant_id", "control_space_id", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("kfs_lifecycle_outbox_space_created_idx", table_name="knowledge_fs_lifecycle_outbox")
|
||||
op.drop_index("kfs_lifecycle_outbox_dispatch_idx", table_name="knowledge_fs_lifecycle_outbox")
|
||||
op.drop_table("knowledge_fs_lifecycle_outbox")
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_audit_trace_idx",
|
||||
table_name="knowledge_fs_capability_issuance_audits",
|
||||
)
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
table_name="knowledge_fs_capability_issuance_audits",
|
||||
)
|
||||
op.drop_table("knowledge_fs_capability_issuance_audits")
|
||||
op.drop_table("knowledge_fs_authorization_revisions")
|
||||
op.drop_index("app_kfs_space_join_app_status_idx", table_name="app_knowledge_fs_space_joins")
|
||||
op.drop_table("app_knowledge_fs_space_joins")
|
||||
op.drop_index("kfs_api_credential_tenant_prefix_idx", table_name="knowledge_fs_api_credentials")
|
||||
op.drop_index("kfs_api_credential_tenant_space_status_idx", table_name="knowledge_fs_api_credentials")
|
||||
op.drop_table("knowledge_fs_api_credentials")
|
||||
op.drop_table("knowledge_fs_external_access_policies")
|
||||
op.drop_index(
|
||||
"kfs_control_space_permission_account_idx",
|
||||
table_name="knowledge_fs_control_space_permissions",
|
||||
)
|
||||
op.drop_table("knowledge_fs_control_space_permissions")
|
||||
op.drop_index("kfs_control_space_tenant_owner_state_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_tenant_state_updated_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_state_updated_idx", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_index("kfs_control_space_tenant_space_uq", table_name="knowledge_fs_control_spaces")
|
||||
op.drop_table("knowledge_fs_control_spaces")
|
||||
@@ -0,0 +1,205 @@
|
||||
"""add KnowledgeFS workspace cutover ledger and migration evidence
|
||||
|
||||
Revision ID: b7f2a9d41c60
|
||||
Revises: a4e7c2f91b30
|
||||
Create Date: 2026-07-21 13:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "b7f2a9d41c60"
|
||||
down_revision = "a4e7c2f91b30"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def _timestamps() -> tuple[sa.Column, sa.Column]:
|
||||
return (
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_workspace_cutover_ledgers",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("phase", sa.String(length=32), server_default=sa.text("'inventory'"), nullable=False),
|
||||
sa.Column("source_revision_watermark", sa.JSON(), nullable=False),
|
||||
sa.Column("final_revision_watermark", sa.JSON(), nullable=True),
|
||||
sa.Column("applied_revision_watermark", sa.JSON(), nullable=False),
|
||||
sa.Column("source_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("final_task_watermark", sa.BigInteger(), nullable=True),
|
||||
sa.Column("applied_task_watermark", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("freeze_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("cutover_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("rolled_back_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("rollback_cutoff_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_started_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_window_ends_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("observation_completed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("maximum_task_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("irreversible_cleanup_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("product_routes_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("capability_v2_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("integrated_mode_enabled", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("legacy_acl_read_only", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("smoke_results", sa.JSON(), nullable=True),
|
||||
sa.Column("legacy_dependency_report", sa.JSON(), nullable=True),
|
||||
sa.Column("legacy_dependency_checked_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("legacy_dependency_ready", sa.Boolean(), server_default=sa.text("false"), nullable=False),
|
||||
sa.Column("cas_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
|
||||
sa.UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
|
||||
sa.CheckConstraint("cas_version >= 0", name="kfs_workspace_cutover_cas_version_ck"),
|
||||
sa.CheckConstraint(
|
||||
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
|
||||
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
|
||||
name="kfs_workspace_cutover_task_watermark_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"cutover_at IS NULL OR freeze_at IS NOT NULL",
|
||||
name="kfs_workspace_cutover_freeze_before_cutover_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
|
||||
name="kfs_workspace_cutover_rollback_cutoff_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_workspace_cutover_ledger_phase_updated_idx",
|
||||
"knowledge_fs_workspace_cutover_ledgers",
|
||||
["phase", "updated_at"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_migration_issues",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("issue_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("kind", sa.String(length=48), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), server_default=sa.text("'open'"), nullable=False),
|
||||
sa.Column("resource_type", sa.String(length=64), nullable=True),
|
||||
sa.Column("resource_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_issue_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_migration_issue_gate_idx",
|
||||
"knowledge_fs_migration_issues",
|
||||
["tenant_id", "ledger_id", "status", "kind"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_migration_quarantine",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("source_kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("reason_code", sa.String(length=64), nullable=False),
|
||||
sa.Column("disposition", sa.String(length=32), nullable=False),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"source_kind",
|
||||
"source_id",
|
||||
name="kfs_migration_quarantine_source_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_quarantine_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_migration_quarantine_disposition_idx",
|
||||
"knowledge_fs_migration_quarantine",
|
||||
["tenant_id", "ledger_id", "disposition"],
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"knowledge_fs_shadow_authorization_diffs",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("diff_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
|
||||
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
|
||||
sa.Column("decision", sa.String(length=32), nullable=False),
|
||||
sa.Column("reason", models.types.LongText(), nullable=False),
|
||||
sa.Column("observed_revision", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(), nullable=True),
|
||||
*_timestamps(),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
|
||||
sa.UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_diff_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_shadow_authorization_diff_gate_idx",
|
||||
"knowledge_fs_shadow_authorization_diffs",
|
||||
["tenant_id", "ledger_id", "status", "decision"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"kfs_shadow_authorization_diff_gate_idx",
|
||||
table_name="knowledge_fs_shadow_authorization_diffs",
|
||||
)
|
||||
op.drop_table("knowledge_fs_shadow_authorization_diffs")
|
||||
op.drop_index(
|
||||
"kfs_migration_quarantine_disposition_idx",
|
||||
table_name="knowledge_fs_migration_quarantine",
|
||||
)
|
||||
op.drop_table("knowledge_fs_migration_quarantine")
|
||||
op.drop_index("kfs_migration_issue_gate_idx", table_name="knowledge_fs_migration_issues")
|
||||
op.drop_table("knowledge_fs_migration_issues")
|
||||
op.drop_index(
|
||||
"kfs_workspace_cutover_ledger_phase_updated_idx",
|
||||
table_name="knowledge_fs_workspace_cutover_ledgers",
|
||||
)
|
||||
op.drop_table("knowledge_fs_workspace_cutover_ledgers")
|
||||
+365
@@ -0,0 +1,365 @@
|
||||
"""add guarded cleanup, cutover evidence, and capability issuance reservations
|
||||
|
||||
Revision ID: c8e31b7d52a4
|
||||
Revises: b7f2a9d41c60
|
||||
Create Date: 2026-07-21 14:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "c8e31b7d52a4"
|
||||
down_revision = "b7f2a9d41c60"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _uuid_column(name: str, **kwargs):
|
||||
if op.get_bind().dialect.name == "postgresql":
|
||||
kwargs.setdefault("server_default", sa.text("uuidv7()"))
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"knowledge_fs_cleanup_authorizations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("request_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("plan_schema_version", sa.String(length=32), nullable=False),
|
||||
sa.Column("plan_digest", sa.String(length=71), nullable=False),
|
||||
sa.Column("targets", sa.JSON(), nullable=False),
|
||||
sa.Column("readiness_evidence", sa.JSON(), nullable=False),
|
||||
sa.Column("requested_by_account_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("requested_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("readiness_ledger_cas_version", sa.BigInteger(), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), server_default=sa.text("'requested'"), nullable=False),
|
||||
sa.Column("approved_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("approved_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("approval_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("approved_ledger_cas_version", sa.BigInteger(), nullable=True),
|
||||
sa.Column("started_by_account_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("started_ledger_cas_version", sa.BigInteger(), nullable=True),
|
||||
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"request_id",
|
||||
name="kfs_cleanup_authorization_request_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_cleanup_authorization_ledger_fk",
|
||||
),
|
||||
sa.CheckConstraint("row_version >= 0", name="kfs_cleanup_authorization_version_ck"),
|
||||
sa.CheckConstraint(
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
|
||||
name="kfs_cleanup_authorization_ledger_versions_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL)",
|
||||
name="kfs_cleanup_authorization_status_fields_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
|
||||
name="kfs_cleanup_authorization_approval_window_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_cleanup_authorization_status_idx",
|
||||
"knowledge_fs_cleanup_authorizations",
|
||||
["tenant_id", "status", "updated_at"],
|
||||
)
|
||||
op.create_table(
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("grant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("trace_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("subject", sa.String(length=255), nullable=False),
|
||||
sa.Column("caller_kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("request_summary", sa.JSON(), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), server_default=sa.text("'reserved'"), nullable=False),
|
||||
sa.Column("issued_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("token_expires_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("failed_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("failure_code", sa.String(length=128), nullable=True),
|
||||
sa.Column("cleanup_after", sa.DateTime(), nullable=True),
|
||||
sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"grant_id",
|
||||
name="kfs_capability_issuance_reservation_grant_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_reservation_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"row_version >= 0",
|
||||
name="kfs_capability_issuance_reservation_version_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
|
||||
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
|
||||
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
|
||||
name="kfs_capability_issuance_reservation_status_fields_ck",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
|
||||
name="kfs_capability_issuance_reservation_cleanup_window_ck",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
["tenant_id", "control_space_id", "subject", "caller_kind"],
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
"knowledge_fs_capability_issuance_reservations",
|
||||
["tenant_id", "trace_id"],
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("resolved_by_operator", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("resolved_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_migration_quarantine_version_ck",
|
||||
"row_version >= 0",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_migration_quarantine_resolution_fields_ck",
|
||||
"(disposition = 'resolved' "
|
||||
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
|
||||
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
|
||||
"(disposition <> 'resolved' "
|
||||
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
|
||||
"AND evidence IS NULL AND resolved_at IS NULL)",
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("shadow_started_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_evidence_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(
|
||||
sa.Column("shadow_observation_count", sa.BigInteger(), server_default=sa.text("0"), nullable=False)
|
||||
)
|
||||
batch_op.add_column(sa.Column("shadow_window_started_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_window_ended_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(
|
||||
sa.Column("shadow_traffic_zero", sa.Boolean(), server_default=sa.text("false"), nullable=False)
|
||||
)
|
||||
batch_op.add_column(sa.Column("shadow_traffic_zero_evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_latest_observed_revision", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_producer", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_by_operator", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("shadow_completed_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_id", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_revision", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_control_space_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_activated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_updated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_acknowledged_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_applied", sa.Boolean(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_activation_replayed", sa.Boolean(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_count_ck",
|
||||
"shadow_observation_count >= 0",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_window_ck",
|
||||
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_shadow_completion_fields_ck",
|
||||
"(shadow_completed_at IS NULL "
|
||||
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
|
||||
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
|
||||
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
|
||||
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
|
||||
"AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
|
||||
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
|
||||
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
|
||||
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
|
||||
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
|
||||
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_activation_fields_ck",
|
||||
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
|
||||
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
|
||||
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
|
||||
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
|
||||
"AND remote_activation_replayed IS NULL) OR "
|
||||
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
|
||||
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
|
||||
"AND remote_activation_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
|
||||
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_activation_time_ck",
|
||||
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("current_evidence_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("last_observed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("row_version", sa.BigInteger(), server_default=sa.text("0"), nullable=False))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_shadow_authorization_diff_version_ck",
|
||||
"row_version >= 0",
|
||||
)
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE knowledge_fs_shadow_authorization_diffs "
|
||||
"SET current_evidence_digest = 'sha256:legacy-unversioned', last_observed_at = created_at "
|
||||
"WHERE current_evidence_digest IS NULL OR last_observed_at IS NULL"
|
||||
)
|
||||
)
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.alter_column("current_evidence_digest", existing_type=sa.String(length=71), nullable=False)
|
||||
batch_op.alter_column("last_observed_at", existing_type=sa.DateTime(), nullable=False)
|
||||
op.create_table(
|
||||
"knowledge_fs_shadow_authorization_observations",
|
||||
_uuid_column("id", nullable=False),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("ledger_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("diff_key", sa.String(length=255), nullable=False),
|
||||
sa.Column("producer", sa.String(length=255), nullable=False),
|
||||
sa.Column("control_space_id", models.types.StringUUID(), nullable=True),
|
||||
sa.Column("principal", sa.String(length=255), nullable=False),
|
||||
sa.Column("legacy_allowed", sa.Boolean(), nullable=True),
|
||||
sa.Column("dify_allowed", sa.Boolean(), nullable=False),
|
||||
sa.Column("decision", sa.String(length=32), nullable=False),
|
||||
sa.Column("reason", models.types.LongText(), nullable=False),
|
||||
sa.Column("observed_revision", sa.JSON(), nullable=False),
|
||||
sa.Column("observed_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("evidence_digest", sa.String(length=71), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"diff_key",
|
||||
"evidence_digest",
|
||||
name="kfs_shadow_authorization_observation_evidence_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_observation_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
"knowledge_fs_shadow_authorization_observations",
|
||||
["tenant_id", "ledger_id", "observed_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
table_name="knowledge_fs_shadow_authorization_observations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_shadow_authorization_observations")
|
||||
with op.batch_alter_table("knowledge_fs_shadow_authorization_diffs", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_shadow_authorization_diff_version_ck", type_="check")
|
||||
batch_op.drop_column("row_version")
|
||||
batch_op.drop_column("last_observed_at")
|
||||
batch_op.drop_column("current_evidence_digest")
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_activation_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_completion_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_window_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_shadow_count_ck", type_="check")
|
||||
batch_op.drop_column("remote_activation_replayed")
|
||||
batch_op.drop_column("remote_activation_applied")
|
||||
batch_op.drop_column("remote_activation_acknowledged_at")
|
||||
batch_op.drop_column("remote_activation_updated_at")
|
||||
batch_op.drop_column("remote_activation_activated_at")
|
||||
batch_op.drop_column("remote_activation_control_space_id")
|
||||
batch_op.drop_column("remote_activation_digest")
|
||||
batch_op.drop_column("remote_activation_revision")
|
||||
batch_op.drop_column("remote_activation_id")
|
||||
batch_op.drop_column("shadow_completed_by_account_id")
|
||||
batch_op.drop_column("shadow_completed_by_operator")
|
||||
batch_op.drop_column("shadow_producer")
|
||||
batch_op.drop_column("shadow_latest_observed_revision")
|
||||
batch_op.drop_column("shadow_traffic_zero_evidence")
|
||||
batch_op.drop_column("shadow_traffic_zero")
|
||||
batch_op.drop_column("shadow_window_ended_at")
|
||||
batch_op.drop_column("shadow_window_started_at")
|
||||
batch_op.drop_column("shadow_observation_count")
|
||||
batch_op.drop_column("shadow_evidence_digest")
|
||||
batch_op.drop_column("shadow_completed_at")
|
||||
batch_op.drop_column("shadow_started_at")
|
||||
with op.batch_alter_table("knowledge_fs_migration_quarantine", schema=None) as batch_op:
|
||||
batch_op.drop_constraint(
|
||||
"kfs_migration_quarantine_resolution_fields_ck",
|
||||
type_="check",
|
||||
)
|
||||
batch_op.drop_constraint("kfs_migration_quarantine_version_ck", type_="check")
|
||||
batch_op.drop_column("row_version")
|
||||
batch_op.drop_column("evidence")
|
||||
batch_op.drop_column("resolved_by_account_id")
|
||||
batch_op.drop_column("resolved_by_operator")
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
table_name="knowledge_fs_capability_issuance_reservations",
|
||||
)
|
||||
op.drop_index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
table_name="knowledge_fs_capability_issuance_reservations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_capability_issuance_reservations")
|
||||
op.drop_index(
|
||||
"kfs_cleanup_authorization_status_idx",
|
||||
table_name="knowledge_fs_cleanup_authorizations",
|
||||
)
|
||||
op.drop_table("knowledge_fs_cleanup_authorizations")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""add durable KnowledgeFS remote freeze evidence
|
||||
|
||||
Revision ID: d4f6e8a1c305
|
||||
Revises: c8e31b7d52a4
|
||||
Create Date: 2026-07-21 15:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "d4f6e8a1c305"
|
||||
down_revision = "c8e31b7d52a4"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.add_column(sa.Column("remote_freeze_id", sa.String(length=255), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_revision", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_digest", sa.String(length=71), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_task_watermark", sa.BigInteger(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_control_space_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_frozen_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_updated_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_acknowledged_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_applied", sa.Boolean(), nullable=True))
|
||||
batch_op.add_column(sa.Column("remote_freeze_replayed", sa.Boolean(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_freeze_fields_ck",
|
||||
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
|
||||
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
|
||||
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
|
||||
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
|
||||
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
|
||||
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
|
||||
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
|
||||
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
|
||||
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_workspace_cutover_remote_freeze_time_ck",
|
||||
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_workspace_cutover_ledgers", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_workspace_cutover_remote_freeze_fields_ck", type_="check")
|
||||
batch_op.drop_column("remote_freeze_replayed")
|
||||
batch_op.drop_column("remote_freeze_applied")
|
||||
batch_op.drop_column("remote_freeze_acknowledged_at")
|
||||
batch_op.drop_column("remote_freeze_updated_at")
|
||||
batch_op.drop_column("remote_freeze_frozen_at")
|
||||
batch_op.drop_column("remote_freeze_control_space_id")
|
||||
batch_op.drop_column("remote_freeze_task_watermark")
|
||||
batch_op.drop_column("remote_freeze_digest")
|
||||
batch_op.drop_column("remote_freeze_revision")
|
||||
batch_op.drop_column("remote_freeze_id")
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
"""add global KnowledgeFS cleanup completion evidence
|
||||
|
||||
Revision ID: e5a7c9b2d416
|
||||
Revises: d4f6e8a1c305
|
||||
Create Date: 2026-07-21 16:00:00.000000
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "e5a7c9b2d416"
|
||||
down_revision = "d4f6e8a1c305"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_LEDGER_VERSIONS_CHECK = (
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
|
||||
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)"
|
||||
)
|
||||
_STATUS_FIELDS_CHECK = (
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'completed' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
|
||||
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)"
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
|
||||
batch_op.add_column(sa.Column("completed_by_account_id", models.types.StringUUID(), nullable=True))
|
||||
batch_op.add_column(sa.Column("completed_at", sa.DateTime(), nullable=True))
|
||||
batch_op.add_column(sa.Column("completion_evidence", sa.JSON(none_as_null=True), nullable=True))
|
||||
batch_op.add_column(sa.Column("completed_ledger_cas_version", sa.BigInteger(), nullable=True))
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_ledger_versions_ck",
|
||||
_LEDGER_VERSIONS_CHECK,
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_status_fields_ck",
|
||||
_STATUS_FIELDS_CHECK,
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_completion_time_ck",
|
||||
"completed_at IS NULL OR completed_at >= started_at",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("knowledge_fs_cleanup_authorizations", schema=None) as batch_op:
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_completion_time_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_status_fields_ck", type_="check")
|
||||
batch_op.drop_constraint("kfs_cleanup_authorization_ledger_versions_ck", type_="check")
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_ledger_versions_ck",
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0)",
|
||||
)
|
||||
batch_op.create_check_constraint(
|
||||
"kfs_cleanup_authorization_status_fields_ck",
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL)",
|
||||
)
|
||||
batch_op.drop_column("completed_ledger_cas_version")
|
||||
batch_op.drop_column("completion_evidence")
|
||||
batch_op.drop_column("completed_at")
|
||||
batch_op.drop_column("completed_by_account_id")
|
||||
@@ -66,6 +66,45 @@ from .enums import (
|
||||
)
|
||||
from .execution_extra_content import ExecutionExtraContent, HumanInputContent
|
||||
from .human_input import HumanInputForm, HumanInputFormUploadFile, HumanInputFormUploadToken
|
||||
from .knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSApiCredentialStatus,
|
||||
KnowledgeFSAppSpaceJoinStatus,
|
||||
KnowledgeFSAppSpaceJoinType,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSCapabilityIssuanceAudit,
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSCapabilityIssuanceReservationStatus,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpacePermissionStatus,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSControlSpaceVisibility,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSLifecycleOperation,
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSLifecycleOutboxStatus,
|
||||
)
|
||||
from .knowledge_fs_cleanup import (
|
||||
KnowledgeFSCleanupAuthorization,
|
||||
KnowledgeFSCleanupAuthorizationStatus,
|
||||
KnowledgeFSCleanupTarget,
|
||||
)
|
||||
from .knowledge_fs_cutover import (
|
||||
KnowledgeFSMigrationIssue,
|
||||
KnowledgeFSMigrationIssueKind,
|
||||
KnowledgeFSMigrationIssueStatus,
|
||||
KnowledgeFSMigrationQuarantine,
|
||||
KnowledgeFSMigrationQuarantineDisposition,
|
||||
KnowledgeFSMigrationQuarantineKind,
|
||||
KnowledgeFSShadowAuthorizationDecision,
|
||||
KnowledgeFSShadowAuthorizationDiff,
|
||||
KnowledgeFSShadowAuthorizationObservation,
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
)
|
||||
from .model import (
|
||||
AccountTrialAppRecord,
|
||||
ApiRequest,
|
||||
@@ -180,6 +219,7 @@ __all__ = [
|
||||
"AppAnnotationHitHistory",
|
||||
"AppAnnotationSetting",
|
||||
"AppDatasetJoin",
|
||||
"AppKnowledgeFSSpaceJoin",
|
||||
"AppMCPServer",
|
||||
"AppMode",
|
||||
"AppModelConfig",
|
||||
@@ -224,6 +264,38 @@ __all__ = [
|
||||
"IconType",
|
||||
"InstalledApp",
|
||||
"InvitationCode",
|
||||
"KnowledgeFSApiCredential",
|
||||
"KnowledgeFSApiCredentialStatus",
|
||||
"KnowledgeFSAppSpaceJoinStatus",
|
||||
"KnowledgeFSAppSpaceJoinType",
|
||||
"KnowledgeFSAuthorizationRevision",
|
||||
"KnowledgeFSCapabilityIssuanceAudit",
|
||||
"KnowledgeFSCapabilityIssuanceReservation",
|
||||
"KnowledgeFSCapabilityIssuanceReservationStatus",
|
||||
"KnowledgeFSCleanupAuthorization",
|
||||
"KnowledgeFSCleanupAuthorizationStatus",
|
||||
"KnowledgeFSCleanupTarget",
|
||||
"KnowledgeFSControlSpace",
|
||||
"KnowledgeFSControlSpacePermission",
|
||||
"KnowledgeFSControlSpacePermissionRole",
|
||||
"KnowledgeFSControlSpacePermissionStatus",
|
||||
"KnowledgeFSControlSpaceState",
|
||||
"KnowledgeFSControlSpaceVisibility",
|
||||
"KnowledgeFSExternalAccessPolicy",
|
||||
"KnowledgeFSLifecycleOperation",
|
||||
"KnowledgeFSLifecycleOutbox",
|
||||
"KnowledgeFSLifecycleOutboxStatus",
|
||||
"KnowledgeFSMigrationIssue",
|
||||
"KnowledgeFSMigrationIssueKind",
|
||||
"KnowledgeFSMigrationIssueStatus",
|
||||
"KnowledgeFSMigrationQuarantine",
|
||||
"KnowledgeFSMigrationQuarantineDisposition",
|
||||
"KnowledgeFSMigrationQuarantineKind",
|
||||
"KnowledgeFSShadowAuthorizationDecision",
|
||||
"KnowledgeFSShadowAuthorizationDiff",
|
||||
"KnowledgeFSShadowAuthorizationObservation",
|
||||
"KnowledgeFSWorkspaceCutoverLedger",
|
||||
"KnowledgeFSWorkspaceCutoverPhase",
|
||||
"LoadBalancingModelConfig",
|
||||
"Message",
|
||||
"MessageAgentThought",
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
"""Dify-owned control-plane persistence for the KnowledgeFS product.
|
||||
|
||||
These models intentionally have no dependency on Dify Dataset or Document
|
||||
models. KnowledgeFS data-plane objects remain owned by KnowledgeFS; these rows
|
||||
only describe Dify authorization, registration, and durable lifecycle intent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import NotRequired, TypedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, LongText, StringUUID
|
||||
|
||||
KnowledgeFSAllowedActions = list[str]
|
||||
|
||||
|
||||
class KnowledgeFSCommandEnvelope(TypedDict):
|
||||
schema_version: int
|
||||
idempotency_key: str
|
||||
expected_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSModelSelectionIntentPayload(TypedDict):
|
||||
pluginId: str
|
||||
provider: str
|
||||
model: str
|
||||
|
||||
|
||||
class KnowledgeFSRerankIntentPayload(TypedDict):
|
||||
enabled: bool
|
||||
model: NotRequired[KnowledgeFSModelSelectionIntentPayload]
|
||||
|
||||
|
||||
class KnowledgeFSScoreThresholdIntentPayload(TypedDict):
|
||||
enabled: bool
|
||||
stage: str
|
||||
value: NotRequired[float]
|
||||
|
||||
|
||||
class KnowledgeFSRetrievalProfileIntentPayload(TypedDict):
|
||||
defaultMode: str
|
||||
reasoningModel: KnowledgeFSModelSelectionIntentPayload
|
||||
rerank: KnowledgeFSRerankIntentPayload
|
||||
scoreThreshold: KnowledgeFSScoreThresholdIntentPayload
|
||||
topK: int
|
||||
|
||||
|
||||
class KnowledgeFSProvisionCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
provisioning_key: str
|
||||
name: str
|
||||
icon: str | None
|
||||
description: str | None
|
||||
slug: str
|
||||
model_intent: KnowledgeFSModelSelectionIntentPayload
|
||||
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
|
||||
|
||||
|
||||
class KnowledgeFSMetadataUpdateCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
metadata: dict[str, object]
|
||||
|
||||
|
||||
class KnowledgeFSDeleteCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
knowledge_space_id: str | None
|
||||
provisioning_key: str
|
||||
|
||||
|
||||
class KnowledgeFSRevokeCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
event_id: str
|
||||
grant_id: str
|
||||
knowledge_space_id: str
|
||||
principal: str
|
||||
reason_code: str
|
||||
revoke_sequence: int
|
||||
|
||||
|
||||
class KnowledgeFSRepairCommandPayload(KnowledgeFSCommandEnvelope):
|
||||
repair_reason: str
|
||||
|
||||
|
||||
KnowledgeFSCommandPayload = (
|
||||
KnowledgeFSProvisionCommandPayload
|
||||
| KnowledgeFSMetadataUpdateCommandPayload
|
||||
| KnowledgeFSDeleteCommandPayload
|
||||
| KnowledgeFSRevokeCommandPayload
|
||||
| KnowledgeFSRepairCommandPayload
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityAuthzRevisionSummary(TypedDict):
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
credential_revision: int | None
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityClaimsSummary(TypedDict):
|
||||
action: str
|
||||
actor: str
|
||||
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
|
||||
caller_kind: str
|
||||
content_policy_revision: int
|
||||
content_scope_ids: list[str]
|
||||
control_space_id: str
|
||||
expires_at: str
|
||||
grant_id: str
|
||||
issued_at: str
|
||||
namespace_id: str
|
||||
operation_id: str
|
||||
resource_id: str
|
||||
resource_parent_id: str | None
|
||||
resource_type: str
|
||||
subject: str
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityReservationSummary(TypedDict):
|
||||
"""Sanitized, exact input bound before a capability can be signed."""
|
||||
|
||||
action: str
|
||||
actor: str
|
||||
authz_revision: KnowledgeFSCapabilityAuthzRevisionSummary
|
||||
caller_kind: str
|
||||
content_policy_revision: int
|
||||
content_scope_ids: list[str]
|
||||
control_space_id: str
|
||||
grant_id: str
|
||||
namespace_id: str
|
||||
operation_id: str
|
||||
resource_id: str
|
||||
resource_parent_id: str | None
|
||||
resource_type: str
|
||||
subject: str
|
||||
trace_id: str
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceVisibility(StrEnum):
|
||||
ONLY_ME = "only_me"
|
||||
ALL_TEAM_MEMBERS = "all_team_members"
|
||||
PARTIAL_MEMBERS = "partial_members"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceState(StrEnum):
|
||||
PROVISIONING = "provisioning"
|
||||
ACTIVE = "active"
|
||||
DELETING = "deleting"
|
||||
DELETED = "deleted"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermissionRole(StrEnum):
|
||||
OWNER = "owner"
|
||||
EDITOR = "editor"
|
||||
VIEWER = "viewer"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermissionStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class KnowledgeFSApiCredentialStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
class KnowledgeFSAppSpaceJoinType(StrEnum):
|
||||
AGENT = "agent"
|
||||
WORKFLOW = "workflow"
|
||||
|
||||
|
||||
class KnowledgeFSAppSpaceJoinStatus(StrEnum):
|
||||
ACTIVE = "active"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservationStatus(StrEnum):
|
||||
RESERVED = "reserved"
|
||||
ISSUED = "issued"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOperation(StrEnum):
|
||||
PROVISION = "provision"
|
||||
METADATA_UPDATE = "metadata_update"
|
||||
DELETE = "delete"
|
||||
REVOKE = "revoke"
|
||||
REPAIR = "repair"
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutboxStatus(StrEnum):
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
SUCCEEDED = "succeeded"
|
||||
RETRY = "retry"
|
||||
DEAD_LETTER = "dead_letter"
|
||||
|
||||
|
||||
class KnowledgeFSControlSpace(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Dify product resource registered to at most one KnowledgeFS Space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_control_spaces"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_pkey"),
|
||||
UniqueConstraint("tenant_id", "id", name="kfs_control_space_tenant_id_uq"),
|
||||
UniqueConstraint("provisioning_key", name="kfs_control_space_provisioning_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id"],
|
||||
["tenants.id"],
|
||||
name="kfs_control_space_workspace_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_control_space_tenant_space_uq",
|
||||
"tenant_id",
|
||||
"knowledge_space_id",
|
||||
unique=True,
|
||||
postgresql_where=sa.text("knowledge_space_id IS NOT NULL"),
|
||||
),
|
||||
Index("kfs_control_space_state_updated_idx", "state", "updated_at"),
|
||||
Index("kfs_control_space_tenant_state_updated_idx", "tenant_id", "state", "updated_at"),
|
||||
Index("kfs_control_space_tenant_owner_state_idx", "tenant_id", "owner_account_id", "state"),
|
||||
sa.CheckConstraint(
|
||||
"resource_version >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_resource_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint("attempt_count >= 0", name=sa.schema.conv("kfs_control_space_attempt_count_ck")),
|
||||
sa.CheckConstraint(
|
||||
"state != 'active' OR knowledge_space_id IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_control_space_active_registration_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"deletion_irreversible_at IS NULL OR state IN ('deleting', 'deleted', 'error')",
|
||||
name=sa.schema.conv("kfs_control_space_irreversible_state_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"knowledge_space_revision >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_remote_revision_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
owner_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
provisioning_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
knowledge_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
knowledge_space_revision: Mapped[int] = mapped_column(
|
||||
sa.BigInteger,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
visibility: Mapped[KnowledgeFSControlSpaceVisibility] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpaceVisibility, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'only_me'"),
|
||||
default=KnowledgeFSControlSpaceVisibility.ONLY_ME,
|
||||
)
|
||||
lifecycle_operation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
state: Mapped[KnowledgeFSControlSpaceState] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpaceState, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'provisioning'"),
|
||||
default=KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
)
|
||||
resource_version: Mapped[int] = mapped_column(
|
||||
sa.BigInteger,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(
|
||||
sa.Integer,
|
||||
nullable=False,
|
||||
server_default=sa.text("0"),
|
||||
default=0,
|
||||
)
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
||||
last_synced_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
deletion_irreversible_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSControlSpacePermission(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Account-level product authorization for one control-space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_control_space_permissions"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_control_space_permission_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"account_id",
|
||||
name="kfs_control_space_permission_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_control_space_permission_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_control_space_permission_account_idx", "tenant_id", "account_id", "status"),
|
||||
sa.CheckConstraint(
|
||||
"revision >= 0",
|
||||
name=sa.schema.conv("kfs_control_space_permission_revision_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
role: Mapped[KnowledgeFSControlSpacePermissionRole] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpacePermissionRole, length=32), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSControlSpacePermissionStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSControlSpacePermissionStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
granted_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSExternalAccessPolicy(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Fail-closed caller-channel policy for a control-space."""
|
||||
|
||||
__tablename__ = "knowledge_fs_external_access_policies"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_external_access_policy_pkey"),
|
||||
UniqueConstraint("tenant_id", "control_space_id", name="kfs_external_access_policy_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_external_access_policy_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_external_access_policy_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
service_api_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
agent_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
workflow_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
mcp_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
updated_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSApiCredential(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Hashed, resource-bound KnowledgeFS API credential metadata."""
|
||||
|
||||
__tablename__ = "knowledge_fs_api_credentials"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_api_credential_pkey"),
|
||||
UniqueConstraint("credential_hash", name="kfs_api_credential_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_api_credential_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_api_credential_tenant_space_status_idx", "tenant_id", "control_space_id", "status"),
|
||||
Index("kfs_api_credential_tenant_prefix_idx", "tenant_id", "credential_prefix"),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("kfs_api_credential_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
credential_hash: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
credential_prefix: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
credential_last4: Mapped[str] = mapped_column(String(4), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
allowed_actions: Mapped[KnowledgeFSAllowedActions] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSApiCredentialStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSApiCredentialStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSApiCredentialStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_used_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoke_reason: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
|
||||
|
||||
class AppKnowledgeFSSpaceJoin(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Explicit Agent or Workflow app authorization for a control-space."""
|
||||
|
||||
__tablename__ = "app_knowledge_fs_space_joins"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="app_kfs_space_join_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"app_id",
|
||||
"control_space_id",
|
||||
"join_type",
|
||||
name="app_kfs_space_join_identity_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="app_kfs_space_join_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("app_kfs_space_join_app_status_idx", "tenant_id", "app_id", "status"),
|
||||
sa.CheckConstraint("revision >= 0", name=sa.schema.conv("app_kfs_space_join_revision_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
join_type: Mapped[KnowledgeFSAppSpaceJoinType] = mapped_column(
|
||||
EnumText(KnowledgeFSAppSpaceJoinType, length=32), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSAppSpaceJoinStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSAppSpaceJoinStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'active'"),
|
||||
default=KnowledgeFSAppSpaceJoinStatus.ACTIVE,
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
created_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
revoked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
revoked_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSAuthorizationRevision(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Monotonic authorization epochs included in future Capability claims."""
|
||||
|
||||
__tablename__ = "knowledge_fs_authorization_revisions"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_authorization_revision_pkey"),
|
||||
UniqueConstraint("tenant_id", "control_space_id", name="kfs_authorization_revision_space_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_authorization_revision_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"membership_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_membership_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"space_acl_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_space_acl_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"external_access_epoch >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_external_access_epoch_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"content_policy_revision >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_content_policy_revision_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"revoke_sequence >= 0",
|
||||
name=sa.schema.conv("kfs_authorization_revoke_sequence_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
membership_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
space_acl_epoch: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
external_access_epoch: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
content_policy_revision: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
revoke_sequence: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceAudit(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Sanitized durable evidence for a Capability v2 issuance."""
|
||||
|
||||
__tablename__ = "knowledge_fs_capability_issuance_audits"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_audit_pkey"),
|
||||
UniqueConstraint("jti_hash", name="kfs_capability_issuance_audit_jti_hash_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_audit_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_audit_space_created_idx",
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"created_at",
|
||||
),
|
||||
Index("kfs_capability_issuance_audit_trace_idx", "tenant_id", "trace_id"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
jti_hash: Mapped[str] = mapped_column(String(80), nullable=False)
|
||||
claims_summary: Mapped[KnowledgeFSCapabilityClaimsSummary] = mapped_column(sa.JSON, nullable=False)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservation(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Durable authorization fence written before signing starts.
|
||||
|
||||
The summary intentionally excludes bearer tokens, signing material, and raw JTI values.
|
||||
Revocation producers scan these rows as well as completed issuance audits, closing the
|
||||
authorization-to-signing race without holding a database transaction across signing.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_fs_capability_issuance_reservations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_capability_issuance_reservation_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"grant_id",
|
||||
name="kfs_capability_issuance_reservation_grant_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_capability_issuance_reservation_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_reservation_subject_idx",
|
||||
"tenant_id",
|
||||
"control_space_id",
|
||||
"subject",
|
||||
"caller_kind",
|
||||
),
|
||||
Index(
|
||||
"kfs_capability_issuance_reservation_trace_idx",
|
||||
"tenant_id",
|
||||
"trace_id",
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"row_version >= 0",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'reserved' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NULL) OR "
|
||||
"(status = 'issued' AND issued_at IS NOT NULL AND token_expires_at IS NOT NULL "
|
||||
"AND failed_at IS NULL AND failure_code IS NULL AND cleanup_after IS NOT NULL) OR "
|
||||
"(status = 'failed' AND issued_at IS NULL AND token_expires_at IS NULL "
|
||||
"AND failed_at IS NOT NULL AND failure_code IS NOT NULL AND cleanup_after IS NOT NULL)",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_status_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"token_expires_at IS NULL OR cleanup_after >= token_expires_at",
|
||||
name=sa.schema.conv("kfs_capability_issuance_reservation_cleanup_window_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
grant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
trace_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
subject: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
caller_kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
request_summary: Mapped[KnowledgeFSCapabilityReservationSummary] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSCapabilityIssuanceReservationStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSCapabilityIssuanceReservationStatus, length=16),
|
||||
nullable=False,
|
||||
server_default=sa.text("'reserved'"),
|
||||
default=KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
|
||||
)
|
||||
issued_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
token_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
failed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
failure_code: Mapped[str | None] = mapped_column(String(128), nullable=True, default=None)
|
||||
cleanup_after: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutbox(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Durable command snapshot; it is never a product read-model source.
|
||||
|
||||
Provision payload writers must snapshot name, icon, description, slug,
|
||||
model/profile intent, schema version, idempotency key, and expected
|
||||
revision before enabling the later P1B dispatcher.
|
||||
"""
|
||||
|
||||
__tablename__ = "knowledge_fs_lifecycle_outbox"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_lifecycle_outbox_pkey"),
|
||||
UniqueConstraint("tenant_id", "operation_id", name="kfs_lifecycle_outbox_operation_uq"),
|
||||
UniqueConstraint("tenant_id", "idempotency_key", name="kfs_lifecycle_outbox_idempotency_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "control_space_id"],
|
||||
["knowledge_fs_control_spaces.tenant_id", "knowledge_fs_control_spaces.id"],
|
||||
name="kfs_lifecycle_outbox_space_fk",
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
Index("kfs_lifecycle_outbox_dispatch_idx", "status", "next_attempt_at", "id"),
|
||||
Index("kfs_lifecycle_outbox_space_created_idx", "tenant_id", "control_space_id", "created_at"),
|
||||
sa.CheckConstraint(
|
||||
"command_schema_version >= 1",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_schema_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"expected_control_space_version >= 0",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_expected_version_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"attempt_count >= 0",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_attempt_count_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"status IN ('pending', 'processing', 'succeeded', 'retry', 'dead_letter')",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_status_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'processing' AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL) "
|
||||
"OR (status != 'processing' AND lease_owner IS NULL AND lease_expires_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_lease_state_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status IN ('succeeded', 'dead_letter') AND completed_at IS NOT NULL) "
|
||||
"OR (status NOT IN ('succeeded', 'dead_letter') AND completed_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_lifecycle_outbox_terminal_state_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
control_space_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
operation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
operation: Mapped[KnowledgeFSLifecycleOperation] = mapped_column(
|
||||
EnumText(KnowledgeFSLifecycleOperation, length=32), nullable=False
|
||||
)
|
||||
command_payload: Mapped[KnowledgeFSCommandPayload] = mapped_column(sa.JSON, nullable=False)
|
||||
expected_control_space_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
|
||||
expected_knowledge_space_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
command_schema_version: Mapped[int] = mapped_column(
|
||||
sa.Integer, nullable=False, server_default=sa.text("1"), default=1
|
||||
)
|
||||
status: Mapped[KnowledgeFSLifecycleOutboxStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSLifecycleOutboxStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'pending'"),
|
||||
default=KnowledgeFSLifecycleOutboxStatus.PENDING,
|
||||
)
|
||||
attempt_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"), default=0)
|
||||
next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
last_error_code: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
last_error_message: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
retain_until: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppKnowledgeFSSpaceJoin",
|
||||
"KnowledgeFSAllowedActions",
|
||||
"KnowledgeFSApiCredential",
|
||||
"KnowledgeFSApiCredentialStatus",
|
||||
"KnowledgeFSAppSpaceJoinStatus",
|
||||
"KnowledgeFSAppSpaceJoinType",
|
||||
"KnowledgeFSAuthorizationRevision",
|
||||
"KnowledgeFSCapabilityAuthzRevisionSummary",
|
||||
"KnowledgeFSCapabilityClaimsSummary",
|
||||
"KnowledgeFSCapabilityIssuanceAudit",
|
||||
"KnowledgeFSCapabilityIssuanceReservation",
|
||||
"KnowledgeFSCapabilityIssuanceReservationStatus",
|
||||
"KnowledgeFSCapabilityReservationSummary",
|
||||
"KnowledgeFSCommandEnvelope",
|
||||
"KnowledgeFSCommandPayload",
|
||||
"KnowledgeFSControlSpace",
|
||||
"KnowledgeFSControlSpacePermission",
|
||||
"KnowledgeFSControlSpacePermissionRole",
|
||||
"KnowledgeFSControlSpacePermissionStatus",
|
||||
"KnowledgeFSControlSpaceState",
|
||||
"KnowledgeFSControlSpaceVisibility",
|
||||
"KnowledgeFSDeleteCommandPayload",
|
||||
"KnowledgeFSExternalAccessPolicy",
|
||||
"KnowledgeFSLifecycleOperation",
|
||||
"KnowledgeFSLifecycleOutbox",
|
||||
"KnowledgeFSLifecycleOutboxStatus",
|
||||
"KnowledgeFSMetadataUpdateCommandPayload",
|
||||
"KnowledgeFSModelSelectionIntentPayload",
|
||||
"KnowledgeFSProvisionCommandPayload",
|
||||
"KnowledgeFSRepairCommandPayload",
|
||||
"KnowledgeFSRerankIntentPayload",
|
||||
"KnowledgeFSRetrievalProfileIntentPayload",
|
||||
"KnowledgeFSRevokeCommandPayload",
|
||||
"KnowledgeFSScoreThresholdIntentPayload",
|
||||
]
|
||||
@@ -0,0 +1,130 @@
|
||||
"""Persistent authorization for the irreversible P9 KnowledgeFS cleanup window."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, StringUUID
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorizationStatus(StrEnum):
|
||||
REQUESTED = "requested"
|
||||
APPROVED = "approved"
|
||||
STARTED = "started"
|
||||
COMPLETED = "completed"
|
||||
|
||||
|
||||
class KnowledgeFSCleanupTarget(StrEnum):
|
||||
LEGACY_SNAPSHOT_FOREIGN_KEYS = "legacy_snapshot_foreign_keys"
|
||||
LEGACY_ACL_ROUTES = "legacy_acl_routes"
|
||||
LEGACY_ACL_SCHEMA = "legacy_acl_schema"
|
||||
LEGACY_API_KEY_SCHEMA = "legacy_api_key_schema"
|
||||
LEGACY_V1_AUTH = "legacy_v1_auth"
|
||||
RAW_LIST_CREATE_PROXY = "raw_list_create_proxy"
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorization(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Four-eyes approval and immutable evidence for one cleanup start fence."""
|
||||
|
||||
__tablename__ = "knowledge_fs_cleanup_authorizations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_cleanup_authorization_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "request_id", name="kfs_cleanup_authorization_request_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_cleanup_authorization_ledger_fk",
|
||||
),
|
||||
Index("kfs_cleanup_authorization_status_idx", "tenant_id", "status", "updated_at"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_cleanup_authorization_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"readiness_ledger_cas_version >= 0 "
|
||||
"AND (approved_ledger_cas_version IS NULL OR approved_ledger_cas_version >= 0) "
|
||||
"AND (started_ledger_cas_version IS NULL OR started_ledger_cas_version >= 0) "
|
||||
"AND (completed_ledger_cas_version IS NULL OR completed_ledger_cas_version >= 0)",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_ledger_versions_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(status = 'requested' "
|
||||
"AND approved_by_account_id IS NULL AND approved_at IS NULL "
|
||||
"AND approval_expires_at IS NULL AND approved_ledger_cas_version IS NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'approved' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NULL AND started_at IS NULL "
|
||||
"AND started_ledger_cas_version IS NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'started' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NULL AND completed_at IS NULL "
|
||||
"AND completion_evidence IS NULL AND completed_ledger_cas_version IS NULL) OR "
|
||||
"(status = 'completed' "
|
||||
"AND approved_by_account_id IS NOT NULL AND approved_at IS NOT NULL "
|
||||
"AND approval_expires_at IS NOT NULL AND approved_ledger_cas_version IS NOT NULL "
|
||||
"AND started_by_account_id IS NOT NULL AND started_at IS NOT NULL "
|
||||
"AND started_ledger_cas_version IS NOT NULL "
|
||||
"AND completed_by_account_id IS NOT NULL AND completed_at IS NOT NULL "
|
||||
"AND completion_evidence IS NOT NULL AND completed_ledger_cas_version IS NOT NULL)",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_status_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"approval_expires_at IS NULL OR approval_expires_at > approved_at",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_approval_window_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"completed_at IS NULL OR completed_at >= started_at",
|
||||
name=sa.schema.conv("kfs_cleanup_authorization_completion_time_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
request_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
plan_schema_version: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
plan_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
targets: Mapped[list[str]] = mapped_column(sa.JSON, nullable=False)
|
||||
readiness_evidence: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False)
|
||||
requested_by_account_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
requested_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
readiness_ledger_cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False)
|
||||
status: Mapped[KnowledgeFSCleanupAuthorizationStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSCleanupAuthorizationStatus, length=16),
|
||||
nullable=False,
|
||||
server_default=sa.text("'requested'"),
|
||||
default=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
|
||||
)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
approval_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
approved_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
started_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
started_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
completion_evidence: Mapped[dict[str, object] | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
completed_ledger_cas_version: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCleanupAuthorization",
|
||||
"KnowledgeFSCleanupAuthorizationStatus",
|
||||
"KnowledgeFSCleanupTarget",
|
||||
]
|
||||
@@ -0,0 +1,507 @@
|
||||
"""Workspace-scoped KnowledgeFS migration and cutover persistence.
|
||||
|
||||
The ledger is Dify control-plane state only. It deliberately stores opaque
|
||||
KnowledgeFS identifiers and authorization watermarks without foreign keys or
|
||||
queries to legacy product tables. Cutover feature switches live on one
|
||||
row so the repository can change them with a single CAS statement. Quarantine
|
||||
remediation is separately CAS-versioned and retains immutable operator evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import TypedDict
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import DateTime, Index, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import DefaultFieldsDCMixin, TypeBase
|
||||
from .types import EnumText, LongText, StringUUID
|
||||
|
||||
|
||||
class KnowledgeFSCutoverRevisionWatermark(TypedDict):
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
content_policy_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeChecks(TypedDict):
|
||||
authorization: bool
|
||||
list_spaces: bool
|
||||
create_space: bool
|
||||
query: bool
|
||||
upload: bool
|
||||
stream: bool
|
||||
deletion: bool
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeEvidenceReferences(TypedDict):
|
||||
authorization: str
|
||||
list_spaces: str
|
||||
create_space: str
|
||||
query: str
|
||||
upload: str
|
||||
stream: str
|
||||
deletion: str
|
||||
|
||||
|
||||
class KnowledgeFSCutoverSmokeResults(TypedDict):
|
||||
schema_version: str
|
||||
tenant_id: str
|
||||
environment: str
|
||||
operator: str
|
||||
operator_account_id: str
|
||||
observed_at: str
|
||||
checks: KnowledgeFSCutoverSmokeChecks
|
||||
evidence_references: KnowledgeFSCutoverSmokeEvidenceReferences
|
||||
|
||||
|
||||
def knowledge_fs_cutover_smoke_results_passed(value: KnowledgeFSCutoverSmokeResults | None) -> bool:
|
||||
"""Reject incomplete or pre-production JSON evidence instead of treating metadata as booleans."""
|
||||
|
||||
if not isinstance(value, dict):
|
||||
return False
|
||||
checks = value.get("checks")
|
||||
references = value.get("evidence_references")
|
||||
expected = {
|
||||
"authorization",
|
||||
"list_spaces",
|
||||
"create_space",
|
||||
"query",
|
||||
"upload",
|
||||
"stream",
|
||||
"deletion",
|
||||
}
|
||||
return bool(
|
||||
value.get("schema_version") == "knowledge-fs-p8-cutover-smoke/v1"
|
||||
and value.get("environment") == "production"
|
||||
and isinstance(checks, dict)
|
||||
and set(checks) == expected
|
||||
and all(checks.get(name) is True for name in expected)
|
||||
and isinstance(references, dict)
|
||||
and set(references) == expected
|
||||
and all(isinstance(reference := references.get(name), str) and bool(reference) for name in expected)
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceCutoverPhase(StrEnum):
|
||||
INVENTORY = "inventory"
|
||||
BACKFILL = "backfill"
|
||||
SHADOW = "shadow"
|
||||
FROZEN = "frozen"
|
||||
CUTOVER = "cutover"
|
||||
OBSERVING = "observing"
|
||||
READY_FOR_CLEANUP = "ready_for_cleanup"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssueStatus(StrEnum):
|
||||
OPEN = "open"
|
||||
APPROVED_FAIL_CLOSED = "approved_fail_closed"
|
||||
RESOLVED = "resolved"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssueKind(StrEnum):
|
||||
REGISTRATION_CONFLICT = "registration_conflict"
|
||||
UNRESOLVED_SUBJECT = "unresolved_subject"
|
||||
UNKNOWN_EXTERNAL_ACCESS = "unknown_external_access"
|
||||
REVISION_DRIFT = "revision_drift"
|
||||
TASK_QUARANTINE = "task_quarantine"
|
||||
LEGACY_SNAPSHOT_DEPENDENCY = "legacy_snapshot_dependency"
|
||||
LEGACY_FOREIGN_KEY_DEPENDENCY = "legacy_foreign_key_dependency"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantineKind(StrEnum):
|
||||
CONTROL_SPACE = "control_space"
|
||||
SUBJECT = "subject"
|
||||
TASK = "task"
|
||||
LEGACY_API_KEY = "legacy_api_key"
|
||||
ORPHAN_RESOURCE = "orphan_resource"
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantineDisposition(StrEnum):
|
||||
PENDING = "pending"
|
||||
MIGRATABLE = "migratable"
|
||||
WAIT_FOR_COMPLETION = "wait_for_completion"
|
||||
CANCEL = "cancel"
|
||||
ISOLATE = "isolate"
|
||||
ROTATE_CREDENTIAL = "rotate_credential"
|
||||
RESOLVED = "resolved"
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationDecision(StrEnum):
|
||||
MATCH = "match"
|
||||
TIGHTENED = "tightened"
|
||||
EXPANDED = "expanded"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceCutoverLedger(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Single-Workspace migration state, watermarks, switches, and time fences."""
|
||||
|
||||
__tablename__ = "knowledge_fs_workspace_cutover_ledgers"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_workspace_cutover_ledger_pkey"),
|
||||
UniqueConstraint("tenant_id", name="kfs_workspace_cutover_ledger_tenant_uq"),
|
||||
UniqueConstraint("tenant_id", "id", name="kfs_workspace_cutover_ledger_tenant_id_uq"),
|
||||
Index("kfs_workspace_cutover_ledger_phase_updated_idx", "phase", "updated_at"),
|
||||
sa.CheckConstraint("cas_version >= 0", name=sa.schema.conv("kfs_workspace_cutover_cas_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"source_task_watermark >= 0 AND applied_task_watermark >= 0 "
|
||||
"AND (final_task_watermark IS NULL OR final_task_watermark >= 0)",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_task_watermark_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"cutover_at IS NULL OR freeze_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_freeze_before_cutover_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"rollback_cutoff_at IS NULL OR cutover_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_rollback_cutoff_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"shadow_observation_count >= 0",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_count_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"shadow_window_ended_at IS NULL OR shadow_window_started_at IS NOT NULL",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_window_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(shadow_completed_at IS NULL "
|
||||
"AND shadow_evidence_digest IS NULL AND shadow_producer IS NULL "
|
||||
"AND shadow_completed_by_operator IS NULL AND shadow_completed_by_account_id IS NULL "
|
||||
"AND shadow_window_started_at IS NULL AND shadow_window_ended_at IS NULL "
|
||||
"AND shadow_traffic_zero = false AND shadow_traffic_zero_evidence IS NULL "
|
||||
"AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_completed_at IS NOT NULL AND shadow_started_at IS NOT NULL "
|
||||
"AND shadow_evidence_digest IS NOT NULL AND shadow_producer IS NOT NULL "
|
||||
"AND shadow_completed_by_operator IS NOT NULL AND shadow_completed_by_account_id IS NOT NULL "
|
||||
"AND ((shadow_traffic_zero = true AND shadow_observation_count = 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NOT NULL AND shadow_window_started_at IS NULL "
|
||||
"AND shadow_window_ended_at IS NULL AND shadow_latest_observed_revision IS NULL) OR "
|
||||
"(shadow_traffic_zero = false AND shadow_observation_count > 0 "
|
||||
"AND shadow_traffic_zero_evidence IS NULL AND shadow_window_started_at IS NOT NULL "
|
||||
"AND shadow_window_ended_at IS NOT NULL AND shadow_latest_observed_revision IS NOT NULL)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_shadow_completion_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(remote_freeze_id IS NULL AND remote_freeze_revision IS NULL "
|
||||
"AND remote_freeze_digest IS NULL AND remote_freeze_task_watermark IS NULL "
|
||||
"AND remote_freeze_control_space_id IS NULL AND remote_freeze_frozen_at IS NULL "
|
||||
"AND remote_freeze_updated_at IS NULL AND remote_freeze_acknowledged_at IS NULL "
|
||||
"AND remote_freeze_applied IS NULL AND remote_freeze_replayed IS NULL) OR "
|
||||
"(remote_freeze_id IS NOT NULL AND remote_freeze_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_freeze_digest IS NOT NULL AND remote_freeze_task_watermark >= 0 "
|
||||
"AND remote_freeze_control_space_id IS NOT NULL AND remote_freeze_frozen_at IS NOT NULL "
|
||||
"AND remote_freeze_updated_at IS NOT NULL AND remote_freeze_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_freeze_applied = true AND remote_freeze_replayed = false) "
|
||||
"OR (remote_freeze_applied = false AND remote_freeze_replayed = true)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"remote_freeze_updated_at IS NULL OR remote_freeze_updated_at >= remote_freeze_frozen_at",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_freeze_time_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"(remote_activation_id IS NULL AND remote_activation_revision IS NULL "
|
||||
"AND remote_activation_digest IS NULL AND remote_activation_control_space_id IS NULL "
|
||||
"AND remote_activation_activated_at IS NULL AND remote_activation_updated_at IS NULL "
|
||||
"AND remote_activation_acknowledged_at IS NULL AND remote_activation_applied IS NULL "
|
||||
"AND remote_activation_replayed IS NULL) OR "
|
||||
"(remote_activation_id IS NOT NULL AND remote_activation_revision BETWEEN 1 AND 9007199254740991 "
|
||||
"AND remote_activation_digest IS NOT NULL AND remote_activation_control_space_id IS NOT NULL "
|
||||
"AND remote_activation_activated_at IS NOT NULL AND remote_activation_updated_at IS NOT NULL "
|
||||
"AND remote_activation_acknowledged_at IS NOT NULL "
|
||||
"AND ((remote_activation_applied = true AND remote_activation_replayed = false) "
|
||||
"OR (remote_activation_applied = false AND remote_activation_replayed = true)))",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_fields_ck"),
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"remote_activation_updated_at IS NULL OR remote_activation_updated_at >= remote_activation_activated_at",
|
||||
name=sa.schema.conv("kfs_workspace_cutover_remote_activation_time_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
source_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
applied_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
phase: Mapped[KnowledgeFSWorkspaceCutoverPhase] = mapped_column(
|
||||
EnumText(KnowledgeFSWorkspaceCutoverPhase, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'inventory'"),
|
||||
default=KnowledgeFSWorkspaceCutoverPhase.INVENTORY,
|
||||
)
|
||||
final_revision_watermark: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
|
||||
sa.JSON, nullable=True, default=None
|
||||
)
|
||||
source_task_watermark: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
final_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
applied_task_watermark: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
shadow_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_evidence_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
shadow_observation_count: Mapped[int] = mapped_column(
|
||||
sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0
|
||||
)
|
||||
shadow_window_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_window_ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
shadow_traffic_zero: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
shadow_traffic_zero_evidence: Mapped[dict[str, object] | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
shadow_latest_observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark | None] = mapped_column(
|
||||
sa.JSON(none_as_null=True), nullable=True, default=None
|
||||
)
|
||||
shadow_producer: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
shadow_completed_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
shadow_completed_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_freeze_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
remote_freeze_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_freeze_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
remote_freeze_task_watermark: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_freeze_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_freeze_frozen_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_freeze_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_freeze_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_activation_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
remote_activation_revision: Mapped[int | None] = mapped_column(sa.BigInteger, nullable=True, default=None)
|
||||
remote_activation_digest: Mapped[str | None] = mapped_column(String(71), nullable=True, default=None)
|
||||
remote_activation_control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
remote_activation_activated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
remote_activation_applied: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
remote_activation_replayed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
freeze_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
cutover_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
rollback_cutoff_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_window_ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
observation_completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
maximum_task_expires_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
irreversible_cleanup_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
product_routes_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
capability_v2_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
integrated_mode_enabled: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
legacy_acl_read_only: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
smoke_results: Mapped[KnowledgeFSCutoverSmokeResults | None] = mapped_column(sa.JSON, nullable=True, default=None)
|
||||
legacy_dependency_report: Mapped[list[dict[str, object]] | None] = mapped_column(
|
||||
sa.JSON, nullable=True, default=None
|
||||
)
|
||||
legacy_dependency_checked_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
legacy_dependency_ready: Mapped[bool] = mapped_column(
|
||||
sa.Boolean, nullable=False, server_default=sa.text("false"), default=False
|
||||
)
|
||||
cas_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSMigrationIssue(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Auditable blocker; unknown access can only be approved as fail-closed."""
|
||||
|
||||
__tablename__ = "knowledge_fs_migration_issues"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_issue_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "issue_key", name="kfs_migration_issue_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_issue_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_migration_issue_gate_idx", "tenant_id", "ledger_id", "status", "kind"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
issue_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
kind: Mapped[KnowledgeFSMigrationIssueKind] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueKind, length=48), nullable=False
|
||||
)
|
||||
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueStatus, length=32),
|
||||
nullable=False,
|
||||
server_default=sa.text("'open'"),
|
||||
default=KnowledgeFSMigrationIssueStatus.OPEN,
|
||||
)
|
||||
resource_type: Mapped[str | None] = mapped_column(String(64), nullable=True, default=None)
|
||||
resource_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
|
||||
|
||||
class KnowledgeFSMigrationQuarantine(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Fail-closed inventory item whose operator resolution is immutable and CAS-versioned."""
|
||||
|
||||
__tablename__ = "knowledge_fs_migration_quarantine"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_migration_quarantine_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"source_kind",
|
||||
"source_id",
|
||||
name="kfs_migration_quarantine_source_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_migration_quarantine_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_migration_quarantine_disposition_idx", "tenant_id", "ledger_id", "disposition"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_migration_quarantine_version_ck")),
|
||||
sa.CheckConstraint(
|
||||
"(disposition = 'resolved' "
|
||||
"AND resolved_by_operator IS NOT NULL AND resolved_by_account_id IS NOT NULL "
|
||||
"AND evidence IS NOT NULL AND resolved_at IS NOT NULL) OR "
|
||||
"(disposition <> 'resolved' "
|
||||
"AND resolved_by_operator IS NULL AND resolved_by_account_id IS NULL "
|
||||
"AND evidence IS NULL AND resolved_at IS NULL)",
|
||||
name=sa.schema.conv("kfs_migration_quarantine_resolution_fields_ck"),
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
source_kind: Mapped[KnowledgeFSMigrationQuarantineKind] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationQuarantineKind, length=32), nullable=False
|
||||
)
|
||||
source_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
reason_code: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
disposition: Mapped[KnowledgeFSMigrationQuarantineDisposition] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationQuarantineDisposition, length=32), nullable=False
|
||||
)
|
||||
details: Mapped[dict[str, object]] = mapped_column(sa.JSON, nullable=False, default_factory=dict)
|
||||
resolved_by_operator: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
evidence: Mapped[dict[str, object] | None] = mapped_column(sa.JSON(none_as_null=True), nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationDiff(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Current disposition for a diff key; append-only observations retain its history."""
|
||||
|
||||
__tablename__ = "knowledge_fs_shadow_authorization_diffs"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_diff_pkey"),
|
||||
UniqueConstraint("tenant_id", "ledger_id", "diff_key", name="kfs_shadow_authorization_diff_key_uq"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_diff_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index("kfs_shadow_authorization_diff_gate_idx", "tenant_id", "ledger_id", "status", "decision"),
|
||||
sa.CheckConstraint("row_version >= 0", name=sa.schema.conv("kfs_shadow_authorization_diff_version_ck")),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
|
||||
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
|
||||
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(LongText, nullable=False)
|
||||
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
status: Mapped[KnowledgeFSMigrationIssueStatus] = mapped_column(
|
||||
EnumText(KnowledgeFSMigrationIssueStatus, length=32), nullable=False
|
||||
)
|
||||
current_evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
last_observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
approved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
approved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
resolved_by_account_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
||||
row_version: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("0"), default=0)
|
||||
|
||||
|
||||
class KnowledgeFSShadowAuthorizationObservation(DefaultFieldsDCMixin, TypeBase):
|
||||
"""Append-only producer evidence used to derive a shadow diff disposition."""
|
||||
|
||||
__tablename__ = "knowledge_fs_shadow_authorization_observations"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="kfs_shadow_authorization_observation_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"diff_key",
|
||||
"evidence_digest",
|
||||
name="kfs_shadow_authorization_observation_evidence_uq",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tenant_id", "ledger_id"],
|
||||
["knowledge_fs_workspace_cutover_ledgers.tenant_id", "knowledge_fs_workspace_cutover_ledgers.id"],
|
||||
name="kfs_shadow_authorization_observation_ledger_fk",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
Index(
|
||||
"kfs_shadow_authorization_observation_window_idx",
|
||||
"tenant_id",
|
||||
"ledger_id",
|
||||
"observed_at",
|
||||
),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
ledger_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
diff_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
producer: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
principal: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
dify_allowed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False)
|
||||
decision: Mapped[KnowledgeFSShadowAuthorizationDecision] = mapped_column(
|
||||
EnumText(KnowledgeFSShadowAuthorizationDecision, length=32), nullable=False
|
||||
)
|
||||
reason: Mapped[str] = mapped_column(LongText, nullable=False)
|
||||
observed_revision: Mapped[KnowledgeFSCutoverRevisionWatermark] = mapped_column(sa.JSON, nullable=False)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
|
||||
evidence_digest: Mapped[str] = mapped_column(String(71), nullable=False)
|
||||
control_space_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
||||
legacy_allowed: Mapped[bool | None] = mapped_column(sa.Boolean, nullable=True, default=None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCutoverRevisionWatermark",
|
||||
"KnowledgeFSCutoverSmokeChecks",
|
||||
"KnowledgeFSCutoverSmokeEvidenceReferences",
|
||||
"KnowledgeFSCutoverSmokeResults",
|
||||
"KnowledgeFSMigrationIssue",
|
||||
"KnowledgeFSMigrationIssueKind",
|
||||
"KnowledgeFSMigrationIssueStatus",
|
||||
"KnowledgeFSMigrationQuarantine",
|
||||
"KnowledgeFSMigrationQuarantineDisposition",
|
||||
"KnowledgeFSMigrationQuarantineKind",
|
||||
"KnowledgeFSShadowAuthorizationDecision",
|
||||
"KnowledgeFSShadowAuthorizationDiff",
|
||||
"KnowledgeFSShadowAuthorizationObservation",
|
||||
"KnowledgeFSWorkspaceCutoverLedger",
|
||||
"KnowledgeFSWorkspaceCutoverPhase",
|
||||
"knowledge_fs_cutover_smoke_results_passed",
|
||||
]
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Persistence contract for irreversible KnowledgeFS cleanup authorization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs_cleanup import (
|
||||
KnowledgeFSCleanupAuthorization,
|
||||
KnowledgeFSCleanupAuthorizationStatus,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSCleanupAuthorizationCASUpdate:
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
request_id: str
|
||||
expected_status: KnowledgeFSCleanupAuthorizationStatus
|
||||
expected_row_version: int
|
||||
new_status: KnowledgeFSCleanupAuthorizationStatus
|
||||
approved_by_account_id: str | None = None
|
||||
approved_at: datetime | None = None
|
||||
approval_expires_at: datetime | None = None
|
||||
approved_ledger_cas_version: int | None = None
|
||||
started_by_account_id: str | None = None
|
||||
started_at: datetime | None = None
|
||||
started_ledger_cas_version: int | None = None
|
||||
completed_by_account_id: str | None = None
|
||||
completed_at: datetime | None = None
|
||||
completion_evidence: dict[str, object] | None = None
|
||||
completed_ledger_cas_version: int | None = None
|
||||
|
||||
|
||||
class KnowledgeFSCleanupAuthorizationRepository(Protocol):
|
||||
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization: ...
|
||||
|
||||
def get(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
request_id: str,
|
||||
) -> KnowledgeFSCleanupAuthorization | None: ...
|
||||
|
||||
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool: ...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCleanupAuthorizationCASUpdate",
|
||||
"KnowledgeFSCleanupAuthorizationRepository",
|
||||
]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Persistence boundary for tenant-scoped KnowledgeFS lifecycle operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSControlSpaceCASUpdate:
|
||||
"""Values applied only when the stored state and resource version match."""
|
||||
|
||||
tenant_id: str
|
||||
control_space_id: str
|
||||
expected_resource_version: int
|
||||
expected_state: KnowledgeFSControlSpaceState
|
||||
new_state: KnowledgeFSControlSpaceState
|
||||
lifecycle_operation_id: str
|
||||
knowledge_space_id: str | None = None
|
||||
knowledge_space_revision: int | None = None
|
||||
attempted_at: datetime | None = None
|
||||
last_error_code: str | None = None
|
||||
last_error_message: str | None = None
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceRepository(Protocol):
|
||||
"""Repository operations that always include the owning Dify tenant."""
|
||||
|
||||
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace: ...
|
||||
|
||||
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def find_by_knowledge_space_id(
|
||||
self, *, tenant_id: str, knowledge_space_id: str
|
||||
) -> KnowledgeFSControlSpace | None: ...
|
||||
|
||||
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]: ...
|
||||
|
||||
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]: ...
|
||||
|
||||
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool: ...
|
||||
|
||||
def mark_deletion_irreversible(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
lifecycle_operation_id: str,
|
||||
irreversible_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]: ...
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSControlSpaceCASUpdate", "KnowledgeFSControlSpaceRepository"]
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Persistence contract for KnowledgeFS workspace migration and cutover state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs_cutover import (
|
||||
KnowledgeFSCutoverRevisionWatermark,
|
||||
KnowledgeFSCutoverSmokeResults,
|
||||
KnowledgeFSMigrationIssue,
|
||||
KnowledgeFSMigrationIssueStatus,
|
||||
KnowledgeFSMigrationQuarantine,
|
||||
KnowledgeFSMigrationQuarantineDisposition,
|
||||
KnowledgeFSMigrationQuarantineKind,
|
||||
KnowledgeFSShadowAuthorizationDecision,
|
||||
KnowledgeFSShadowAuthorizationDiff,
|
||||
KnowledgeFSShadowAuthorizationObservation,
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSCutoverCASUpdate:
|
||||
"""Optional values changed with the phase and CAS version in one SQL statement."""
|
||||
|
||||
tenant_id: str
|
||||
expected_phase: KnowledgeFSWorkspaceCutoverPhase
|
||||
expected_cas_version: int
|
||||
new_phase: KnowledgeFSWorkspaceCutoverPhase
|
||||
source_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
final_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
applied_revision_watermark: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
source_task_watermark: int | None = None
|
||||
final_task_watermark: int | None = None
|
||||
applied_task_watermark: int | None = None
|
||||
shadow_started_at: datetime | None = None
|
||||
shadow_completed_at: datetime | None = None
|
||||
shadow_evidence_digest: str | None = None
|
||||
shadow_observation_count: int | None = None
|
||||
shadow_window_started_at: datetime | None = None
|
||||
shadow_window_ended_at: datetime | None = None
|
||||
shadow_traffic_zero: bool | None = None
|
||||
shadow_traffic_zero_evidence: dict[str, object] | None = None
|
||||
shadow_latest_observed_revision: KnowledgeFSCutoverRevisionWatermark | None = None
|
||||
shadow_producer: str | None = None
|
||||
shadow_completed_by_operator: str | None = None
|
||||
shadow_completed_by_account_id: str | None = None
|
||||
remote_freeze_id: str | None = None
|
||||
remote_freeze_revision: int | None = None
|
||||
remote_freeze_digest: str | None = None
|
||||
remote_freeze_task_watermark: int | None = None
|
||||
remote_freeze_control_space_id: str | None = None
|
||||
remote_freeze_frozen_at: datetime | None = None
|
||||
remote_freeze_updated_at: datetime | None = None
|
||||
remote_freeze_acknowledged_at: datetime | None = None
|
||||
remote_freeze_applied: bool | None = None
|
||||
remote_freeze_replayed: bool | None = None
|
||||
remote_activation_id: str | None = None
|
||||
remote_activation_revision: int | None = None
|
||||
remote_activation_digest: str | None = None
|
||||
remote_activation_control_space_id: str | None = None
|
||||
remote_activation_activated_at: datetime | None = None
|
||||
remote_activation_updated_at: datetime | None = None
|
||||
remote_activation_acknowledged_at: datetime | None = None
|
||||
remote_activation_applied: bool | None = None
|
||||
remote_activation_replayed: bool | None = None
|
||||
clear_smoke_results: bool = False
|
||||
freeze_at: datetime | None = None
|
||||
cutover_at: datetime | None = None
|
||||
rolled_back_at: datetime | None = None
|
||||
rollback_cutoff_at: datetime | None = None
|
||||
observation_started_at: datetime | None = None
|
||||
observation_window_ends_at: datetime | None = None
|
||||
observation_completed_at: datetime | None = None
|
||||
maximum_task_expires_at: datetime | None = None
|
||||
irreversible_cleanup_at: datetime | None = None
|
||||
smoke_results: KnowledgeFSCutoverSmokeResults | None = None
|
||||
legacy_dependency_report: list[dict[str, object]] | None = None
|
||||
legacy_dependency_checked_at: datetime | None = None
|
||||
legacy_dependency_ready: bool | None = None
|
||||
product_routes_enabled: bool | None = None
|
||||
capability_v2_enabled: bool | None = None
|
||||
integrated_mode_enabled: bool | None = None
|
||||
legacy_acl_read_only: bool | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSQuarantineCASUpdate:
|
||||
"""Tenant/item-scoped transition from an operator disposition to resolved."""
|
||||
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
source_kind: KnowledgeFSMigrationQuarantineKind
|
||||
source_id: str
|
||||
expected_disposition: KnowledgeFSMigrationQuarantineDisposition
|
||||
expected_row_version: int
|
||||
resolved_by_operator: str
|
||||
resolved_by_account_id: str
|
||||
evidence: dict[str, object]
|
||||
resolved_at: datetime
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnowledgeFSShadowDiffCASUpdate:
|
||||
"""Replace one current diff disposition after append-only safe re-evaluation."""
|
||||
|
||||
tenant_id: str
|
||||
ledger_id: str
|
||||
diff_key: str
|
||||
expected_row_version: int
|
||||
control_space_id: str | None
|
||||
principal: str
|
||||
legacy_allowed: bool | None
|
||||
dify_allowed: bool
|
||||
decision: KnowledgeFSShadowAuthorizationDecision
|
||||
reason: str
|
||||
observed_revision: KnowledgeFSCutoverRevisionWatermark
|
||||
status: KnowledgeFSMigrationIssueStatus
|
||||
current_evidence_digest: str
|
||||
last_observed_at: datetime
|
||||
|
||||
|
||||
class KnowledgeFSCutoverRepository(Protocol):
|
||||
"""Tenant-scoped storage; callers own the surrounding transaction."""
|
||||
|
||||
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger: ...
|
||||
|
||||
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None: ...
|
||||
|
||||
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]: ...
|
||||
|
||||
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool: ...
|
||||
|
||||
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue: ...
|
||||
|
||||
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None: ...
|
||||
|
||||
def set_issue_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
issue_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine: ...
|
||||
|
||||
def get_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kind: KnowledgeFSMigrationQuarantineKind,
|
||||
source_id: str,
|
||||
) -> KnowledgeFSMigrationQuarantine | None: ...
|
||||
|
||||
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool: ...
|
||||
|
||||
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff: ...
|
||||
|
||||
def get_shadow_diff(
|
||||
self, *, tenant_id: str, ledger_id: str, diff_key: str
|
||||
) -> KnowledgeFSShadowAuthorizationDiff | None: ...
|
||||
|
||||
def set_shadow_diff_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool: ...
|
||||
|
||||
def add_shadow_observation(
|
||||
self, observation: KnowledgeFSShadowAuthorizationObservation
|
||||
) -> KnowledgeFSShadowAuthorizationObservation: ...
|
||||
|
||||
def get_shadow_observation(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
evidence_digest: str,
|
||||
) -> KnowledgeFSShadowAuthorizationObservation | None: ...
|
||||
|
||||
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int: ...
|
||||
|
||||
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int: ...
|
||||
|
||||
def count_unresolved_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
|
||||
) -> int: ...
|
||||
|
||||
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]: ...
|
||||
|
||||
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]: ...
|
||||
|
||||
def list_shadow_diffs(
|
||||
self, *, tenant_id: str, ledger_id: str
|
||||
) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]: ...
|
||||
|
||||
def list_shadow_observations(
|
||||
self, *, tenant_id: str, ledger_id: str
|
||||
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]: ...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCutoverCASUpdate",
|
||||
"KnowledgeFSCutoverRepository",
|
||||
"KnowledgeFSQuarantineCASUpdate",
|
||||
"KnowledgeFSShadowDiffCASUpdate",
|
||||
]
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Persistence boundary for durable KnowledgeFS lifecycle commands."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Protocol
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSLifecycleOperation, KnowledgeFSLifecycleOutbox
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleOutboxRepository(Protocol):
|
||||
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox: ...
|
||||
|
||||
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def find_open_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def find_latest_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def reactivate_dead_letter(self, *, outbox_id: str) -> bool: ...
|
||||
|
||||
def supersede_unattempted(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def supersede_after_remote_absence(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
observed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
lease_owner: str,
|
||||
now: datetime,
|
||||
lease_duration: timedelta,
|
||||
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
|
||||
) -> KnowledgeFSLifecycleOutbox | None: ...
|
||||
|
||||
def acknowledge(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
) -> bool: ...
|
||||
|
||||
def schedule_retry(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
next_attempt_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
def mark_dead_letter(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSLifecycleOutboxRepository"]
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Durable SQLAlchemy sink for sanitized KnowledgeFS capability issuance audits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSCapabilityClaimsSummary, KnowledgeFSCapabilityIssuanceAudit
|
||||
from services.knowledge_fs_capability import CapabilityIssuanceAuditEvent, CapabilityIssuanceAuditor
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor(CapabilityIssuanceAuditor):
|
||||
"""Commit the audit independently so token issuance fails closed on audit failure."""
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session]):
|
||||
self._session_maker = session_maker
|
||||
|
||||
@override
|
||||
def record(self, event: CapabilityIssuanceAuditEvent) -> None:
|
||||
serialized = event.model_dump(mode="json")
|
||||
trace_id = cast(str, serialized.pop("trace_id"))
|
||||
jti_hash = cast(str, serialized.pop("jti_hash"))
|
||||
audit = KnowledgeFSCapabilityIssuanceAudit(
|
||||
tenant_id=event.namespace_id,
|
||||
control_space_id=event.control_space_id,
|
||||
trace_id=trace_id,
|
||||
jti_hash=jti_hash,
|
||||
claims_summary=cast(KnowledgeFSCapabilityClaimsSummary, serialized),
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
session.add(audit)
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSCapabilityIssuanceAuditor"]
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
"""Transactional persistence for pre-signing KnowledgeFS capability reservations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import cast
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSCapabilityIssuanceReservationStatus,
|
||||
KnowledgeFSCapabilityReservationSummary,
|
||||
)
|
||||
from services.knowledge_fs_capability import (
|
||||
CAPABILITY_ISSUANCE_PROFILES,
|
||||
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
|
||||
CapabilityIssueRequest,
|
||||
)
|
||||
|
||||
_TERMINAL_RETENTION = timedelta(days=1)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceReservationError(RuntimeError):
|
||||
"""A retry attempted to change or lose a durable issuance binding."""
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository:
|
||||
def __init__(self, session: Session) -> None:
|
||||
self._session = session
|
||||
|
||||
def reserve(self, request: CapabilityIssueRequest) -> KnowledgeFSCapabilityIssuanceReservation:
|
||||
summary = _request_summary(request)
|
||||
reservation = self._session.scalar(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.tenant_id == request.namespace_id,
|
||||
KnowledgeFSCapabilityIssuanceReservation.grant_id == request.grant_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if reservation is None:
|
||||
reservation = KnowledgeFSCapabilityIssuanceReservation(
|
||||
tenant_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
grant_id=request.grant_id,
|
||||
trace_id=request.trace_id,
|
||||
subject=summary["subject"],
|
||||
caller_kind=request.caller_kind,
|
||||
request_summary=summary,
|
||||
)
|
||||
self._session.add(reservation)
|
||||
self._session.flush()
|
||||
return reservation
|
||||
if (
|
||||
reservation.control_space_id != request.control_space_id
|
||||
or reservation.trace_id != request.trace_id
|
||||
or reservation.subject != summary["subject"]
|
||||
or reservation.caller_kind != request.caller_kind
|
||||
or reservation.request_summary != summary
|
||||
):
|
||||
raise KnowledgeFSCapabilityIssuanceReservationError(
|
||||
"Capability grant retry does not match its durable reservation"
|
||||
)
|
||||
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.FAILED:
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED
|
||||
reservation.failed_at = None
|
||||
reservation.failure_code = None
|
||||
reservation.cleanup_after = None
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
return reservation
|
||||
|
||||
def mark_issued(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
grant_id: str,
|
||||
issued_at: datetime,
|
||||
token_expires_at: datetime,
|
||||
) -> None:
|
||||
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED
|
||||
reservation.issued_at = issued_at
|
||||
reservation.token_expires_at = token_expires_at
|
||||
reservation.failed_at = None
|
||||
reservation.failure_code = None
|
||||
reservation.cleanup_after = token_expires_at + _TERMINAL_RETENTION
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
|
||||
def mark_failed(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
grant_id: str,
|
||||
failed_at: datetime,
|
||||
failure_code: str,
|
||||
) -> None:
|
||||
reservation = self._locked(tenant_id=tenant_id, grant_id=grant_id)
|
||||
if reservation.status is KnowledgeFSCapabilityIssuanceReservationStatus.ISSUED:
|
||||
return
|
||||
reservation.status = KnowledgeFSCapabilityIssuanceReservationStatus.FAILED
|
||||
reservation.issued_at = None
|
||||
reservation.token_expires_at = None
|
||||
reservation.failed_at = failed_at
|
||||
reservation.failure_code = failure_code[:128]
|
||||
reservation.cleanup_after = failed_at + _TERMINAL_RETENTION
|
||||
reservation.row_version += 1
|
||||
self._session.flush()
|
||||
|
||||
def cleanup_terminal(self, *, before: datetime, limit: int = 1_000) -> int:
|
||||
if limit <= 0:
|
||||
raise ValueError("Capability reservation cleanup limit must be positive")
|
||||
reservation_ids = tuple(
|
||||
self._session.scalars(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation.id)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.status
|
||||
!= KnowledgeFSCapabilityIssuanceReservationStatus.RESERVED,
|
||||
KnowledgeFSCapabilityIssuanceReservation.cleanup_after <= before,
|
||||
)
|
||||
.order_by(
|
||||
KnowledgeFSCapabilityIssuanceReservation.cleanup_after,
|
||||
KnowledgeFSCapabilityIssuanceReservation.id,
|
||||
)
|
||||
.limit(limit)
|
||||
.with_for_update()
|
||||
)
|
||||
)
|
||||
if not reservation_ids:
|
||||
return 0
|
||||
result = self._session.execute(
|
||||
sa.delete(KnowledgeFSCapabilityIssuanceReservation).where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.id.in_(reservation_ids)
|
||||
)
|
||||
)
|
||||
return cast(CursorResult[tuple[object, ...]], result).rowcount or 0
|
||||
|
||||
def _locked(self, *, tenant_id: str, grant_id: str) -> KnowledgeFSCapabilityIssuanceReservation:
|
||||
reservation = self._session.scalar(
|
||||
sa.select(KnowledgeFSCapabilityIssuanceReservation)
|
||||
.where(
|
||||
KnowledgeFSCapabilityIssuanceReservation.tenant_id == tenant_id,
|
||||
KnowledgeFSCapabilityIssuanceReservation.grant_id == grant_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if reservation is None:
|
||||
raise KnowledgeFSCapabilityIssuanceReservationError("Capability issuance reservation is missing")
|
||||
return reservation
|
||||
|
||||
|
||||
def _request_summary(request: CapabilityIssueRequest) -> KnowledgeFSCapabilityReservationSummary:
|
||||
operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[request.operation_id]
|
||||
profile = CAPABILITY_ISSUANCE_PROFILES[request.caller_kind]
|
||||
subject = f"{profile.subject_prefix}:{request.principal_id}"
|
||||
return cast(
|
||||
KnowledgeFSCapabilityReservationSummary,
|
||||
{
|
||||
"action": operation.action,
|
||||
"actor": request.actor,
|
||||
"authz_revision": request.authz_revision.model_dump(mode="json"),
|
||||
"caller_kind": request.caller_kind,
|
||||
"content_policy_revision": request.content_policy_revision,
|
||||
"content_scope_ids": list(request.content_scope_ids),
|
||||
"control_space_id": request.control_space_id,
|
||||
"grant_id": request.grant_id,
|
||||
"namespace_id": request.namespace_id,
|
||||
"operation_id": request.operation_id,
|
||||
"resource_id": request.resource.id,
|
||||
"resource_parent_id": request.resource.parent_id,
|
||||
"resource_type": request.resource.type,
|
||||
"subject": subject,
|
||||
"trace_id": request.trace_id,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCapabilityIssuanceReservationError",
|
||||
"SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository",
|
||||
]
|
||||
@@ -0,0 +1,87 @@
|
||||
"""SQLAlchemy cleanup-authorization repository with status and version CAS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs_cleanup import KnowledgeFSCleanupAuthorization
|
||||
from repositories.knowledge_fs_cleanup_repository import (
|
||||
KnowledgeFSCleanupAuthorizationCASUpdate,
|
||||
KnowledgeFSCleanupAuthorizationRepository,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(KnowledgeFSCleanupAuthorizationRepository):
|
||||
_session: Session
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
def add(self, authorization: KnowledgeFSCleanupAuthorization) -> KnowledgeFSCleanupAuthorization:
|
||||
self._session.add(authorization)
|
||||
self._session.flush()
|
||||
return authorization
|
||||
|
||||
@override
|
||||
def get(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
request_id: str,
|
||||
) -> KnowledgeFSCleanupAuthorization | None:
|
||||
statement = (
|
||||
select(KnowledgeFSCleanupAuthorization)
|
||||
.where(
|
||||
KnowledgeFSCleanupAuthorization.tenant_id == tenant_id,
|
||||
KnowledgeFSCleanupAuthorization.ledger_id == ledger_id,
|
||||
KnowledgeFSCleanupAuthorization.request_id == request_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def compare_and_set(self, update_values: KnowledgeFSCleanupAuthorizationCASUpdate) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"status": update_values.new_status,
|
||||
"row_version": KnowledgeFSCleanupAuthorization.row_version + 1,
|
||||
}
|
||||
for field_name in (
|
||||
"approved_by_account_id",
|
||||
"approved_at",
|
||||
"approval_expires_at",
|
||||
"approved_ledger_cas_version",
|
||||
"started_by_account_id",
|
||||
"started_at",
|
||||
"started_ledger_cas_version",
|
||||
"completed_by_account_id",
|
||||
"completed_at",
|
||||
"completion_evidence",
|
||||
"completed_ledger_cas_version",
|
||||
):
|
||||
value = getattr(update_values, field_name)
|
||||
if value is not None:
|
||||
values[field_name] = value
|
||||
statement = (
|
||||
update(KnowledgeFSCleanupAuthorization)
|
||||
.where(
|
||||
KnowledgeFSCleanupAuthorization.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSCleanupAuthorization.ledger_id == update_values.ledger_id,
|
||||
KnowledgeFSCleanupAuthorization.request_id == update_values.request_id,
|
||||
KnowledgeFSCleanupAuthorization.status == update_values.expected_status,
|
||||
KnowledgeFSCleanupAuthorization.row_version == update_values.expected_row_version,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSCleanupAuthorizationRepository"]
|
||||
@@ -0,0 +1,187 @@
|
||||
"""SQLAlchemy implementation of the KnowledgeFS control-space repository."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy import case, select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
from repositories.knowledge_fs_control_space_repository import (
|
||||
KnowledgeFSControlSpaceCASUpdate,
|
||||
KnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSControlSpaceRepository(KnowledgeFSControlSpaceRepository):
|
||||
"""Caller-transaction-owned repository with tenant and CAS guards."""
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
def add(self, control_space: KnowledgeFSControlSpace) -> KnowledgeFSControlSpace:
|
||||
self._session.add(control_space)
|
||||
self._session.flush()
|
||||
return control_space
|
||||
|
||||
@override
|
||||
def get(self, *, tenant_id: str, control_space_id: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_by_provisioning_key(self, *, provisioning_key: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.provisioning_key == provisioning_key)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_by_knowledge_space_id(self, *, tenant_id: str, knowledge_space_id: str) -> KnowledgeFSControlSpace | None:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.knowledge_space_id == knowledge_space_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def list_for_tenant(self, *, tenant_id: str) -> tuple[KnowledgeFSControlSpace, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
|
||||
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def list_for_reconciliation(self, *, limit: int) -> tuple[KnowledgeFSControlSpace, ...]:
|
||||
if limit <= 0:
|
||||
return ()
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED)
|
||||
.order_by(
|
||||
case(
|
||||
(
|
||||
KnowledgeFSControlSpace.state.in_(
|
||||
(
|
||||
KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
)
|
||||
),
|
||||
0,
|
||||
),
|
||||
else_=1,
|
||||
),
|
||||
KnowledgeFSControlSpace.updated_at,
|
||||
KnowledgeFSControlSpace.id,
|
||||
)
|
||||
.limit(limit)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def compare_and_set_lifecycle(self, update_values: KnowledgeFSControlSpaceCASUpdate) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"state": update_values.new_state,
|
||||
"resource_version": KnowledgeFSControlSpace.resource_version + 1,
|
||||
"lifecycle_operation_id": update_values.lifecycle_operation_id,
|
||||
"last_error_code": update_values.last_error_code,
|
||||
"last_error_message": update_values.last_error_message,
|
||||
}
|
||||
if update_values.knowledge_space_id is not None:
|
||||
values["knowledge_space_id"] = update_values.knowledge_space_id
|
||||
if update_values.knowledge_space_revision is not None:
|
||||
values["knowledge_space_revision"] = update_values.knowledge_space_revision
|
||||
if update_values.attempted_at is not None:
|
||||
values["attempt_count"] = KnowledgeFSControlSpace.attempt_count + 1
|
||||
values["last_attempt_at"] = update_values.attempted_at
|
||||
|
||||
statement = (
|
||||
update(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSControlSpace.id == update_values.control_space_id,
|
||||
KnowledgeFSControlSpace.resource_version == update_values.expected_resource_version,
|
||||
KnowledgeFSControlSpace.state == update_values.expected_state,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
if update_values.knowledge_space_id is not None:
|
||||
statement = statement.where(
|
||||
(KnowledgeFSControlSpace.knowledge_space_id.is_(None))
|
||||
| (KnowledgeFSControlSpace.knowledge_space_id == update_values.knowledge_space_id)
|
||||
)
|
||||
if update_values.knowledge_space_revision is not None:
|
||||
statement = statement.where(
|
||||
KnowledgeFSControlSpace.knowledge_space_revision <= update_values.knowledge_space_revision
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def mark_deletion_irreversible(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
lifecycle_operation_id: str,
|
||||
irreversible_at: datetime,
|
||||
) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
KnowledgeFSControlSpace.lifecycle_operation_id == lifecycle_operation_id,
|
||||
KnowledgeFSControlSpace.state.in_(
|
||||
(
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
KnowledgeFSControlSpaceState.DELETED,
|
||||
)
|
||||
),
|
||||
(KnowledgeFSControlSpace.deletion_irreversible_at.is_(None))
|
||||
| (KnowledgeFSControlSpace.deletion_irreversible_at == irreversible_at),
|
||||
)
|
||||
.values(deletion_irreversible_at=irreversible_at)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def list_workspace_deletion_blockers(self, *, tenant_id: str) -> tuple[str, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace.id)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
|
||||
)
|
||||
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceRepository"]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Database aggregation for low-cardinality KnowledgeFS current-state metrics."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSControlSpaceStateCountReader:
|
||||
"""Read global operational counts without exposing tenant or resource labels."""
|
||||
|
||||
_states = (
|
||||
KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
)
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session]) -> None:
|
||||
self._session_maker = session_maker
|
||||
|
||||
def __call__(self) -> dict[str, int]:
|
||||
statement = (
|
||||
select(KnowledgeFSControlSpace.state, func.count())
|
||||
.where(KnowledgeFSControlSpace.state.in_(self._states))
|
||||
.group_by(KnowledgeFSControlSpace.state)
|
||||
)
|
||||
with self._session_maker() as session:
|
||||
rows = session.execute(statement).all()
|
||||
counts = {state.value: 0 for state in self._states}
|
||||
counts.update({state.value: int(count) for state, count in rows})
|
||||
return counts
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSControlSpaceStateCountReader"]
|
||||
@@ -0,0 +1,442 @@
|
||||
"""SQLAlchemy repository for the KnowledgeFS workspace cutover ledger."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import cast, override
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs_cutover import (
|
||||
KnowledgeFSMigrationIssue,
|
||||
KnowledgeFSMigrationIssueStatus,
|
||||
KnowledgeFSMigrationQuarantine,
|
||||
KnowledgeFSMigrationQuarantineDisposition,
|
||||
KnowledgeFSMigrationQuarantineKind,
|
||||
KnowledgeFSShadowAuthorizationDiff,
|
||||
KnowledgeFSShadowAuthorizationObservation,
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
)
|
||||
from repositories.knowledge_fs_cutover_repository import (
|
||||
KnowledgeFSCutoverCASUpdate,
|
||||
KnowledgeFSCutoverRepository,
|
||||
KnowledgeFSQuarantineCASUpdate,
|
||||
KnowledgeFSShadowDiffCASUpdate,
|
||||
)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSCutoverRepository(KnowledgeFSCutoverRepository):
|
||||
"""Persist one tenant ledger and evidence using tenant-scoped queries and CAS writes."""
|
||||
|
||||
_session: Session
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._session = session
|
||||
|
||||
@override
|
||||
def add_ledger(self, ledger: KnowledgeFSWorkspaceCutoverLedger) -> KnowledgeFSWorkspaceCutoverLedger:
|
||||
self._session.add(ledger)
|
||||
self._session.flush()
|
||||
return ledger
|
||||
|
||||
@override
|
||||
def get_ledger(self, *, tenant_id: str) -> KnowledgeFSWorkspaceCutoverLedger | None:
|
||||
statement = (
|
||||
select(KnowledgeFSWorkspaceCutoverLedger)
|
||||
.where(KnowledgeFSWorkspaceCutoverLedger.tenant_id == tenant_id)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def list_ledgers(self) -> tuple[KnowledgeFSWorkspaceCutoverLedger, ...]:
|
||||
statement = select(KnowledgeFSWorkspaceCutoverLedger).order_by(KnowledgeFSWorkspaceCutoverLedger.tenant_id)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def compare_and_set(self, update_values: KnowledgeFSCutoverCASUpdate) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"phase": update_values.new_phase,
|
||||
"cas_version": KnowledgeFSWorkspaceCutoverLedger.cas_version + 1,
|
||||
}
|
||||
for field_name in (
|
||||
"source_revision_watermark",
|
||||
"final_revision_watermark",
|
||||
"applied_revision_watermark",
|
||||
"source_task_watermark",
|
||||
"final_task_watermark",
|
||||
"applied_task_watermark",
|
||||
"shadow_started_at",
|
||||
"shadow_completed_at",
|
||||
"shadow_evidence_digest",
|
||||
"shadow_observation_count",
|
||||
"shadow_window_started_at",
|
||||
"shadow_window_ended_at",
|
||||
"shadow_traffic_zero",
|
||||
"shadow_traffic_zero_evidence",
|
||||
"shadow_latest_observed_revision",
|
||||
"shadow_producer",
|
||||
"shadow_completed_by_operator",
|
||||
"shadow_completed_by_account_id",
|
||||
"remote_freeze_id",
|
||||
"remote_freeze_revision",
|
||||
"remote_freeze_digest",
|
||||
"remote_freeze_task_watermark",
|
||||
"remote_freeze_control_space_id",
|
||||
"remote_freeze_frozen_at",
|
||||
"remote_freeze_updated_at",
|
||||
"remote_freeze_acknowledged_at",
|
||||
"remote_freeze_applied",
|
||||
"remote_freeze_replayed",
|
||||
"remote_activation_id",
|
||||
"remote_activation_revision",
|
||||
"remote_activation_digest",
|
||||
"remote_activation_control_space_id",
|
||||
"remote_activation_activated_at",
|
||||
"remote_activation_updated_at",
|
||||
"remote_activation_acknowledged_at",
|
||||
"remote_activation_applied",
|
||||
"remote_activation_replayed",
|
||||
"freeze_at",
|
||||
"cutover_at",
|
||||
"rolled_back_at",
|
||||
"rollback_cutoff_at",
|
||||
"observation_started_at",
|
||||
"observation_window_ends_at",
|
||||
"observation_completed_at",
|
||||
"maximum_task_expires_at",
|
||||
"irreversible_cleanup_at",
|
||||
"smoke_results",
|
||||
"legacy_dependency_report",
|
||||
"legacy_dependency_checked_at",
|
||||
"legacy_dependency_ready",
|
||||
"product_routes_enabled",
|
||||
"capability_v2_enabled",
|
||||
"integrated_mode_enabled",
|
||||
"legacy_acl_read_only",
|
||||
):
|
||||
value = getattr(update_values, field_name)
|
||||
if value is not None:
|
||||
values[field_name] = value
|
||||
if update_values.clear_smoke_results:
|
||||
values["smoke_results"] = None
|
||||
statement = (
|
||||
update(KnowledgeFSWorkspaceCutoverLedger)
|
||||
.where(
|
||||
KnowledgeFSWorkspaceCutoverLedger.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSWorkspaceCutoverLedger.phase == update_values.expected_phase,
|
||||
KnowledgeFSWorkspaceCutoverLedger.cas_version == update_values.expected_cas_version,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def add_issue(self, issue: KnowledgeFSMigrationIssue) -> KnowledgeFSMigrationIssue:
|
||||
self._session.add(issue)
|
||||
self._session.flush()
|
||||
return issue
|
||||
|
||||
@override
|
||||
def get_issue(self, *, tenant_id: str, ledger_id: str, issue_key: str) -> KnowledgeFSMigrationIssue | None:
|
||||
statement = select(KnowledgeFSMigrationIssue).where(
|
||||
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
|
||||
KnowledgeFSMigrationIssue.issue_key == issue_key,
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def set_issue_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
issue_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool:
|
||||
values: dict[str, object] = {"status": new_status}
|
||||
if new_status is KnowledgeFSMigrationIssueStatus.APPROVED_FAIL_CLOSED:
|
||||
values.update(approved_at=changed_at, approved_by_account_id=account_id)
|
||||
elif new_status is KnowledgeFSMigrationIssueStatus.RESOLVED:
|
||||
values.update(resolved_at=changed_at, resolved_by_account_id=account_id)
|
||||
statement = (
|
||||
update(KnowledgeFSMigrationIssue)
|
||||
.where(
|
||||
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
|
||||
KnowledgeFSMigrationIssue.issue_key == issue_key,
|
||||
KnowledgeFSMigrationIssue.status == expected_status,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def reevaluate_shadow_diff(self, update_values: KnowledgeFSShadowDiffCASUpdate) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSShadowAuthorizationDiff)
|
||||
.where(
|
||||
KnowledgeFSShadowAuthorizationDiff.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.ledger_id == update_values.ledger_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.diff_key == update_values.diff_key,
|
||||
KnowledgeFSShadowAuthorizationDiff.row_version == update_values.expected_row_version,
|
||||
)
|
||||
.values(
|
||||
control_space_id=update_values.control_space_id,
|
||||
principal=update_values.principal,
|
||||
legacy_allowed=update_values.legacy_allowed,
|
||||
dify_allowed=update_values.dify_allowed,
|
||||
decision=update_values.decision,
|
||||
reason=update_values.reason,
|
||||
observed_revision=update_values.observed_revision,
|
||||
status=update_values.status,
|
||||
current_evidence_digest=update_values.current_evidence_digest,
|
||||
last_observed_at=update_values.last_observed_at,
|
||||
approved_by_account_id=None,
|
||||
approved_at=None,
|
||||
resolved_by_account_id=None,
|
||||
resolved_at=update_values.last_observed_at,
|
||||
row_version=KnowledgeFSShadowAuthorizationDiff.row_version + 1,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def add_shadow_observation(
|
||||
self, observation: KnowledgeFSShadowAuthorizationObservation
|
||||
) -> KnowledgeFSShadowAuthorizationObservation:
|
||||
self._session.add(observation)
|
||||
self._session.flush()
|
||||
return observation
|
||||
|
||||
@override
|
||||
def get_shadow_observation(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
evidence_digest: str,
|
||||
) -> KnowledgeFSShadowAuthorizationObservation | None:
|
||||
statement = select(KnowledgeFSShadowAuthorizationObservation).where(
|
||||
KnowledgeFSShadowAuthorizationObservation.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationObservation.ledger_id == ledger_id,
|
||||
KnowledgeFSShadowAuthorizationObservation.diff_key == diff_key,
|
||||
KnowledgeFSShadowAuthorizationObservation.evidence_digest == evidence_digest,
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def add_quarantine(self, item: KnowledgeFSMigrationQuarantine) -> KnowledgeFSMigrationQuarantine:
|
||||
self._session.add(item)
|
||||
self._session.flush()
|
||||
return item
|
||||
|
||||
@override
|
||||
def get_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kind: KnowledgeFSMigrationQuarantineKind,
|
||||
source_id: str,
|
||||
) -> KnowledgeFSMigrationQuarantine | None:
|
||||
statement = select(KnowledgeFSMigrationQuarantine).where(
|
||||
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
|
||||
KnowledgeFSMigrationQuarantine.source_kind == source_kind,
|
||||
KnowledgeFSMigrationQuarantine.source_id == source_id,
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def resolve_quarantine(self, update_values: KnowledgeFSQuarantineCASUpdate) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSMigrationQuarantine)
|
||||
.where(
|
||||
KnowledgeFSMigrationQuarantine.tenant_id == update_values.tenant_id,
|
||||
KnowledgeFSMigrationQuarantine.ledger_id == update_values.ledger_id,
|
||||
KnowledgeFSMigrationQuarantine.source_kind == update_values.source_kind,
|
||||
KnowledgeFSMigrationQuarantine.source_id == update_values.source_id,
|
||||
KnowledgeFSMigrationQuarantine.disposition == update_values.expected_disposition,
|
||||
KnowledgeFSMigrationQuarantine.row_version == update_values.expected_row_version,
|
||||
)
|
||||
.values(
|
||||
disposition=KnowledgeFSMigrationQuarantineDisposition.RESOLVED,
|
||||
resolved_by_operator=update_values.resolved_by_operator,
|
||||
resolved_by_account_id=update_values.resolved_by_account_id,
|
||||
evidence=update_values.evidence,
|
||||
resolved_at=update_values.resolved_at,
|
||||
row_version=KnowledgeFSMigrationQuarantine.row_version + 1,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def add_shadow_diff(self, diff: KnowledgeFSShadowAuthorizationDiff) -> KnowledgeFSShadowAuthorizationDiff:
|
||||
self._session.add(diff)
|
||||
self._session.flush()
|
||||
return diff
|
||||
|
||||
@override
|
||||
def get_shadow_diff(
|
||||
self, *, tenant_id: str, ledger_id: str, diff_key: str
|
||||
) -> KnowledgeFSShadowAuthorizationDiff | None:
|
||||
statement = select(KnowledgeFSShadowAuthorizationDiff).where(
|
||||
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.diff_key == diff_key,
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def set_shadow_diff_status(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
diff_key: str,
|
||||
expected_status: KnowledgeFSMigrationIssueStatus,
|
||||
new_status: KnowledgeFSMigrationIssueStatus,
|
||||
account_id: str,
|
||||
changed_at: datetime,
|
||||
) -> bool:
|
||||
values: dict[str, object] = {
|
||||
"status": new_status,
|
||||
"row_version": KnowledgeFSShadowAuthorizationDiff.row_version + 1,
|
||||
}
|
||||
if new_status is KnowledgeFSMigrationIssueStatus.APPROVED_FAIL_CLOSED:
|
||||
values.update(approved_at=changed_at, approved_by_account_id=account_id)
|
||||
elif new_status is KnowledgeFSMigrationIssueStatus.RESOLVED:
|
||||
values.update(resolved_at=changed_at, resolved_by_account_id=account_id)
|
||||
statement = (
|
||||
update(KnowledgeFSShadowAuthorizationDiff)
|
||||
.where(
|
||||
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.diff_key == diff_key,
|
||||
KnowledgeFSShadowAuthorizationDiff.status == expected_status,
|
||||
)
|
||||
.values(**values)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
@override
|
||||
def count_open_issues(self, *, tenant_id: str, ledger_id: str) -> int:
|
||||
statement = (
|
||||
select(func.count())
|
||||
.select_from(KnowledgeFSMigrationIssue)
|
||||
.where(
|
||||
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
|
||||
KnowledgeFSMigrationIssue.status == KnowledgeFSMigrationIssueStatus.OPEN,
|
||||
)
|
||||
)
|
||||
return self._session.scalar(statement) or 0
|
||||
|
||||
@override
|
||||
def count_unapproved_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> int:
|
||||
statement = (
|
||||
select(func.count())
|
||||
.select_from(KnowledgeFSShadowAuthorizationDiff)
|
||||
.where(
|
||||
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.status == KnowledgeFSMigrationIssueStatus.OPEN,
|
||||
)
|
||||
)
|
||||
return self._session.scalar(statement) or 0
|
||||
|
||||
@override
|
||||
def count_unresolved_quarantine(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
ledger_id: str,
|
||||
source_kinds: tuple[KnowledgeFSMigrationQuarantineKind, ...],
|
||||
) -> int:
|
||||
if not source_kinds:
|
||||
return 0
|
||||
statement = (
|
||||
select(func.count())
|
||||
.select_from(KnowledgeFSMigrationQuarantine)
|
||||
.where(
|
||||
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
|
||||
KnowledgeFSMigrationQuarantine.source_kind.in_(source_kinds),
|
||||
KnowledgeFSMigrationQuarantine.disposition != KnowledgeFSMigrationQuarantineDisposition.RESOLVED,
|
||||
)
|
||||
)
|
||||
return self._session.scalar(statement) or 0
|
||||
|
||||
@override
|
||||
def list_issues(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationIssue, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSMigrationIssue)
|
||||
.where(
|
||||
KnowledgeFSMigrationIssue.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationIssue.ledger_id == ledger_id,
|
||||
)
|
||||
.order_by(KnowledgeFSMigrationIssue.created_at, KnowledgeFSMigrationIssue.id)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def list_quarantine(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSMigrationQuarantine, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSMigrationQuarantine)
|
||||
.where(
|
||||
KnowledgeFSMigrationQuarantine.tenant_id == tenant_id,
|
||||
KnowledgeFSMigrationQuarantine.ledger_id == ledger_id,
|
||||
)
|
||||
.order_by(KnowledgeFSMigrationQuarantine.created_at, KnowledgeFSMigrationQuarantine.id)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def list_shadow_diffs(self, *, tenant_id: str, ledger_id: str) -> tuple[KnowledgeFSShadowAuthorizationDiff, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSShadowAuthorizationDiff)
|
||||
.where(
|
||||
KnowledgeFSShadowAuthorizationDiff.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationDiff.ledger_id == ledger_id,
|
||||
)
|
||||
.order_by(KnowledgeFSShadowAuthorizationDiff.created_at, KnowledgeFSShadowAuthorizationDiff.id)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
@override
|
||||
def list_shadow_observations(
|
||||
self, *, tenant_id: str, ledger_id: str
|
||||
) -> tuple[KnowledgeFSShadowAuthorizationObservation, ...]:
|
||||
statement = (
|
||||
select(KnowledgeFSShadowAuthorizationObservation)
|
||||
.where(
|
||||
KnowledgeFSShadowAuthorizationObservation.tenant_id == tenant_id,
|
||||
KnowledgeFSShadowAuthorizationObservation.ledger_id == ledger_id,
|
||||
)
|
||||
.order_by(
|
||||
KnowledgeFSShadowAuthorizationObservation.observed_at,
|
||||
KnowledgeFSShadowAuthorizationObservation.evidence_digest,
|
||||
)
|
||||
)
|
||||
return tuple(self._session.scalars(statement).all())
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSCutoverRepository"]
|
||||
@@ -0,0 +1,394 @@
|
||||
"""SQLAlchemy CAS implementation for the KnowledgeFS lifecycle outbox."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import cast, override
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSLifecycleOperation,
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSLifecycleOutboxStatus,
|
||||
)
|
||||
from repositories.knowledge_fs_lifecycle_outbox_repository import KnowledgeFSLifecycleOutboxRepository
|
||||
from services.knowledge_fs.observability import (
|
||||
KnowledgeFSLifecycleTaskMetric,
|
||||
KnowledgeFSOperationalMetricsPort,
|
||||
get_knowledge_fs_operational_metrics,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SQLAlchemyKnowledgeFSLifecycleOutboxRepository(KnowledgeFSLifecycleOutboxRepository):
|
||||
"""Claim and settle commands inside a caller-owned database transaction."""
|
||||
|
||||
def __init__(self, session: Session, metrics: KnowledgeFSOperationalMetricsPort | None = None):
|
||||
self._session = session
|
||||
self._metrics = metrics or get_knowledge_fs_operational_metrics()
|
||||
|
||||
@override
|
||||
def add(self, command: KnowledgeFSLifecycleOutbox) -> KnowledgeFSLifecycleOutbox:
|
||||
self._session.add(command)
|
||||
self._session.flush()
|
||||
try:
|
||||
self._metrics.record_lifecycle_task(KnowledgeFSLifecycleTaskMetric(None, command.operation.value, "queued"))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"KnowledgeFS lifecycle metric export failed operation=%s status=queued",
|
||||
command.operation.value,
|
||||
exc_info=True,
|
||||
)
|
||||
return command
|
||||
|
||||
@override
|
||||
def get(self, *, outbox_id: str) -> KnowledgeFSLifecycleOutbox | None:
|
||||
statement = (
|
||||
select(KnowledgeFSLifecycleOutbox)
|
||||
.where(KnowledgeFSLifecycleOutbox.id == outbox_id)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def get_by_operation_id(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSLifecycleOutbox | None:
|
||||
statement = (
|
||||
select(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
|
||||
KnowledgeFSLifecycleOutbox.operation_id == operation_id,
|
||||
)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_open_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None:
|
||||
statement = (
|
||||
select(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
|
||||
KnowledgeFSLifecycleOutbox.control_space_id == control_space_id,
|
||||
KnowledgeFSLifecycleOutbox.operation == operation,
|
||||
KnowledgeFSLifecycleOutbox.status.in_(
|
||||
(
|
||||
KnowledgeFSLifecycleOutboxStatus.PENDING,
|
||||
KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
KnowledgeFSLifecycleOutboxStatus.RETRY,
|
||||
)
|
||||
),
|
||||
)
|
||||
.order_by(KnowledgeFSLifecycleOutbox.created_at.desc(), KnowledgeFSLifecycleOutbox.id.desc())
|
||||
.limit(1)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def find_latest_for_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation: KnowledgeFSLifecycleOperation,
|
||||
) -> KnowledgeFSLifecycleOutbox | None:
|
||||
statement = (
|
||||
select(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.tenant_id == tenant_id,
|
||||
KnowledgeFSLifecycleOutbox.control_space_id == control_space_id,
|
||||
KnowledgeFSLifecycleOutbox.operation == operation,
|
||||
)
|
||||
.order_by(KnowledgeFSLifecycleOutbox.created_at.desc(), KnowledgeFSLifecycleOutbox.id.desc())
|
||||
.limit(1)
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
return self._session.scalar(statement)
|
||||
|
||||
@override
|
||||
def reactivate_dead_letter(self, *, outbox_id: str) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.id == outbox_id,
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
|
||||
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.RETRY,
|
||||
next_attempt_at=None,
|
||||
completed_at=None,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
@override
|
||||
def supersede_unattempted(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.id == outbox_id,
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PENDING,
|
||||
KnowledgeFSLifecycleOutbox.attempt_count == 0,
|
||||
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
|
||||
completed_at=completed_at,
|
||||
last_error_code=error_code,
|
||||
last_error_message=error_message,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
@override
|
||||
def supersede_after_remote_absence(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
observed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool:
|
||||
safely_unleased = sa.and_(
|
||||
KnowledgeFSLifecycleOutbox.status.in_(
|
||||
(
|
||||
KnowledgeFSLifecycleOutboxStatus.PENDING,
|
||||
KnowledgeFSLifecycleOutboxStatus.RETRY,
|
||||
KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
|
||||
)
|
||||
),
|
||||
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
|
||||
)
|
||||
expired_lease = sa.and_(
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at <= observed_at,
|
||||
)
|
||||
statement = (
|
||||
update(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.id == outbox_id,
|
||||
sa.or_(safely_unleased, expired_lease),
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
completed_at=sa.func.coalesce(KnowledgeFSLifecycleOutbox.completed_at, observed_at),
|
||||
last_error_code=error_code,
|
||||
last_error_message=error_message,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
@override
|
||||
def claim_next(
|
||||
self,
|
||||
*,
|
||||
lease_owner: str,
|
||||
now: datetime,
|
||||
lease_duration: timedelta,
|
||||
allowed_operations: tuple[KnowledgeFSLifecycleOperation, ...],
|
||||
) -> KnowledgeFSLifecycleOutbox | None:
|
||||
if not lease_owner.strip():
|
||||
raise ValueError("lease_owner must not be blank")
|
||||
if lease_duration <= timedelta(0):
|
||||
raise ValueError("lease_duration must be positive")
|
||||
if not allowed_operations:
|
||||
return None
|
||||
|
||||
ready_to_attempt = sa.and_(
|
||||
KnowledgeFSLifecycleOutbox.status.in_(
|
||||
(KnowledgeFSLifecycleOutboxStatus.PENDING, KnowledgeFSLifecycleOutboxStatus.RETRY)
|
||||
),
|
||||
sa.or_(
|
||||
KnowledgeFSLifecycleOutbox.next_attempt_at.is_(None),
|
||||
KnowledgeFSLifecycleOutbox.next_attempt_at <= now,
|
||||
),
|
||||
)
|
||||
expired_lease = sa.and_(
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at <= now,
|
||||
)
|
||||
candidate_statement = (
|
||||
select(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.operation.in_(allowed_operations),
|
||||
sa.or_(ready_to_attempt, expired_lease),
|
||||
)
|
||||
.order_by(
|
||||
sa.func.coalesce(KnowledgeFSLifecycleOutbox.next_attempt_at, KnowledgeFSLifecycleOutbox.created_at),
|
||||
KnowledgeFSLifecycleOutbox.id,
|
||||
)
|
||||
.limit(1)
|
||||
.with_for_update(skip_locked=True)
|
||||
)
|
||||
candidate = self._session.scalar(candidate_statement)
|
||||
if candidate is None:
|
||||
return None
|
||||
|
||||
lease_expires_at = now + lease_duration
|
||||
claim_statement = update(KnowledgeFSLifecycleOutbox).where(
|
||||
KnowledgeFSLifecycleOutbox.id == candidate.id,
|
||||
KnowledgeFSLifecycleOutbox.status == candidate.status,
|
||||
KnowledgeFSLifecycleOutbox.attempt_count == candidate.attempt_count,
|
||||
)
|
||||
if candidate.status is KnowledgeFSLifecycleOutboxStatus.PROCESSING:
|
||||
claim_statement = claim_statement.where(
|
||||
KnowledgeFSLifecycleOutbox.lease_owner == candidate.lease_owner,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at == candidate.lease_expires_at,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at <= now,
|
||||
)
|
||||
else:
|
||||
claim_statement = claim_statement.where(
|
||||
KnowledgeFSLifecycleOutbox.lease_owner.is_(None),
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at.is_(None),
|
||||
)
|
||||
result = self._session.execute(
|
||||
claim_statement.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
lease_owner=lease_owner,
|
||||
lease_expires_at=lease_expires_at,
|
||||
last_attempt_at=now,
|
||||
attempt_count=candidate.attempt_count + 1,
|
||||
completed_at=None,
|
||||
).execution_options(synchronize_session=False)
|
||||
)
|
||||
if (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) != 1:
|
||||
return None
|
||||
return self.get(outbox_id=candidate.id)
|
||||
|
||||
@override
|
||||
def acknowledge(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
) -> bool:
|
||||
statement = (
|
||||
update(KnowledgeFSLifecycleOutbox)
|
||||
.where(
|
||||
KnowledgeFSLifecycleOutbox.id == outbox_id,
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
KnowledgeFSLifecycleOutbox.lease_owner == lease_owner,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at == expected_lease_expires_at,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at > completed_at,
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.SUCCEEDED,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
completed_at=completed_at,
|
||||
last_error_code=None,
|
||||
last_error_message=None,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
@override
|
||||
def schedule_retry(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
next_attempt_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool:
|
||||
statement = (
|
||||
self._leased_update(
|
||||
outbox_id=outbox_id,
|
||||
lease_owner=lease_owner,
|
||||
expected_lease_expires_at=expected_lease_expires_at,
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.RETRY,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
next_attempt_at=next_attempt_at,
|
||||
completed_at=None,
|
||||
last_error_code=error_code,
|
||||
last_error_message=error_message,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
@override
|
||||
def mark_dead_letter(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
completed_at: datetime,
|
||||
error_code: str,
|
||||
error_message: str,
|
||||
) -> bool:
|
||||
statement = (
|
||||
self._leased_update(
|
||||
outbox_id=outbox_id,
|
||||
lease_owner=lease_owner,
|
||||
expected_lease_expires_at=expected_lease_expires_at,
|
||||
)
|
||||
.values(
|
||||
status=KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
completed_at=completed_at,
|
||||
last_error_code=error_code,
|
||||
last_error_message=error_message,
|
||||
)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return self._updated_once(statement)
|
||||
|
||||
def _leased_update(
|
||||
self,
|
||||
*,
|
||||
outbox_id: str,
|
||||
lease_owner: str,
|
||||
expected_lease_expires_at: datetime,
|
||||
) -> sa.Update:
|
||||
return update(KnowledgeFSLifecycleOutbox).where(
|
||||
KnowledgeFSLifecycleOutbox.id == outbox_id,
|
||||
KnowledgeFSLifecycleOutbox.status == KnowledgeFSLifecycleOutboxStatus.PROCESSING,
|
||||
KnowledgeFSLifecycleOutbox.lease_owner == lease_owner,
|
||||
KnowledgeFSLifecycleOutbox.lease_expires_at == expected_lease_expires_at,
|
||||
)
|
||||
|
||||
def _updated_once(self, statement: sa.Update) -> bool:
|
||||
result = self._session.execute(statement)
|
||||
return (cast(CursorResult[tuple[object, ...]], result).rowcount or 0) == 1
|
||||
|
||||
|
||||
__all__ = ["SQLAlchemyKnowledgeFSLifecycleOutboxRepository"]
|
||||
@@ -1380,12 +1380,31 @@ class TenantService:
|
||||
.where(TenantAccountJoin.tenant_id == tenant.id, TenantAccountJoin.account_id == account.id)
|
||||
.limit(1)
|
||||
)
|
||||
previous_role = TenantAccountRole(ta.role) if ta else None
|
||||
if ta:
|
||||
ta.role = TenantAccountRole(role)
|
||||
else:
|
||||
ta = TenantAccountJoin(tenant_id=tenant.id, account_id=account.id, role=TenantAccountRole(role))
|
||||
session.add(ta)
|
||||
|
||||
requested_role = TenantAccountRole(role)
|
||||
if previous_role is None or previous_role is not requested_role:
|
||||
from services.knowledge_fs.membership_changes import (
|
||||
KnowledgeFSWorkspaceMembershipChange,
|
||||
apply_workspace_membership_change,
|
||||
)
|
||||
|
||||
apply_workspace_membership_change(
|
||||
session=session,
|
||||
tenant_id=str(tenant.id),
|
||||
actor_account_id=str(account.id),
|
||||
account_ids=(str(account.id),),
|
||||
change=(
|
||||
KnowledgeFSWorkspaceMembershipChange.MEMBER_ADDED
|
||||
if previous_role is None
|
||||
else KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED
|
||||
),
|
||||
)
|
||||
session.commit()
|
||||
if dify_config.BILLING_ENABLED:
|
||||
BillingService.clean_billing_info_cache(tenant.id)
|
||||
@@ -1796,6 +1815,20 @@ class TenantService:
|
||||
)
|
||||
.values(maintainer=owner_id)
|
||||
)
|
||||
from services.knowledge_fs.membership_changes import (
|
||||
KnowledgeFSWorkspaceMembershipChange,
|
||||
apply_workspace_membership_change,
|
||||
)
|
||||
|
||||
apply_workspace_membership_change(
|
||||
session=session,
|
||||
tenant_id=str(tenant.id),
|
||||
actor_account_id=str(operator.id),
|
||||
account_ids=(str(account_id),),
|
||||
change=KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED,
|
||||
removed_account_id=str(account_id),
|
||||
replacement_owner_account_id=str(owner_id),
|
||||
)
|
||||
session.delete(ta)
|
||||
|
||||
# Clean up orphaned pending accounts (invited but never activated)
|
||||
@@ -1863,6 +1896,7 @@ class TenantService:
|
||||
if target_member_join.role == new_role:
|
||||
raise RoleAlreadyAssignedError("The provided role is already assigned to the member.")
|
||||
|
||||
affected_account_ids = [str(member.id)]
|
||||
if new_role == "owner":
|
||||
# Find the current owner and change their role to 'admin'
|
||||
current_owner_join = session.scalar(
|
||||
@@ -1886,6 +1920,8 @@ class TenantService:
|
||||
role_ids=[admin_role_id],
|
||||
session=session,
|
||||
)
|
||||
if current_owner_join and str(current_owner_join.account_id) != str(member.id):
|
||||
affected_account_ids.append(str(current_owner_join.account_id))
|
||||
|
||||
# Update the role of the target member
|
||||
if dify_config.RBAC_ENABLED:
|
||||
@@ -1903,6 +1939,18 @@ class TenantService:
|
||||
)
|
||||
else:
|
||||
target_member_join.role = new_tenant_role
|
||||
from services.knowledge_fs.membership_changes import (
|
||||
KnowledgeFSWorkspaceMembershipChange,
|
||||
apply_workspace_membership_change,
|
||||
)
|
||||
|
||||
apply_workspace_membership_change(
|
||||
session=session,
|
||||
tenant_id=str(tenant.id),
|
||||
actor_account_id=str(operator.id),
|
||||
account_ids=tuple(affected_account_ids),
|
||||
change=KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED,
|
||||
)
|
||||
session.commit()
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -19,9 +19,9 @@ from __future__ import annotations
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.agent.entities import AgentToolEntity
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom, UserFrom
|
||||
from core.callback_handler.workflow_tool_callback_handler import DifyWorkflowCallbackHandler
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
from core.tools.entities.tool_entities import ToolInvokeFrom, ToolProviderType
|
||||
from core.tools.errors import (
|
||||
ToolInvokeError,
|
||||
ToolNotFoundError,
|
||||
@@ -64,16 +64,28 @@ class AgentToolInnerService:
|
||||
credential_id=request.tool.credential_id,
|
||||
)
|
||||
try:
|
||||
dify_run_context = DifyRunContext(
|
||||
tenant_id=request.caller.tenant_id,
|
||||
app_id=request.caller.app_id,
|
||||
user_id=request.caller.user_id,
|
||||
user_from=UserFrom(request.caller.user_from),
|
||||
invoke_from=InvokeFrom.value_of(request.caller.invoke_from),
|
||||
trace_session_id=request.caller.trace_session_id,
|
||||
)
|
||||
tool_runtime = ToolManager.get_agent_tool_runtime(
|
||||
tenant_id=request.caller.tenant_id,
|
||||
app_id=request.caller.app_id,
|
||||
agent_tool=agent_tool,
|
||||
user_id=request.caller.user_id,
|
||||
invoke_from=InvokeFrom.value_of(request.caller.invoke_from),
|
||||
invoke_from=dify_run_context.invoke_from,
|
||||
variable_pool=None,
|
||||
allow_file_parameters=True,
|
||||
use_default_for_missing_form_parameters=True,
|
||||
)
|
||||
tool_runtime.runtime.dify_run_context = dify_run_context
|
||||
tool_runtime.runtime.tool_invoke_from = (
|
||||
ToolInvokeFrom.WORKFLOW if request.caller.workflow_id is not None else ToolInvokeFrom.AGENT
|
||||
)
|
||||
messages = ToolEngine.generic_invoke(
|
||||
session=session,
|
||||
tool=tool_runtime,
|
||||
|
||||
@@ -208,7 +208,12 @@ class DatasourceProviderService:
|
||||
if credential_id:
|
||||
datasource_provider = session.scalar(
|
||||
select(DatasourceProvider)
|
||||
.where(DatasourceProvider.tenant_id == tenant_id, DatasourceProvider.id == credential_id)
|
||||
.where(
|
||||
DatasourceProvider.tenant_id == tenant_id,
|
||||
DatasourceProvider.id == credential_id,
|
||||
DatasourceProvider.provider == provider,
|
||||
DatasourceProvider.plugin_id == plugin_id,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -46,6 +46,7 @@ class RBACResourceType(StrEnum):
|
||||
|
||||
APP = "app"
|
||||
DATASET = "dataset"
|
||||
KNOWLEDGE_FS = "knowledge_space"
|
||||
|
||||
|
||||
class RBACRoleType(StrEnum):
|
||||
@@ -287,9 +288,9 @@ class ResourcePermissionSnapshot(_RBACModel):
|
||||
overrides: list[ResourcePermissionKeys] = Field(default_factory=list)
|
||||
|
||||
def permission_keys_by_resource_ids(self, resource_ids: list[str]) -> dict[str, list[str]]:
|
||||
result = {str(resource_id): list(self.default_permission_keys) for resource_id in resource_ids}
|
||||
result = {resource_id: list(self.default_permission_keys) for resource_id in resource_ids}
|
||||
for override in self.overrides:
|
||||
resource_id = str(override.resource_id)
|
||||
resource_id = override.resource_id
|
||||
if resource_id in result:
|
||||
result[resource_id] = list(override.permission_keys)
|
||||
return result
|
||||
@@ -608,7 +609,43 @@ def _legacy_resource_permission_keys_batch(
|
||||
permission_keys = snapshot.app.default_permission_keys
|
||||
else:
|
||||
permission_keys = snapshot.dataset.default_permission_keys
|
||||
return {str(resource_id): list(permission_keys) for resource_id in resource_ids}
|
||||
return {resource_id: list(permission_keys) for resource_id in resource_ids}
|
||||
|
||||
|
||||
def _commit_knowledge_fs_rbac_invalidation(
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
member_account_id: str | None = None,
|
||||
) -> None:
|
||||
"""Commit one local invalidation after its remote RBAC mutation succeeded."""
|
||||
from services.knowledge_fs.membership_changes import (
|
||||
KnowledgeFSWorkspaceMembershipChange,
|
||||
apply_workspace_membership_change,
|
||||
apply_workspace_rbac_role_change,
|
||||
)
|
||||
|
||||
try:
|
||||
if member_account_id is None:
|
||||
apply_workspace_rbac_role_change(session=session, tenant_id=tenant_id)
|
||||
else:
|
||||
apply_workspace_membership_change(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
account_ids=(member_account_id,),
|
||||
change=KnowledgeFSWorkspaceMembershipChange.ROLE_CHANGED,
|
||||
)
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
logger.exception(
|
||||
"KnowledgeFS RBAC invalidation failed after remote mutation tenant_id=%s actor_account_id=%s",
|
||||
tenant_id,
|
||||
actor_account_id,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
# ---------- Mutation request models ----------
|
||||
@@ -752,7 +789,7 @@ def _inner_call(
|
||||
|
||||
|
||||
def _resource_id_params(resource_type: RBACResourceType | str, resource_id: str) -> dict[str, str]:
|
||||
resource_type_value = resource_type.value if isinstance(resource_type, RBACResourceType) else str(resource_type)
|
||||
resource_type_value = resource_type.value if isinstance(resource_type, RBACResourceType) else resource_type
|
||||
resource_id = resource_id.strip()
|
||||
if resource_type_value == RBACResourceType.APP.value:
|
||||
return {"resource_type": resource_type_value, "app_id": resource_id}
|
||||
@@ -1703,6 +1740,71 @@ class RBACService:
|
||||
)
|
||||
return data
|
||||
|
||||
class KnowledgeFSRoleMutations:
|
||||
"""Couple successful RBAC writes to durable KnowledgeFS invalidation.
|
||||
|
||||
Enterprise RBAC is remote, so the remote mutation must succeed before
|
||||
this service commits the local epoch and revoke outbox transaction. A
|
||||
local failure is rolled back and propagated so callers cannot mistake
|
||||
a partially invalidated authorization change for success.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def replace_member_roles(
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
member_account_id: str,
|
||||
role_ids: list[str],
|
||||
*,
|
||||
session: Session,
|
||||
) -> MemberRolesResponse:
|
||||
result = RBACService.MemberRoles.replace(
|
||||
tenant_id,
|
||||
actor_account_id,
|
||||
member_account_id,
|
||||
role_ids,
|
||||
session=session,
|
||||
)
|
||||
_commit_knowledge_fs_rbac_invalidation(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
member_account_id=member_account_id,
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def update_role(
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
role_id: str,
|
||||
payload: RoleMutation,
|
||||
*,
|
||||
session: Session,
|
||||
) -> RBACRole:
|
||||
result = RBACService.Roles.update(tenant_id, actor_account_id, role_id, payload)
|
||||
_commit_knowledge_fs_rbac_invalidation(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def delete_role(
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
role_id: str,
|
||||
*,
|
||||
session: Session,
|
||||
) -> None:
|
||||
RBACService.Roles.delete(tenant_id, actor_account_id, role_id)
|
||||
_commit_knowledge_fs_rbac_invalidation(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
)
|
||||
|
||||
class CheckAccess:
|
||||
"""Call the ``/inner/api/rbac/check-access`` endpoint."""
|
||||
|
||||
@@ -1786,6 +1888,41 @@ class RBACService:
|
||||
)
|
||||
return _parse_resource_permission_keys_batch(data, resource_id_key="dataset_id")
|
||||
|
||||
class KnowledgeFSPermissions:
|
||||
"""Batch permission lookup for independent KnowledgeFS control-space IDs."""
|
||||
|
||||
@staticmethod
|
||||
def batch_get(
|
||||
tenant_id: str,
|
||||
account_id: str | None,
|
||||
control_space_ids: list[str],
|
||||
*,
|
||||
session: Session,
|
||||
) -> dict[str, list[str]]:
|
||||
if not control_space_ids:
|
||||
return {}
|
||||
if not dify_config.RBAC_ENABLED:
|
||||
_ = session
|
||||
permissions = [
|
||||
"knowledge_space_read",
|
||||
"knowledge_space_create",
|
||||
"knowledge_space_edit",
|
||||
"knowledge_space_delete",
|
||||
"knowledge_space_access_config",
|
||||
"knowledge_space_api_key_manage",
|
||||
"knowledge_space_document_write",
|
||||
"knowledge_space_query",
|
||||
]
|
||||
return dict.fromkeys(control_space_ids, permissions)
|
||||
data = _inner_call(
|
||||
"POST",
|
||||
f"{_INNER_PREFIX}/knowledge-fs/permission-keys/batch",
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
json={"control_space_ids": control_space_ids},
|
||||
)
|
||||
return _parse_resource_permission_keys_batch(data, resource_id_key="control_space_id")
|
||||
|
||||
class MyPermissions:
|
||||
@staticmethod
|
||||
def get(
|
||||
|
||||
@@ -22,6 +22,7 @@ class AgentToolInvokeCaller(BaseModel):
|
||||
node_execution_id: str | None = None
|
||||
agent_id: str | None = None
|
||||
agent_config_version_id: str | None = None
|
||||
trace_session_id: str | None = Field(default=None, exclude_if=lambda value: value is None)
|
||||
|
||||
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Dify-owned KnowledgeFS control-plane services."""
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Fail-closed Agent and Workflow admission for KnowledgeFS control-spaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSAppSpaceJoinStatus,
|
||||
KnowledgeFSAppSpaceJoinType,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import product_operation_action
|
||||
from services.knowledge_fs.revocation_commands import (
|
||||
KnowledgeFSRevocationCommandPort,
|
||||
KnowledgeFSRevocationCommandProducer,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSAppAdmissionError(RuntimeError):
|
||||
"""The app is not explicitly bound to an enabled KnowledgeFS channel."""
|
||||
|
||||
|
||||
class KnowledgeFSAppPrincipalProfile(NamedTuple):
|
||||
tenant_id: str
|
||||
control_space_id: str
|
||||
app_id: str
|
||||
join_id: str
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType
|
||||
action: str
|
||||
knowledge_space_id: str
|
||||
knowledge_space_revision: int
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
content_policy_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSAppAdmissionService:
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
revocations: KnowledgeFSRevocationCommandPort | None = None,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
|
||||
|
||||
def revoke_binding(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
control_space_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
revoked_by_account_id: str,
|
||||
) -> None:
|
||||
with self._session_maker.begin() as session:
|
||||
join = session.scalar(
|
||||
sa.select(AppKnowledgeFSSpaceJoin)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.app_id == app_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
|
||||
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if join is None:
|
||||
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding was not found")
|
||||
if join.status is not KnowledgeFSAppSpaceJoinStatus.ACTIVE:
|
||||
return
|
||||
join.status = KnowledgeFSAppSpaceJoinStatus.REVOKED
|
||||
join.revision += 1
|
||||
join.revoked_at = naive_utc_now()
|
||||
join.revoked_by_account_id = revoked_by_account_id
|
||||
revision = session.scalar(
|
||||
sa.select(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if revision is None:
|
||||
raise KnowledgeFSAppAdmissionError("KnowledgeFS authorization revision is missing")
|
||||
revision.external_access_epoch += 1
|
||||
self._revocations.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
subject=f"dify-app:{app_id}",
|
||||
reason_code="app_binding_revoked",
|
||||
caller_kinds=(caller_kind.value,),
|
||||
)
|
||||
|
||||
def admit(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
control_space_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
operation_id: str,
|
||||
) -> KnowledgeFSAppPrincipalProfile:
|
||||
try:
|
||||
action = product_operation_action(operation_id)
|
||||
except KeyError as exc:
|
||||
raise KnowledgeFSAppAdmissionError("KnowledgeFS operation is not registered") from exc
|
||||
with self._session_maker() as session:
|
||||
row = session.execute(
|
||||
sa.select(
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSControlSpace,
|
||||
sa.and_(
|
||||
KnowledgeFSControlSpace.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
|
||||
KnowledgeFSControlSpace.id == AppKnowledgeFSSpaceJoin.control_space_id,
|
||||
),
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
sa.and_(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
|
||||
),
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
sa.and_(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.app_id == app_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
|
||||
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
|
||||
AppKnowledgeFSSpaceJoin.status == KnowledgeFSAppSpaceJoinStatus.ACTIVE,
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding is not enabled")
|
||||
join, control_space, policy, revision = row._t
|
||||
channel_enabled = (
|
||||
policy.agent_enabled if caller_kind is KnowledgeFSAppSpaceJoinType.AGENT else policy.workflow_enabled
|
||||
)
|
||||
if (
|
||||
not channel_enabled
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.ACTIVE
|
||||
or control_space.knowledge_space_id is None
|
||||
):
|
||||
raise KnowledgeFSAppAdmissionError("KnowledgeFS app binding is not enabled")
|
||||
return KnowledgeFSAppPrincipalProfile(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
app_id=app_id,
|
||||
join_id=join.id,
|
||||
caller_kind=caller_kind,
|
||||
action=action,
|
||||
knowledge_space_id=control_space.knowledge_space_id,
|
||||
knowledge_space_revision=control_space.knowledge_space_revision,
|
||||
membership_epoch=revision.membership_epoch,
|
||||
space_acl_epoch=revision.space_acl_epoch,
|
||||
external_access_epoch=revision.external_access_epoch,
|
||||
content_policy_revision=revision.content_policy_revision,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSAppAdmissionError",
|
||||
"KnowledgeFSAppAdmissionService",
|
||||
"KnowledgeFSAppPrincipalProfile",
|
||||
]
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Authorized lifecycle management for Agent and Workflow KnowledgeFS bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.enums import AppStatus
|
||||
from models.knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSAppSpaceJoinStatus,
|
||||
KnowledgeFSAppSpaceJoinType,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
)
|
||||
from models.model import App, AppMode
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSAppBindingListResponse,
|
||||
KnowledgeFSAppBindingPayload,
|
||||
KnowledgeFSAppBindingResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
|
||||
from services.knowledge_fs.revocation_commands import KnowledgeFSRevocationCommandPort
|
||||
|
||||
|
||||
class KnowledgeFSAppBindingManagementError(RuntimeError):
|
||||
"""An app binding cannot be created or changed safely."""
|
||||
|
||||
|
||||
class KnowledgeFSAppBindingAuthorizationPort(Protocol):
|
||||
def authorize_control_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> object: ...
|
||||
|
||||
|
||||
class KnowledgeFSAppCatalogPort(Protocol):
|
||||
def supports_binding(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class SQLKnowledgeFSAppCatalog:
|
||||
_MODES_BY_CALLER = {
|
||||
KnowledgeFSAppSpaceJoinType.AGENT: (AppMode.AGENT, AppMode.AGENT_CHAT),
|
||||
KnowledgeFSAppSpaceJoinType.WORKFLOW: (AppMode.ADVANCED_CHAT, AppMode.WORKFLOW),
|
||||
}
|
||||
|
||||
def supports_binding(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
) -> bool:
|
||||
return (
|
||||
session.scalar(
|
||||
sa.select(App.id)
|
||||
.where(
|
||||
App.id == app_id,
|
||||
App.tenant_id == tenant_id,
|
||||
App.status == AppStatus.NORMAL,
|
||||
App.mode.in_(self._MODES_BY_CALLER[caller_kind]),
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSAppBindingManagementService:
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
product: KnowledgeFSAppBindingAuthorizationPort,
|
||||
apps: KnowledgeFSAppCatalogPort,
|
||||
revocations: KnowledgeFSRevocationCommandPort,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._product = product
|
||||
self._apps = apps
|
||||
self._revocations = revocations
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSAppBindingListResponse:
|
||||
self._authorize(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
with self._session_maker() as session:
|
||||
bindings = tuple(
|
||||
session.scalars(
|
||||
sa.select(AppKnowledgeFSSpaceJoin)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
|
||||
)
|
||||
.order_by(AppKnowledgeFSSpaceJoin.app_id, AppKnowledgeFSSpaceJoin.join_type)
|
||||
)
|
||||
)
|
||||
return KnowledgeFSAppBindingListResponse(data=[_response(binding) for binding in bindings])
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSAppBindingPayload,
|
||||
) -> KnowledgeFSAppBindingResponse:
|
||||
self._authorize(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
if not self._apps.supports_binding(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
app_id=payload.app_id,
|
||||
caller_kind=payload.caller_kind,
|
||||
):
|
||||
raise KnowledgeFSAppBindingManagementError("App is not eligible for this KnowledgeFS caller channel")
|
||||
authorization_revision = _authorization_revision(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
binding = session.scalar(
|
||||
sa.select(AppKnowledgeFSSpaceJoin)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
|
||||
AppKnowledgeFSSpaceJoin.app_id == payload.app_id,
|
||||
AppKnowledgeFSSpaceJoin.join_type == payload.caller_kind,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if binding is None:
|
||||
binding = AppKnowledgeFSSpaceJoin(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
app_id=payload.app_id,
|
||||
join_type=payload.caller_kind,
|
||||
created_by_account_id=actor_account_id,
|
||||
)
|
||||
session.add(binding)
|
||||
session.flush()
|
||||
authorization_revision.external_access_epoch += 1
|
||||
elif binding.status is KnowledgeFSAppSpaceJoinStatus.REVOKED:
|
||||
binding.status = KnowledgeFSAppSpaceJoinStatus.ACTIVE
|
||||
binding.revision += 1
|
||||
binding.revoked_at = None
|
||||
binding.revoked_by_account_id = None
|
||||
authorization_revision.external_access_epoch += 1
|
||||
return _response(binding)
|
||||
|
||||
def revoke(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
app_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
) -> None:
|
||||
self._authorize(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
authorization_revision = _authorization_revision(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
binding = session.scalar(
|
||||
sa.select(AppKnowledgeFSSpaceJoin)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == control_space_id,
|
||||
AppKnowledgeFSSpaceJoin.app_id == app_id,
|
||||
AppKnowledgeFSSpaceJoin.join_type == caller_kind,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if binding is None:
|
||||
raise KnowledgeFSAppBindingManagementError("KnowledgeFS app binding was not found")
|
||||
if binding.status is KnowledgeFSAppSpaceJoinStatus.REVOKED:
|
||||
return
|
||||
binding.status = KnowledgeFSAppSpaceJoinStatus.REVOKED
|
||||
binding.revision += 1
|
||||
binding.revoked_at = naive_utc_now()
|
||||
binding.revoked_by_account_id = actor_account_id
|
||||
authorization_revision.external_access_epoch += 1
|
||||
self._revocations.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
subject=f"dify-app:{app_id}",
|
||||
reason_code="app_binding_revoked",
|
||||
caller_kinds=(caller_kind.value,),
|
||||
)
|
||||
|
||||
def _authorize(self, *, tenant_id: str, actor_account_id: str, control_space_id: str) -> None:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
|
||||
|
||||
def _authorization_revision(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSAuthorizationRevision:
|
||||
revision = session.scalar(
|
||||
sa.select(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if revision is None:
|
||||
raise KnowledgeFSAppBindingManagementError("KnowledgeFS authorization revision is missing")
|
||||
return revision
|
||||
|
||||
|
||||
def _response(binding: AppKnowledgeFSSpaceJoin) -> KnowledgeFSAppBindingResponse:
|
||||
return KnowledgeFSAppBindingResponse(
|
||||
id=binding.id,
|
||||
app_id=binding.app_id,
|
||||
caller_kind=binding.join_type,
|
||||
status=binding.status,
|
||||
revision=binding.revision,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSAppBindingManagementError",
|
||||
"KnowledgeFSAppBindingManagementService",
|
||||
"SQLKnowledgeFSAppCatalog",
|
||||
]
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Single production entry point for Agent and Workflow KnowledgeFS capabilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal, cast
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
|
||||
|
||||
from core.app.entities.app_invoke_entities import DifyRunContext
|
||||
from models.knowledge_fs import KnowledgeFSAppSpaceJoinType
|
||||
from services.knowledge_fs.app_admission_service import KnowledgeFSAppAdmissionService
|
||||
from services.knowledge_fs.capability_broker import KnowledgeFSCapabilityBroker, KnowledgeFSIssuedProductCapability
|
||||
from services.knowledge_fs.operation_admission import KnowledgeFSOperationAdmissionService
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSResearchTaskCreatePayload,
|
||||
KnowledgeFSResearchTaskResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
|
||||
from services.knowledge_fs.product_remote import (
|
||||
KnowledgeFSOperationUnavailableError,
|
||||
KnowledgeFSProductRemotePort,
|
||||
KnowledgeFSRemoteJSONRequest,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeResourceRef(BaseModel):
|
||||
"""A typed app configuration reference to one Dify-owned KnowledgeFS control-space."""
|
||||
|
||||
kind: Literal["knowledge_fs"]
|
||||
control_space_id: str = Field(min_length=1, max_length=1_000)
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
@field_validator("control_space_id")
|
||||
@classmethod
|
||||
def normalize_control_space_id(cls, value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized:
|
||||
raise ValueError("KnowledgeFS control-space reference is required")
|
||||
return normalized
|
||||
|
||||
|
||||
class KnowledgeFSAppExecutionCapabilityService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
admission: KnowledgeFSAppAdmissionService,
|
||||
broker: KnowledgeFSCapabilityBroker,
|
||||
operation_admission: KnowledgeFSOperationAdmissionService,
|
||||
remote: KnowledgeFSProductRemotePort,
|
||||
) -> None:
|
||||
self._admission = admission
|
||||
self._broker = broker
|
||||
self._operation_admission = operation_admission
|
||||
self._remote = remote
|
||||
|
||||
def issue(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
control_space_id: str,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
profile = self._admission.admit(
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
control_space_id=control_space_id,
|
||||
caller_kind=caller_kind,
|
||||
operation_id=operation_id,
|
||||
)
|
||||
return self._broker.issue_app(
|
||||
profile=profile,
|
||||
operation_id=operation_id,
|
||||
resource_id=resource_id,
|
||||
trace_id=trace_id,
|
||||
)
|
||||
|
||||
def create_research_task(
|
||||
self,
|
||||
*,
|
||||
run_context: DifyRunContext,
|
||||
caller_kind: KnowledgeFSAppSpaceJoinType,
|
||||
resource: KnowledgeResourceRef,
|
||||
payload: KnowledgeFSResearchTaskCreatePayload,
|
||||
) -> KnowledgeFSResearchTaskResponse:
|
||||
"""Create one Research task through app admission and a bounded product operation."""
|
||||
|
||||
operation_id = "createResearchTask"
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
|
||||
if (
|
||||
not is_product_operation_ready(operation_id)
|
||||
or operation.transport != "json"
|
||||
or operation.kfs_path is None
|
||||
or "{" in operation.kfs_path
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app Research task creation is unavailable")
|
||||
with self._admitted(tenant_id=run_context.tenant_id, operation_id=operation_id):
|
||||
issued = self.issue(
|
||||
tenant_id=run_context.tenant_id,
|
||||
app_id=run_context.app_id,
|
||||
control_space_id=resource.control_space_id,
|
||||
caller_kind=caller_kind,
|
||||
operation_id=operation_id,
|
||||
trace_id=run_context.trace_session_id,
|
||||
)
|
||||
remote_payload = cast(
|
||||
dict[str, JsonValue],
|
||||
payload.model_dump(mode="json", exclude_none=True, by_alias=True),
|
||||
)
|
||||
remote_payload["knowledgeSpaceId"] = issued.knowledge_space_id
|
||||
raw = self._remote.execute_json(
|
||||
KnowledgeFSRemoteJSONRequest(
|
||||
operation_id=operation_id,
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
namespace_id=run_context.tenant_id,
|
||||
knowledge_space_id=issued.knowledge_space_id,
|
||||
capability_token=issued.token,
|
||||
trace_id=issued.trace_id,
|
||||
payload=remote_payload,
|
||||
)
|
||||
)
|
||||
response = KnowledgeFSResearchTaskResponse.model_validate(raw)
|
||||
return response
|
||||
|
||||
@contextmanager
|
||||
def _admitted(self, *, tenant_id: str, operation_id: str) -> Generator[None, None, None]:
|
||||
charge = self._operation_admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
|
||||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
charge.refund()
|
||||
raise
|
||||
else:
|
||||
charge.commit()
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSAppExecutionCapabilityService", "KnowledgeResourceRef"]
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Issue one namespace Capability over an exact, already-authorized Space batch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpaceState,
|
||||
)
|
||||
from services.knowledge_fs.cutover_runtime_gate import KnowledgeFSWorkspaceRuntimeGatePort
|
||||
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
|
||||
from services.knowledge_fs_capability import (
|
||||
CapabilityAuthzRevision,
|
||||
CapabilityIssueRequest,
|
||||
CapabilityResource,
|
||||
IssuedKnowledgeFSCapability,
|
||||
)
|
||||
|
||||
MAX_BATCH_SPACE_SUMMARIES = 100
|
||||
|
||||
|
||||
class KnowledgeFSBatchSpaceBinding(NamedTuple):
|
||||
control_space_id: str
|
||||
knowledge_space_id: str
|
||||
|
||||
|
||||
class KnowledgeFSIssuedBatchCapability(NamedTuple):
|
||||
token: str
|
||||
expires_at: datetime
|
||||
knowledge_space_ids: tuple[str, ...]
|
||||
trace_id: str
|
||||
|
||||
|
||||
class KnowledgeFSBatchCapabilityIssuerPort(Protocol):
|
||||
def issue_interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
bindings: tuple[KnowledgeFSBatchSpaceBinding, ...],
|
||||
trace_id: str,
|
||||
) -> KnowledgeFSIssuedBatchCapability: ...
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuerPort(Protocol):
|
||||
def issue(self, request: CapabilityIssueRequest) -> IssuedKnowledgeFSCapability: ...
|
||||
|
||||
|
||||
class KnowledgeFSBatchCapabilityBroker:
|
||||
"""Revalidates exact tenant/registration bindings before signing the scoped ID set."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
cutover_gate: KnowledgeFSWorkspaceRuntimeGatePort,
|
||||
issuer: KnowledgeFSCapabilityIssuerPort | None,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._cutover_gate = cutover_gate
|
||||
self._issuer = issuer
|
||||
|
||||
def issue_interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
bindings: tuple[KnowledgeFSBatchSpaceBinding, ...],
|
||||
trace_id: str,
|
||||
) -> KnowledgeFSIssuedBatchCapability:
|
||||
self._cutover_gate.require_capability_v2(tenant_id=tenant_id)
|
||||
if self._issuer is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Capability v2 is disabled")
|
||||
_validate_bindings(bindings)
|
||||
control_space_ids = tuple(binding.control_space_id for binding in bindings)
|
||||
with self._session_maker() as session:
|
||||
rows = session.execute(
|
||||
sa.select(KnowledgeFSControlSpace, KnowledgeFSAuthorizationRevision)
|
||||
.join(
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
sa.and_(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == KnowledgeFSControlSpace.tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == KnowledgeFSControlSpace.id,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id.in_(control_space_ids),
|
||||
KnowledgeFSControlSpace.state == KnowledgeFSControlSpaceState.ACTIVE,
|
||||
)
|
||||
).all()
|
||||
|
||||
rows_by_control_id = {space.id: (space, revision) for space, revision in rows}
|
||||
if len(rows_by_control_id) != len(bindings):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch authorization snapshot is incomplete")
|
||||
ordered_revisions: list[KnowledgeFSAuthorizationRevision] = []
|
||||
for binding in bindings:
|
||||
space, revision = rows_by_control_id[binding.control_space_id]
|
||||
if space.knowledge_space_id != binding.knowledge_space_id:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch registration changed")
|
||||
ordered_revisions.append(revision)
|
||||
|
||||
knowledge_space_ids = tuple(binding.knowledge_space_id for binding in bindings)
|
||||
digest = hashlib.sha256("\0".join((*control_space_ids, *knowledge_space_ids)).encode()).hexdigest()[:24]
|
||||
issued = self._issuer.issue(
|
||||
CapabilityIssueRequest(
|
||||
actor=f"dify-account:{account_id}",
|
||||
authz_revision=CapabilityAuthzRevision(
|
||||
membership_epoch=max(revision.membership_epoch for revision in ordered_revisions),
|
||||
space_acl_epoch=max(revision.space_acl_epoch for revision in ordered_revisions),
|
||||
external_access_epoch=max(revision.external_access_epoch for revision in ordered_revisions),
|
||||
credential_revision=None,
|
||||
),
|
||||
caller_kind="interactive",
|
||||
content_policy_revision=max(revision.content_policy_revision for revision in ordered_revisions),
|
||||
content_scope_ids=knowledge_space_ids,
|
||||
control_space_id=bindings[0].control_space_id,
|
||||
grant_id=f"account:{account_id}:batch:{digest}",
|
||||
namespace_id=tenant_id,
|
||||
operation_id="batchKnowledgeSpaceProductSummaries",
|
||||
principal_id=account_id,
|
||||
resource=CapabilityResource(type="namespace", id=tenant_id),
|
||||
trace_id=trace_id,
|
||||
)
|
||||
)
|
||||
return KnowledgeFSIssuedBatchCapability(
|
||||
token=issued.token,
|
||||
expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC),
|
||||
knowledge_space_ids=knowledge_space_ids,
|
||||
trace_id=trace_id,
|
||||
)
|
||||
|
||||
|
||||
def _validate_bindings(bindings: tuple[KnowledgeFSBatchSpaceBinding, ...]) -> None:
|
||||
if not bindings or len(bindings) > MAX_BATCH_SPACE_SUMMARIES:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch size must be between 1 and 100")
|
||||
control_ids = tuple(binding.control_space_id.strip() for binding in bindings)
|
||||
space_ids = tuple(binding.knowledge_space_id.strip() for binding in bindings)
|
||||
if (
|
||||
any(not identifier for identifier in (*control_ids, *space_ids))
|
||||
or len(set(control_ids)) != len(control_ids)
|
||||
or len(set(space_ids)) != len(space_ids)
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch bindings must be unique and non-empty")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MAX_BATCH_SPACE_SUMMARIES",
|
||||
"KnowledgeFSBatchCapabilityBroker",
|
||||
"KnowledgeFSBatchCapabilityIssuerPort",
|
||||
"KnowledgeFSBatchSpaceBinding",
|
||||
"KnowledgeFSIssuedBatchCapability",
|
||||
]
|
||||
@@ -0,0 +1,492 @@
|
||||
"""Issue operation capabilities from already-authorized Dify product state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import NamedTuple, Protocol, cast
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSApiCredentialStatus,
|
||||
KnowledgeFSAppSpaceJoinStatus,
|
||||
KnowledgeFSAppSpaceJoinType,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
from repositories.sqlalchemy_knowledge_fs_capability_issuance_reservation_repository import (
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository,
|
||||
)
|
||||
from services.knowledge_fs.app_admission_service import KnowledgeFSAppPrincipalProfile
|
||||
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
|
||||
from services.knowledge_fs.cutover_runtime_gate import KnowledgeFSWorkspaceRuntimeGatePort
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
KnowledgeFSProductOperation,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
|
||||
from services.knowledge_fs.product_service import KnowledgeFSProductService
|
||||
from services.knowledge_fs_capability import (
|
||||
KNOWLEDGE_FS_CAPABILITY_OPERATIONS,
|
||||
CapabilityAuthzRevision,
|
||||
CapabilityCallerKind,
|
||||
CapabilityIssueRequest,
|
||||
CapabilityResource,
|
||||
IssuedKnowledgeFSCapability,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuerPort(Protocol):
|
||||
def issue(self, request: CapabilityIssueRequest) -> IssuedKnowledgeFSCapability: ...
|
||||
|
||||
|
||||
class KnowledgeFSIssuedProductCapability(NamedTuple):
|
||||
token: str
|
||||
expires_at: datetime
|
||||
operation_id: str
|
||||
knowledge_space_id: str
|
||||
knowledge_space_revision: int
|
||||
trace_id: str
|
||||
|
||||
|
||||
class KnowledgeFSAuthorizationSnapshot(NamedTuple):
|
||||
authz_revision: CapabilityAuthzRevision
|
||||
content_policy_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityBroker:
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
cutover_gate: KnowledgeFSWorkspaceRuntimeGatePort,
|
||||
product: KnowledgeFSProductService,
|
||||
issuer: KnowledgeFSCapabilityIssuerPort | None,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._cutover_gate = cutover_gate
|
||||
self._product = product
|
||||
self._issuer = issuer
|
||||
|
||||
def issue_interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
self._cutover_gate.require_capability_v2(tenant_id=tenant_id)
|
||||
product_operation, capability_operation_id = _operation_contract(operation_id)
|
||||
issuer = self._require_issuer()
|
||||
normalized_trace_id = _trace_id(trace_id)
|
||||
with self._session_maker.begin() as session:
|
||||
revision = _lock_authorization_revision(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
authorized = self._product.authorize_control_space_in_session(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=product_operation.permission,
|
||||
require_active=True,
|
||||
)
|
||||
space = authorized.control_space
|
||||
knowledge_space_id = space.knowledge_space_id
|
||||
if knowledge_space_id is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS control-space is not registered")
|
||||
request = _issue_request(
|
||||
capability_operation_id=capability_operation_id,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
principal_id=account_id,
|
||||
actor=f"dify-account:{account_id}",
|
||||
caller_kind="interactive",
|
||||
credential_revision=None,
|
||||
revision=revision,
|
||||
resource_id=resource_id,
|
||||
trace_id=normalized_trace_id,
|
||||
)
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
|
||||
knowledge_space_revision = space.knowledge_space_revision
|
||||
return self._issue_reserved(
|
||||
issuer=issuer,
|
||||
request=request,
|
||||
operation_id=operation_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
|
||||
def issue_service(
|
||||
self,
|
||||
*,
|
||||
profile: KnowledgeFSServiceCredentialProfile,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
self._cutover_gate.require_capability_v2(tenant_id=profile.tenant_id)
|
||||
_, capability_operation_id = _operation_contract(operation_id)
|
||||
issuer = self._require_issuer()
|
||||
normalized_trace_id = _trace_id(trace_id)
|
||||
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
|
||||
with self._session_maker.begin() as session:
|
||||
revision = _lock_authorization_revision(
|
||||
session,
|
||||
tenant_id=profile.tenant_id,
|
||||
control_space_id=profile.control_space_id,
|
||||
)
|
||||
credential, space, policy = _load_service_authorization(session, profile=profile)
|
||||
if (
|
||||
credential.status is not KnowledgeFSApiCredentialStatus.ACTIVE
|
||||
or credential.principal != profile.principal_id
|
||||
or (credential.expires_at is not None and credential.expires_at <= naive_utc_now())
|
||||
or policy is None
|
||||
or not policy.service_api_enabled
|
||||
or capability_operation.action not in credential.allowed_actions
|
||||
or space.state is not KnowledgeFSControlSpaceState.ACTIVE
|
||||
or space.knowledge_space_id is None
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS credential is no longer authorized")
|
||||
knowledge_space_id = space.knowledge_space_id
|
||||
request = _issue_request(
|
||||
capability_operation_id=capability_operation_id,
|
||||
tenant_id=profile.tenant_id,
|
||||
control_space_id=profile.control_space_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
principal_id=credential.principal,
|
||||
actor=f"dify-kfs-credential:{credential.principal}",
|
||||
caller_kind="service",
|
||||
credential_revision=credential.revision,
|
||||
revision=revision,
|
||||
resource_id=resource_id,
|
||||
trace_id=normalized_trace_id,
|
||||
)
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
|
||||
knowledge_space_revision = space.knowledge_space_revision
|
||||
return self._issue_reserved(
|
||||
issuer=issuer,
|
||||
request=request,
|
||||
operation_id=operation_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
|
||||
def issue_app(
|
||||
self,
|
||||
*,
|
||||
profile: KnowledgeFSAppPrincipalProfile,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
self._cutover_gate.require_capability_v2(tenant_id=profile.tenant_id)
|
||||
_, capability_operation_id = _operation_contract(operation_id)
|
||||
issuer = self._require_issuer()
|
||||
normalized_trace_id = _trace_id(trace_id)
|
||||
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
|
||||
with self._session_maker.begin() as session:
|
||||
revision = _lock_authorization_revision(
|
||||
session,
|
||||
tenant_id=profile.tenant_id,
|
||||
control_space_id=profile.control_space_id,
|
||||
)
|
||||
join, space, policy = _load_app_authorization(session, profile=profile)
|
||||
caller_kind = _app_caller_kind(profile.caller_kind)
|
||||
channel_enabled = (
|
||||
policy.agent_enabled
|
||||
if profile.caller_kind is KnowledgeFSAppSpaceJoinType.AGENT
|
||||
else policy.workflow_enabled
|
||||
)
|
||||
if (
|
||||
join.status is not KnowledgeFSAppSpaceJoinStatus.ACTIVE
|
||||
or capability_operation.action != profile.action
|
||||
or not channel_enabled
|
||||
or space.state is not KnowledgeFSControlSpaceState.ACTIVE
|
||||
or space.knowledge_space_id is None
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app binding is no longer authorized")
|
||||
knowledge_space_id = space.knowledge_space_id
|
||||
request = _issue_request(
|
||||
capability_operation_id=capability_operation_id,
|
||||
tenant_id=profile.tenant_id,
|
||||
control_space_id=profile.control_space_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
principal_id=profile.app_id,
|
||||
actor=f"dify-app:{profile.app_id}",
|
||||
caller_kind=caller_kind,
|
||||
credential_revision=None,
|
||||
revision=revision,
|
||||
resource_id=resource_id,
|
||||
trace_id=normalized_trace_id,
|
||||
)
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).reserve(request)
|
||||
knowledge_space_revision = space.knowledge_space_revision
|
||||
return self._issue_reserved(
|
||||
issuer=issuer,
|
||||
request=request,
|
||||
operation_id=operation_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
|
||||
def _require_issuer(self) -> KnowledgeFSCapabilityIssuerPort:
|
||||
if self._issuer is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Capability v2 is disabled")
|
||||
return self._issuer
|
||||
|
||||
def _issue_reserved(
|
||||
self,
|
||||
*,
|
||||
issuer: KnowledgeFSCapabilityIssuerPort,
|
||||
request: CapabilityIssueRequest,
|
||||
operation_id: str,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
try:
|
||||
issued = issuer.issue(request)
|
||||
with self._session_maker.begin() as session:
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).mark_issued(
|
||||
tenant_id=request.namespace_id,
|
||||
grant_id=request.grant_id,
|
||||
issued_at=datetime.fromtimestamp(issued.claims.iat, tz=UTC).replace(tzinfo=None),
|
||||
token_expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC).replace(tzinfo=None),
|
||||
)
|
||||
except Exception as issuance_error:
|
||||
try:
|
||||
with self._session_maker.begin() as session:
|
||||
SQLAlchemyKnowledgeFSCapabilityIssuanceReservationRepository(session).mark_failed(
|
||||
tenant_id=request.namespace_id,
|
||||
grant_id=request.grant_id,
|
||||
failed_at=naive_utc_now(),
|
||||
failure_code=type(issuance_error).__name__,
|
||||
)
|
||||
except Exception as terminal_error:
|
||||
raise KnowledgeFSOperationUnavailableError(
|
||||
"KnowledgeFS capability issuance terminal state could not be persisted"
|
||||
) from terminal_error
|
||||
raise
|
||||
return KnowledgeFSIssuedProductCapability(
|
||||
token=issued.token,
|
||||
expires_at=datetime.fromtimestamp(issued.claims.exp, tz=UTC),
|
||||
operation_id=operation_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
trace_id=request.trace_id,
|
||||
)
|
||||
|
||||
|
||||
def _lock_authorization_revision(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSAuthorizationSnapshot:
|
||||
revision = session.scalar(
|
||||
sa.select(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if revision is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS authorization revision is missing")
|
||||
result = session.execute(
|
||||
sa.update(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.id == revision.id,
|
||||
KnowledgeFSAuthorizationRevision.membership_epoch == revision.membership_epoch,
|
||||
KnowledgeFSAuthorizationRevision.space_acl_epoch == revision.space_acl_epoch,
|
||||
KnowledgeFSAuthorizationRevision.external_access_epoch == revision.external_access_epoch,
|
||||
KnowledgeFSAuthorizationRevision.content_policy_revision == revision.content_policy_revision,
|
||||
KnowledgeFSAuthorizationRevision.revoke_sequence == revision.revoke_sequence,
|
||||
)
|
||||
.values(updated_at=sa.func.current_timestamp())
|
||||
)
|
||||
if cast(CursorResult[tuple[object, ...]], result).rowcount != 1:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS authorization revision changed during issuance")
|
||||
return KnowledgeFSAuthorizationSnapshot(
|
||||
authz_revision=CapabilityAuthzRevision(
|
||||
membership_epoch=revision.membership_epoch,
|
||||
space_acl_epoch=revision.space_acl_epoch,
|
||||
external_access_epoch=revision.external_access_epoch,
|
||||
credential_revision=None,
|
||||
),
|
||||
content_policy_revision=revision.content_policy_revision,
|
||||
)
|
||||
|
||||
|
||||
def _load_service_authorization(
|
||||
session: Session,
|
||||
*,
|
||||
profile: KnowledgeFSServiceCredentialProfile,
|
||||
) -> tuple[KnowledgeFSApiCredential, KnowledgeFSControlSpace, KnowledgeFSExternalAccessPolicy | None]:
|
||||
row = session.execute(
|
||||
sa.select(
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSControlSpace,
|
||||
sa.and_(
|
||||
KnowledgeFSControlSpace.tenant_id == KnowledgeFSApiCredential.tenant_id,
|
||||
KnowledgeFSControlSpace.id == KnowledgeFSApiCredential.control_space_id,
|
||||
),
|
||||
)
|
||||
.outerjoin(
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
sa.and_(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == KnowledgeFSApiCredential.tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == KnowledgeFSApiCredential.control_space_id,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
KnowledgeFSApiCredential.tenant_id == profile.tenant_id,
|
||||
KnowledgeFSApiCredential.control_space_id == profile.control_space_id,
|
||||
KnowledgeFSApiCredential.id == profile.credential_id,
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS credential is no longer authorized")
|
||||
return row._t
|
||||
|
||||
|
||||
def _load_app_authorization(
|
||||
session: Session,
|
||||
*,
|
||||
profile: KnowledgeFSAppPrincipalProfile,
|
||||
) -> tuple[AppKnowledgeFSSpaceJoin, KnowledgeFSControlSpace, KnowledgeFSExternalAccessPolicy]:
|
||||
row = session.execute(
|
||||
sa.select(
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSControlSpace,
|
||||
sa.and_(
|
||||
KnowledgeFSControlSpace.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
|
||||
KnowledgeFSControlSpace.id == AppKnowledgeFSSpaceJoin.control_space_id,
|
||||
),
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
sa.and_(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == AppKnowledgeFSSpaceJoin.tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == AppKnowledgeFSSpaceJoin.control_space_id,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
AppKnowledgeFSSpaceJoin.tenant_id == profile.tenant_id,
|
||||
AppKnowledgeFSSpaceJoin.control_space_id == profile.control_space_id,
|
||||
AppKnowledgeFSSpaceJoin.app_id == profile.app_id,
|
||||
AppKnowledgeFSSpaceJoin.id == profile.join_id,
|
||||
AppKnowledgeFSSpaceJoin.join_type == profile.caller_kind,
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS app binding is no longer authorized")
|
||||
return row._t
|
||||
|
||||
|
||||
def _issue_request(
|
||||
*,
|
||||
capability_operation_id: str,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
knowledge_space_id: str,
|
||||
principal_id: str,
|
||||
actor: str,
|
||||
caller_kind: CapabilityCallerKind,
|
||||
credential_revision: int | None,
|
||||
revision: KnowledgeFSAuthorizationSnapshot,
|
||||
resource_id: str | None,
|
||||
trace_id: str,
|
||||
) -> CapabilityIssueRequest:
|
||||
capability_operation = KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id]
|
||||
if capability_operation.resource_type == "knowledge_space":
|
||||
resource = CapabilityResource(type="knowledge_space", id=knowledge_space_id)
|
||||
elif capability_operation.resource_type in {
|
||||
"document",
|
||||
"job",
|
||||
"query",
|
||||
"research_task",
|
||||
"source",
|
||||
"upload_session",
|
||||
}:
|
||||
if resource_id is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS child resource is required")
|
||||
resource = CapabilityResource(
|
||||
type=capability_operation.resource_type,
|
||||
id=resource_id,
|
||||
parent_id=knowledge_space_id,
|
||||
)
|
||||
else:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product resource resolver is not registered")
|
||||
grant_id = str(
|
||||
uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
f"dify-kfs-capability:{tenant_id}:{caller_kind}:{principal_id}:{trace_id}",
|
||||
)
|
||||
)
|
||||
return CapabilityIssueRequest(
|
||||
actor=actor,
|
||||
authz_revision=revision.authz_revision.model_copy(update={"credential_revision": credential_revision}),
|
||||
caller_kind=caller_kind,
|
||||
content_policy_revision=revision.content_policy_revision,
|
||||
control_space_id=control_space_id,
|
||||
grant_id=grant_id,
|
||||
namespace_id=tenant_id,
|
||||
operation_id=capability_operation_id,
|
||||
principal_id=principal_id,
|
||||
resource=resource,
|
||||
trace_id=trace_id,
|
||||
)
|
||||
|
||||
|
||||
def _trace_id(trace_id: str | None) -> str:
|
||||
normalized = (trace_id or str(uuid.uuid4())).strip()
|
||||
if not normalized:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS capability trace id is required")
|
||||
return normalized
|
||||
|
||||
|
||||
def _operation_contract(operation_id: str) -> tuple[KnowledgeFSProductOperation, str]:
|
||||
product_operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
|
||||
if product_operation is None or not is_product_operation_ready(operation_id):
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
|
||||
capability_operation_id = product_operation.capability_operation_id
|
||||
if capability_operation_id is None:
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
|
||||
return product_operation, capability_operation_id
|
||||
|
||||
|
||||
def _app_caller_kind(caller_kind: KnowledgeFSAppSpaceJoinType) -> CapabilityCallerKind:
|
||||
if caller_kind is KnowledgeFSAppSpaceJoinType.AGENT:
|
||||
return "agent"
|
||||
return "workflow"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCapabilityBroker",
|
||||
"KnowledgeFSCapabilityIssuerPort",
|
||||
"KnowledgeFSIssuedProductCapability",
|
||||
]
|
||||
@@ -0,0 +1,963 @@
|
||||
"""Default-deny P9 cleanup readiness, approval, and irreversible-start fence.
|
||||
|
||||
This module never executes schema or data deletion. It validates operator-supplied
|
||||
production evidence, persists four-eyes approval, and atomically closes rollback
|
||||
before an external, separately controlled migration may begin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Literal, NamedTuple
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import ensure_naive_utc, naive_utc_now
|
||||
from models.knowledge_fs_cleanup import (
|
||||
KnowledgeFSCleanupAuthorization,
|
||||
KnowledgeFSCleanupAuthorizationStatus,
|
||||
KnowledgeFSCleanupTarget,
|
||||
)
|
||||
from models.knowledge_fs_cutover import (
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
knowledge_fs_cutover_smoke_results_passed,
|
||||
)
|
||||
from repositories.knowledge_fs_cleanup_repository import KnowledgeFSCleanupAuthorizationCASUpdate
|
||||
from repositories.knowledge_fs_cutover_repository import KnowledgeFSCutoverCASUpdate
|
||||
from repositories.sqlalchemy_knowledge_fs_cleanup_repository import (
|
||||
SQLAlchemyKnowledgeFSCleanupAuthorizationRepository,
|
||||
)
|
||||
from repositories.sqlalchemy_knowledge_fs_cutover_repository import SQLAlchemyKnowledgeFSCutoverRepository
|
||||
from services.knowledge_fs.cutover import (
|
||||
knowledge_fs_remote_activation_evidence_consistent,
|
||||
knowledge_fs_remote_freeze_evidence_consistent,
|
||||
)
|
||||
|
||||
_PLAN_SCHEMA_VERSION = "knowledge-fs-p9-cleanup/v1"
|
||||
_START_CONFIRMATION = "START-KNOWLEDGE-FS-IRREVERSIBLE-CLEANUP"
|
||||
_MAX_APPROVAL_LIFETIME = timedelta(hours=24)
|
||||
_REQUIRED_TARGETS = frozenset(KnowledgeFSCleanupTarget)
|
||||
|
||||
|
||||
def cleanup_workspace_cohort_digest(tenant_ids: Iterable[str | UUID]) -> str:
|
||||
"""Return the canonical digest for the complete, sorted cleanup cohort."""
|
||||
|
||||
canonical_ids = sorted(str(tenant_id) for tenant_id in tenant_ids)
|
||||
canonical = json.dumps(canonical_ids, ensure_ascii=False, separators=(",", ":")).encode()
|
||||
return f"sha256:{hashlib.sha256(canonical).hexdigest()}"
|
||||
|
||||
|
||||
class StrictCleanupInput(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class CleanupReadinessEvidenceInput(StrictCleanupInput):
|
||||
schema_version: Literal["knowledge-fs-p9-cleanup/v1"]
|
||||
tenant_id: UUID
|
||||
request_id: UUID
|
||||
expected_cas_version: int = Field(ge=0)
|
||||
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
targets: list[KnowledgeFSCleanupTarget] = Field(min_length=6, max_length=6)
|
||||
expected_workspace_count: int = Field(gt=0, le=100_000)
|
||||
workspace_tenant_ids: list[UUID] = Field(min_length=1, max_length=100_000)
|
||||
workspace_cohort_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
workspace_inventory_reference: str = Field(min_length=1, max_length=1024)
|
||||
evidence_environment: Literal["production"]
|
||||
observed_at: datetime
|
||||
requested_at: datetime
|
||||
legacy_route_zero_window_started_at: datetime
|
||||
legacy_route_zero_window_ends_at: datetime
|
||||
rollback_window_seconds: int = Field(gt=0)
|
||||
legacy_route_calls: int = Field(ge=0)
|
||||
legacy_access_route_calls: int = Field(ge=0)
|
||||
legacy_member_route_calls: int = Field(ge=0)
|
||||
legacy_api_key_route_calls: int = Field(ge=0)
|
||||
legacy_route_metric_reference: str = Field(min_length=1, max_length=1024)
|
||||
maximum_token_expires_at: datetime
|
||||
backup_reference: str = Field(min_length=1, max_length=1024)
|
||||
backup_verified_at: datetime
|
||||
restore_drill_reference: str = Field(min_length=1, max_length=1024)
|
||||
restore_drill_verified_at: datetime
|
||||
change_window_approval_reference: str = Field(min_length=1, max_length=1024)
|
||||
requested_by_account_id: UUID
|
||||
|
||||
@field_validator(
|
||||
"observed_at",
|
||||
"requested_at",
|
||||
"legacy_route_zero_window_started_at",
|
||||
"legacy_route_zero_window_ends_at",
|
||||
"maximum_token_expires_at",
|
||||
"backup_verified_at",
|
||||
"restore_drill_verified_at",
|
||||
)
|
||||
@classmethod
|
||||
def require_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("cleanup evidence timestamps require an explicit timezone")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_complete_unique_plan(self) -> CleanupReadinessEvidenceInput:
|
||||
if frozenset(self.targets) != _REQUIRED_TARGETS or len(set(self.targets)) != len(self.targets):
|
||||
raise ValueError("cleanup plan must contain every destructive target exactly once")
|
||||
tenant_ids = [str(tenant_id) for tenant_id in self.workspace_tenant_ids]
|
||||
if tenant_ids != sorted(tenant_ids) or len(tenant_ids) != len(set(tenant_ids)):
|
||||
raise ValueError("cleanup Workspace cohort must be unique and sorted")
|
||||
if str(self.tenant_id) not in tenant_ids:
|
||||
raise ValueError("cleanup authorization tenant must be in the Workspace cohort")
|
||||
if self.expected_workspace_count != len(tenant_ids):
|
||||
raise ValueError("cleanup Workspace count does not match the cohort")
|
||||
if self.workspace_cohort_digest != cleanup_workspace_cohort_digest(tenant_ids):
|
||||
raise ValueError("cleanup Workspace cohort digest does not match the canonical tenant list")
|
||||
if self.legacy_route_calls != (
|
||||
self.legacy_access_route_calls + self.legacy_member_route_calls + self.legacy_api_key_route_calls
|
||||
):
|
||||
raise ValueError("legacy route total does not match access/member/API-key counts")
|
||||
return self
|
||||
|
||||
|
||||
class CleanupApprovalInput(StrictCleanupInput):
|
||||
schema_version: Literal["knowledge-fs-p9-cleanup-approval/v1"]
|
||||
tenant_id: UUID
|
||||
request_id: UUID
|
||||
expected_cas_version: int = Field(ge=0)
|
||||
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
approved_by_account_id: UUID
|
||||
approved_at: datetime
|
||||
approval_expires_at: datetime
|
||||
|
||||
@field_validator("approved_at", "approval_expires_at")
|
||||
@classmethod
|
||||
def require_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("cleanup approval timestamps require an explicit timezone")
|
||||
return value
|
||||
|
||||
|
||||
class CleanupStartInput(StrictCleanupInput):
|
||||
schema_version: Literal["knowledge-fs-p9-cleanup-start/v1"]
|
||||
tenant_id: UUID
|
||||
request_id: UUID
|
||||
expected_cas_version: int = Field(ge=0)
|
||||
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
started_by_account_id: UUID
|
||||
started_at: datetime
|
||||
confirmation: Literal["START-KNOWLEDGE-FS-IRREVERSIBLE-CLEANUP"]
|
||||
|
||||
@field_validator("started_at")
|
||||
@classmethod
|
||||
def require_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("cleanup start timestamp requires an explicit timezone")
|
||||
return value
|
||||
|
||||
|
||||
class CleanupArchivedRowCounts(StrictCleanupInput):
|
||||
knowledge_space_members: int = Field(ge=0)
|
||||
knowledge_space_access_policies: int = Field(ge=0)
|
||||
knowledge_space_access_policy_members: int = Field(ge=0)
|
||||
knowledge_space_api_access: int = Field(ge=0)
|
||||
knowledge_space_api_keys: int = Field(ge=0)
|
||||
knowledge_space_permission_snapshots: int = Field(ge=0)
|
||||
|
||||
|
||||
class CleanupCompletionChecks(StrictCleanupInput):
|
||||
legacy_foreign_keys_remaining: Literal[0]
|
||||
legacy_tables_remaining: Literal[0]
|
||||
legacy_routes_registered: Literal[0]
|
||||
legacy_v1_auth_acceptances: Literal[0]
|
||||
raw_proxy_routes_registered: Literal[0]
|
||||
post_cleanup_smoke_passed: Literal[True]
|
||||
recovery_material_verified: Literal[True]
|
||||
|
||||
|
||||
class CleanupCompletionEvidenceInput(StrictCleanupInput):
|
||||
schema_version: Literal["knowledge-fs-p9-cleanup-completion/v1"]
|
||||
tenant_id: UUID
|
||||
request_id: UUID
|
||||
expected_cas_version: int = Field(ge=0)
|
||||
plan_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
migration_bundle_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
database_engine: Literal["postgresql", "tidb"]
|
||||
migration_revision: str = Field(min_length=1, max_length=255)
|
||||
archived_row_counts: CleanupArchivedRowCounts
|
||||
checks: CleanupCompletionChecks
|
||||
archive_reference: str = Field(min_length=1, max_length=1024)
|
||||
catalog_verification_reference: str = Field(min_length=1, max_length=1024)
|
||||
route_metric_reference: str = Field(min_length=1, max_length=1024)
|
||||
post_cleanup_smoke_reference: str = Field(min_length=1, max_length=1024)
|
||||
recovery_material_reference: str = Field(min_length=1, max_length=1024)
|
||||
completed_by_account_id: UUID
|
||||
completed_at: datetime
|
||||
|
||||
@field_validator("completed_at")
|
||||
@classmethod
|
||||
def require_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("cleanup completion timestamp requires an explicit timezone")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_reviewed_bundle(self) -> CleanupCompletionEvidenceInput:
|
||||
if self.migration_bundle_digest != self.plan_digest:
|
||||
raise ValueError("cleanup completion bundle digest must match the approved plan")
|
||||
return self
|
||||
|
||||
|
||||
class CleanupReadinessReport(NamedTuple):
|
||||
tenant_id: str
|
||||
request_id: str
|
||||
ready: bool
|
||||
reasons: tuple[str, ...]
|
||||
status: str
|
||||
applied: bool
|
||||
replayed: bool
|
||||
operator_attested_evidence: bool
|
||||
|
||||
|
||||
class CleanupApprovalReport(NamedTuple):
|
||||
tenant_id: str
|
||||
request_id: str
|
||||
approvable: bool
|
||||
reasons: tuple[str, ...]
|
||||
status: str
|
||||
applied: bool
|
||||
replayed: bool
|
||||
|
||||
|
||||
class CleanupStartReport(NamedTuple):
|
||||
tenant_id: str
|
||||
request_id: str
|
||||
startable: bool
|
||||
reasons: tuple[str, ...]
|
||||
status: str
|
||||
applied: bool
|
||||
replayed: bool
|
||||
irreversible_cleanup_at: str | None
|
||||
destructive_actions_executed: bool
|
||||
|
||||
|
||||
class CleanupCompletionReport(NamedTuple):
|
||||
tenant_id: str
|
||||
request_id: str
|
||||
completable: bool
|
||||
reasons: tuple[str, ...]
|
||||
status: str
|
||||
applied: bool
|
||||
replayed: bool
|
||||
destructive_actions_executed: bool
|
||||
|
||||
|
||||
class KnowledgeFSCleanupError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSCleanupNotFoundError(KnowledgeFSCleanupError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSCleanupConflictError(KnowledgeFSCleanupError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSCleanupGateBlockedError(KnowledgeFSCleanupError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSCleanupService:
|
||||
"""Persist readiness and approval without executing destructive operations."""
|
||||
|
||||
_session_maker: sessionmaker[Session]
|
||||
_clock: Callable[[], datetime]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
clock: Callable[[], datetime] = naive_utc_now,
|
||||
):
|
||||
self._session_maker = session_maker
|
||||
self._clock = clock
|
||||
|
||||
def request_cleanup(
|
||||
self,
|
||||
payload: CleanupReadinessEvidenceInput,
|
||||
*,
|
||||
apply: bool,
|
||||
) -> CleanupReadinessReport:
|
||||
tenant_id = str(payload.tenant_id)
|
||||
request_id = str(payload.request_id)
|
||||
with self._session_maker.begin() as session:
|
||||
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
|
||||
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
|
||||
ledger = self._require_ledger(cutover_repository, tenant_id)
|
||||
reasons = self._readiness_reasons(cutover_repository, ledger, payload)
|
||||
existing = cleanup_repository.get(
|
||||
tenant_id=tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
)
|
||||
if existing is not None:
|
||||
self._assert_request_replay(existing, payload)
|
||||
replay_payload = payload.model_copy(update={"expected_cas_version": ledger.cas_version})
|
||||
reasons = self._readiness_reasons(cutover_repository, ledger, replay_payload)
|
||||
return CleanupReadinessReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
not reasons,
|
||||
reasons,
|
||||
existing.status.value,
|
||||
apply,
|
||||
True,
|
||||
True,
|
||||
)
|
||||
if not apply:
|
||||
return CleanupReadinessReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
not reasons,
|
||||
reasons,
|
||||
"dry_run",
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
if reasons:
|
||||
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
|
||||
self._cas_ledger(
|
||||
cutover_repository,
|
||||
KnowledgeFSCutoverCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
expected_phase=ledger.phase,
|
||||
expected_cas_version=ledger.cas_version,
|
||||
new_phase=ledger.phase,
|
||||
),
|
||||
)
|
||||
cleanup_repository.add(
|
||||
KnowledgeFSCleanupAuthorization(
|
||||
tenant_id=tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
plan_schema_version=_PLAN_SCHEMA_VERSION,
|
||||
plan_digest=payload.plan_digest,
|
||||
targets=[target.value for target in payload.targets],
|
||||
readiness_evidence=payload.model_dump(mode="json"),
|
||||
requested_by_account_id=str(payload.requested_by_account_id),
|
||||
requested_at=ensure_naive_utc(payload.requested_at),
|
||||
readiness_ledger_cas_version=ledger.cas_version + 1,
|
||||
)
|
||||
)
|
||||
return CleanupReadinessReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
True,
|
||||
(),
|
||||
KnowledgeFSCleanupAuthorizationStatus.REQUESTED.value,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
|
||||
def approve_cleanup(self, payload: CleanupApprovalInput, *, apply: bool) -> CleanupApprovalReport:
|
||||
tenant_id = str(payload.tenant_id)
|
||||
request_id = str(payload.request_id)
|
||||
with self._session_maker.begin() as session:
|
||||
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
|
||||
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
|
||||
ledger = self._require_ledger(cutover_repository, tenant_id)
|
||||
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
|
||||
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.APPROVED:
|
||||
self._assert_approval_replay(authorization, payload)
|
||||
return CleanupApprovalReport(tenant_id, request_id, True, (), "approved", apply, True)
|
||||
if authorization.status in {
|
||||
KnowledgeFSCleanupAuthorizationStatus.STARTED,
|
||||
KnowledgeFSCleanupAuthorizationStatus.COMPLETED,
|
||||
}:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup authorization has already started")
|
||||
reasons = self._approval_reasons(cutover_repository, ledger, authorization, payload)
|
||||
if not apply:
|
||||
return CleanupApprovalReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
not reasons,
|
||||
reasons,
|
||||
authorization.status.value,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
if reasons:
|
||||
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
|
||||
self._cas_ledger(
|
||||
cutover_repository,
|
||||
KnowledgeFSCutoverCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
expected_phase=ledger.phase,
|
||||
expected_cas_version=ledger.cas_version,
|
||||
new_phase=ledger.phase,
|
||||
),
|
||||
)
|
||||
changed = cleanup_repository.compare_and_set(
|
||||
KnowledgeFSCleanupAuthorizationCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
expected_status=KnowledgeFSCleanupAuthorizationStatus.REQUESTED,
|
||||
expected_row_version=authorization.row_version,
|
||||
new_status=KnowledgeFSCleanupAuthorizationStatus.APPROVED,
|
||||
approved_by_account_id=str(payload.approved_by_account_id),
|
||||
approved_at=ensure_naive_utc(payload.approved_at),
|
||||
approval_expires_at=ensure_naive_utc(payload.approval_expires_at),
|
||||
approved_ledger_cas_version=ledger.cas_version + 1,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during approval")
|
||||
return CleanupApprovalReport(tenant_id, request_id, True, (), "approved", True, False)
|
||||
|
||||
def start_cleanup(self, payload: CleanupStartInput, *, apply: bool) -> CleanupStartReport:
|
||||
tenant_id = str(payload.tenant_id)
|
||||
request_id = str(payload.request_id)
|
||||
with self._session_maker.begin() as session:
|
||||
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
|
||||
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
|
||||
ledger = self._require_ledger(cutover_repository, tenant_id)
|
||||
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
|
||||
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.STARTED:
|
||||
self._assert_start_replay(authorization, payload)
|
||||
return CleanupStartReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
True,
|
||||
(),
|
||||
"started",
|
||||
apply,
|
||||
True,
|
||||
_iso(ledger.irreversible_cleanup_at),
|
||||
False,
|
||||
)
|
||||
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup authorization has already completed")
|
||||
reasons = self._start_reasons(cutover_repository, ledger, authorization, payload)
|
||||
if not apply:
|
||||
return CleanupStartReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
not reasons,
|
||||
reasons,
|
||||
authorization.status.value,
|
||||
False,
|
||||
False,
|
||||
_iso(ledger.irreversible_cleanup_at),
|
||||
False,
|
||||
)
|
||||
if reasons:
|
||||
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
|
||||
started_at = ensure_naive_utc(payload.started_at)
|
||||
self._fence_cleanup_cohort(cutover_repository, started_at=started_at)
|
||||
changed = cleanup_repository.compare_and_set(
|
||||
KnowledgeFSCleanupAuthorizationCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
expected_status=KnowledgeFSCleanupAuthorizationStatus.APPROVED,
|
||||
expected_row_version=authorization.row_version,
|
||||
new_status=KnowledgeFSCleanupAuthorizationStatus.STARTED,
|
||||
started_by_account_id=str(payload.started_by_account_id),
|
||||
started_at=started_at,
|
||||
started_ledger_cas_version=ledger.cas_version + 1,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during start")
|
||||
return CleanupStartReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
True,
|
||||
(),
|
||||
"started",
|
||||
True,
|
||||
False,
|
||||
_iso(started_at),
|
||||
False,
|
||||
)
|
||||
|
||||
def complete_cleanup(
|
||||
self,
|
||||
payload: CleanupCompletionEvidenceInput,
|
||||
*,
|
||||
apply: bool,
|
||||
) -> CleanupCompletionReport:
|
||||
tenant_id = str(payload.tenant_id)
|
||||
request_id = str(payload.request_id)
|
||||
with self._session_maker.begin() as session:
|
||||
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
|
||||
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
|
||||
ledger = self._require_ledger(cutover_repository, tenant_id)
|
||||
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
|
||||
if authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED:
|
||||
self._assert_completion_replay(authorization, payload)
|
||||
return CleanupCompletionReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
True,
|
||||
(),
|
||||
"completed",
|
||||
apply,
|
||||
True,
|
||||
True,
|
||||
)
|
||||
reasons = self._completion_reasons(cutover_repository, ledger, authorization, payload)
|
||||
if not apply:
|
||||
return CleanupCompletionReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
not reasons,
|
||||
reasons,
|
||||
authorization.status.value,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
if reasons:
|
||||
raise KnowledgeFSCleanupGateBlockedError("; ".join(reasons))
|
||||
self._cas_ledger(
|
||||
cutover_repository,
|
||||
KnowledgeFSCutoverCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
expected_phase=ledger.phase,
|
||||
expected_cas_version=ledger.cas_version,
|
||||
new_phase=ledger.phase,
|
||||
),
|
||||
)
|
||||
completed_at = ensure_naive_utc(payload.completed_at)
|
||||
changed = cleanup_repository.compare_and_set(
|
||||
KnowledgeFSCleanupAuthorizationCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
expected_status=KnowledgeFSCleanupAuthorizationStatus.STARTED,
|
||||
expected_row_version=authorization.row_version,
|
||||
new_status=KnowledgeFSCleanupAuthorizationStatus.COMPLETED,
|
||||
completed_by_account_id=str(payload.completed_by_account_id),
|
||||
completed_at=completed_at,
|
||||
completion_evidence=payload.model_dump(mode="json"),
|
||||
completed_ledger_cas_version=ledger.cas_version + 1,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup authorization changed during completion")
|
||||
return CleanupCompletionReport(
|
||||
tenant_id,
|
||||
request_id,
|
||||
True,
|
||||
(),
|
||||
"completed",
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
)
|
||||
|
||||
def status(self, *, tenant_id: str, request_id: str) -> dict[str, object]:
|
||||
with self._session_maker() as session:
|
||||
cutover_repository = SQLAlchemyKnowledgeFSCutoverRepository(session)
|
||||
cleanup_repository = SQLAlchemyKnowledgeFSCleanupAuthorizationRepository(session)
|
||||
ledger = self._require_ledger(cutover_repository, tenant_id)
|
||||
authorization = self._require_authorization(cleanup_repository, ledger, request_id)
|
||||
return {
|
||||
"tenant_id": tenant_id,
|
||||
"request_id": request_id,
|
||||
"status": authorization.status.value,
|
||||
"plan_schema_version": authorization.plan_schema_version,
|
||||
"plan_digest": authorization.plan_digest,
|
||||
"targets": authorization.targets,
|
||||
"readiness_evidence": authorization.readiness_evidence,
|
||||
"requested_by_account_id": authorization.requested_by_account_id,
|
||||
"requested_at": _iso(authorization.requested_at),
|
||||
"readiness_ledger_cas_version": authorization.readiness_ledger_cas_version,
|
||||
"approved_by_account_id": authorization.approved_by_account_id,
|
||||
"approved_at": _iso(authorization.approved_at),
|
||||
"approval_expires_at": _iso(authorization.approval_expires_at),
|
||||
"approved_ledger_cas_version": authorization.approved_ledger_cas_version,
|
||||
"started_by_account_id": authorization.started_by_account_id,
|
||||
"started_at": _iso(authorization.started_at),
|
||||
"started_ledger_cas_version": authorization.started_ledger_cas_version,
|
||||
"completed_by_account_id": authorization.completed_by_account_id,
|
||||
"completed_at": _iso(authorization.completed_at),
|
||||
"completion_evidence": authorization.completion_evidence,
|
||||
"completed_ledger_cas_version": authorization.completed_ledger_cas_version,
|
||||
"row_version": authorization.row_version,
|
||||
"irreversible_cleanup_at": _iso(ledger.irreversible_cleanup_at),
|
||||
"destructive_actions_executed": (
|
||||
authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED
|
||||
),
|
||||
"operator_attested_evidence": True,
|
||||
}
|
||||
|
||||
def _readiness_reasons(
|
||||
self,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
payload: CleanupReadinessEvidenceInput,
|
||||
*,
|
||||
allow_irreversible: bool = False,
|
||||
) -> tuple[str, ...]:
|
||||
reasons: list[str] = []
|
||||
now = ensure_naive_utc(self._clock())
|
||||
observed_at = ensure_naive_utc(payload.observed_at)
|
||||
requested_at = ensure_naive_utc(payload.requested_at)
|
||||
zero_window_start = ensure_naive_utc(payload.legacy_route_zero_window_started_at)
|
||||
zero_window_end = ensure_naive_utc(payload.legacy_route_zero_window_ends_at)
|
||||
maximum_token_expires_at = ensure_naive_utc(payload.maximum_token_expires_at)
|
||||
backup_verified_at = ensure_naive_utc(payload.backup_verified_at)
|
||||
restore_drill_verified_at = ensure_naive_utc(payload.restore_drill_verified_at)
|
||||
if observed_at > now or requested_at > now:
|
||||
reasons.append("cleanup evidence cannot be future-dated")
|
||||
if requested_at < observed_at:
|
||||
reasons.append("cleanup request predates its evidence observation")
|
||||
if payload.legacy_access_route_calls != 0:
|
||||
reasons.append("legacy access route calls are nonzero")
|
||||
if payload.legacy_member_route_calls != 0:
|
||||
reasons.append("legacy member route calls are nonzero")
|
||||
if payload.legacy_api_key_route_calls != 0:
|
||||
reasons.append("legacy API-key route calls are nonzero")
|
||||
if payload.legacy_route_calls != 0:
|
||||
reasons.append("legacy route calls are nonzero")
|
||||
if zero_window_end > observed_at or zero_window_end < zero_window_start:
|
||||
reasons.append("legacy route zero window is invalid")
|
||||
elif zero_window_end - zero_window_start < timedelta(seconds=payload.rollback_window_seconds):
|
||||
reasons.append("legacy route zero window is shorter than one rollback window")
|
||||
if maximum_token_expires_at > observed_at:
|
||||
reasons.append("maximum token TTL has not elapsed")
|
||||
if backup_verified_at > observed_at or restore_drill_verified_at > observed_at:
|
||||
reasons.append("backup or restore drill verification is future evidence")
|
||||
|
||||
ledgers = repository.list_ledgers()
|
||||
expected_tenant_ids = tuple(str(tenant_id) for tenant_id in payload.workspace_tenant_ids)
|
||||
actual_tenant_ids = tuple(candidate.tenant_id for candidate in ledgers)
|
||||
if len(ledgers) != payload.expected_workspace_count:
|
||||
reasons.append("persisted Workspace count does not match the cleanup cohort")
|
||||
if actual_tenant_ids != expected_tenant_ids:
|
||||
reasons.append("persisted Workspace cohort does not exactly match the cleanup inventory")
|
||||
if cleanup_workspace_cohort_digest(actual_tenant_ids) != payload.workspace_cohort_digest:
|
||||
reasons.append("persisted Workspace cohort digest changed")
|
||||
|
||||
for candidate in ledgers:
|
||||
workspace_reasons = self._workspace_readiness_reasons(
|
||||
repository,
|
||||
candidate,
|
||||
payload,
|
||||
allow_irreversible=allow_irreversible,
|
||||
)
|
||||
if candidate.tenant_id == ledger.tenant_id and candidate.cas_version != payload.expected_cas_version:
|
||||
workspace_reasons.insert(0, "cutover ledger CAS version changed")
|
||||
for reason in workspace_reasons:
|
||||
if candidate.tenant_id == ledger.tenant_id:
|
||||
reasons.append(reason)
|
||||
else:
|
||||
reasons.append(f"Workspace {candidate.tenant_id}: {reason}")
|
||||
return tuple(dict.fromkeys(reasons))
|
||||
|
||||
def _workspace_readiness_reasons(
|
||||
self,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
payload: CleanupReadinessEvidenceInput,
|
||||
*,
|
||||
allow_irreversible: bool,
|
||||
) -> list[str]:
|
||||
reasons: list[str] = []
|
||||
now = ensure_naive_utc(self._clock())
|
||||
observed_at = ensure_naive_utc(payload.observed_at)
|
||||
zero_window_start = ensure_naive_utc(payload.legacy_route_zero_window_started_at)
|
||||
backup_verified_at = ensure_naive_utc(payload.backup_verified_at)
|
||||
restore_drill_verified_at = ensure_naive_utc(payload.restore_drill_verified_at)
|
||||
if ledger.phase is not KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP:
|
||||
reasons.append("workspace is not ready_for_cleanup")
|
||||
if allow_irreversible:
|
||||
if ledger.irreversible_cleanup_at is None:
|
||||
reasons.append("irreversible cleanup fence is missing")
|
||||
elif ledger.irreversible_cleanup_at is not None:
|
||||
reasons.append("irreversible cleanup has already started")
|
||||
if ledger.observation_completed_at is None:
|
||||
reasons.append("production observation is not persisted as complete")
|
||||
if ledger.observation_window_ends_at is None or ledger.observation_completed_at is None:
|
||||
reasons.append("observation window evidence is incomplete")
|
||||
elif ledger.observation_completed_at < ledger.observation_window_ends_at:
|
||||
reasons.append("persisted observation window has not elapsed")
|
||||
if ledger.maximum_task_expires_at is None or ledger.observation_completed_at is None:
|
||||
reasons.append("maximum task TTL evidence is incomplete")
|
||||
elif ledger.observation_completed_at < ledger.maximum_task_expires_at:
|
||||
reasons.append("persisted maximum task TTL has not elapsed")
|
||||
if ledger.observation_completed_at is not None and observed_at < ledger.observation_completed_at:
|
||||
reasons.append("cleanup evidence predates completed observation")
|
||||
if (
|
||||
ledger.rollback_cutoff_at is None
|
||||
or observed_at < ledger.rollback_cutoff_at
|
||||
or now < ledger.rollback_cutoff_at
|
||||
):
|
||||
reasons.append("rollback cutoff has not elapsed")
|
||||
if ledger.cutover_at is None or zero_window_start < ledger.cutover_at:
|
||||
reasons.append("legacy route zero window predates cutover")
|
||||
if ledger.observation_completed_at is not None and (
|
||||
backup_verified_at < ledger.observation_completed_at
|
||||
or restore_drill_verified_at < ledger.observation_completed_at
|
||||
):
|
||||
reasons.append("backup and restore drill must be verified after observation completion")
|
||||
if not ledger.legacy_dependency_ready or ledger.legacy_dependency_checked_at is None:
|
||||
reasons.append("legacy snapshot/FK dependency dashboard is not ready")
|
||||
if repository.count_open_issues(tenant_id=ledger.tenant_id, ledger_id=ledger.id) > 0:
|
||||
reasons.append("open migration issues remain")
|
||||
if repository.count_unapproved_shadow_diffs(tenant_id=ledger.tenant_id, ledger_id=ledger.id) > 0:
|
||||
reasons.append("open shadow authorization differences remain")
|
||||
if (
|
||||
not knowledge_fs_cutover_smoke_results_passed(ledger.smoke_results)
|
||||
or ledger.smoke_results is None
|
||||
or ledger.smoke_results.get("tenant_id") != ledger.tenant_id
|
||||
):
|
||||
reasons.append("cutover smoke evidence is incomplete")
|
||||
if not knowledge_fs_remote_freeze_evidence_consistent(ledger):
|
||||
reasons.append("remote Workspace freeze evidence is incomplete or inconsistent")
|
||||
if not knowledge_fs_remote_activation_evidence_consistent(ledger):
|
||||
reasons.append("remote Workspace activation evidence is incomplete or inconsistent")
|
||||
if (
|
||||
ledger.final_revision_watermark is None
|
||||
or ledger.final_revision_watermark != ledger.applied_revision_watermark
|
||||
or ledger.final_task_watermark is None
|
||||
or ledger.final_task_watermark != ledger.applied_task_watermark
|
||||
):
|
||||
reasons.append("final delta watermarks are not fully applied")
|
||||
if not (
|
||||
ledger.product_routes_enabled
|
||||
and ledger.capability_v2_enabled
|
||||
and ledger.integrated_mode_enabled
|
||||
and ledger.legacy_acl_read_only
|
||||
):
|
||||
reasons.append("atomic cutover feature state is not fully enabled")
|
||||
return reasons
|
||||
|
||||
def _approval_reasons(
|
||||
self,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupApprovalInput,
|
||||
) -> tuple[str, ...]:
|
||||
reasons: list[str] = []
|
||||
approved_at = ensure_naive_utc(payload.approved_at)
|
||||
expires_at = ensure_naive_utc(payload.approval_expires_at)
|
||||
now = ensure_naive_utc(self._clock())
|
||||
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.REQUESTED:
|
||||
reasons.append("cleanup authorization is not requested")
|
||||
if ledger.cas_version != payload.expected_cas_version:
|
||||
reasons.append("cutover ledger CAS version changed")
|
||||
if authorization.plan_digest != payload.plan_digest:
|
||||
reasons.append("cleanup plan digest changed")
|
||||
if authorization.requested_by_account_id == str(payload.approved_by_account_id):
|
||||
reasons.append("cleanup requires a distinct approver")
|
||||
if approved_at > now:
|
||||
reasons.append("cleanup approval cannot be future-dated")
|
||||
if expires_at <= now or expires_at <= approved_at:
|
||||
reasons.append("cleanup approval must be currently valid")
|
||||
if expires_at - approved_at > _MAX_APPROVAL_LIFETIME:
|
||||
reasons.append("cleanup approval lifetime exceeds 24 hours")
|
||||
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
|
||||
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
|
||||
reasons.extend(self._readiness_reasons(repository, ledger, evidence))
|
||||
return tuple(dict.fromkeys(reasons))
|
||||
|
||||
def _start_reasons(
|
||||
self,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupStartInput,
|
||||
) -> tuple[str, ...]:
|
||||
reasons: list[str] = []
|
||||
started_at = ensure_naive_utc(payload.started_at)
|
||||
now = ensure_naive_utc(self._clock())
|
||||
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.APPROVED:
|
||||
reasons.append("cleanup authorization is not approved")
|
||||
if ledger.cas_version != payload.expected_cas_version:
|
||||
reasons.append("cutover ledger CAS version changed")
|
||||
if authorization.plan_digest != payload.plan_digest:
|
||||
reasons.append("cleanup plan digest changed")
|
||||
if authorization.approved_at is None or started_at < authorization.approved_at:
|
||||
reasons.append("cleanup start predates approval")
|
||||
if started_at > now:
|
||||
reasons.append("cleanup start cannot be future-dated")
|
||||
if authorization.approval_expires_at is None or now >= authorization.approval_expires_at:
|
||||
reasons.append("cleanup approval has expired")
|
||||
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
|
||||
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
|
||||
reasons.extend(self._readiness_reasons(repository, ledger, evidence))
|
||||
return tuple(dict.fromkeys(reasons))
|
||||
|
||||
def _completion_reasons(
|
||||
self,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupCompletionEvidenceInput,
|
||||
) -> tuple[str, ...]:
|
||||
reasons: list[str] = []
|
||||
completed_at = ensure_naive_utc(payload.completed_at)
|
||||
now = ensure_naive_utc(self._clock())
|
||||
if authorization.status is not KnowledgeFSCleanupAuthorizationStatus.STARTED:
|
||||
reasons.append("cleanup authorization is not started")
|
||||
if ledger.cas_version != payload.expected_cas_version:
|
||||
reasons.append("cutover ledger CAS version changed")
|
||||
if authorization.plan_digest != payload.plan_digest:
|
||||
reasons.append("cleanup plan digest changed")
|
||||
if payload.migration_bundle_digest != authorization.plan_digest:
|
||||
reasons.append("executed migration bundle does not match the approved plan")
|
||||
if authorization.started_at is None or completed_at < authorization.started_at:
|
||||
reasons.append("cleanup completion predates the irreversible start")
|
||||
if completed_at > now:
|
||||
reasons.append("cleanup completion cannot be future-dated")
|
||||
evidence = CleanupReadinessEvidenceInput.model_validate(authorization.readiness_evidence)
|
||||
evidence = evidence.model_copy(update={"expected_cas_version": ledger.cas_version})
|
||||
reasons.extend(
|
||||
self._readiness_reasons(
|
||||
repository,
|
||||
ledger,
|
||||
evidence,
|
||||
allow_irreversible=True,
|
||||
)
|
||||
)
|
||||
if authorization.started_at is not None:
|
||||
for candidate in repository.list_ledgers():
|
||||
if candidate.irreversible_cleanup_at != authorization.started_at:
|
||||
reasons.append(f"Workspace {candidate.tenant_id}: irreversible cleanup fence changed")
|
||||
return tuple(dict.fromkeys(reasons))
|
||||
|
||||
@staticmethod
|
||||
def _require_ledger(
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
tenant_id: str,
|
||||
) -> KnowledgeFSWorkspaceCutoverLedger:
|
||||
ledger = repository.get_ledger(tenant_id=tenant_id)
|
||||
if ledger is None:
|
||||
raise KnowledgeFSCleanupNotFoundError("Workspace cutover ledger was not found")
|
||||
return ledger
|
||||
|
||||
@staticmethod
|
||||
def _require_authorization(
|
||||
repository: SQLAlchemyKnowledgeFSCleanupAuthorizationRepository,
|
||||
ledger: KnowledgeFSWorkspaceCutoverLedger,
|
||||
request_id: str,
|
||||
) -> KnowledgeFSCleanupAuthorization:
|
||||
authorization = repository.get(
|
||||
tenant_id=ledger.tenant_id,
|
||||
ledger_id=ledger.id,
|
||||
request_id=request_id,
|
||||
)
|
||||
if authorization is None:
|
||||
raise KnowledgeFSCleanupNotFoundError("Cleanup authorization was not found")
|
||||
return authorization
|
||||
|
||||
@staticmethod
|
||||
def _cas_ledger(
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
update: KnowledgeFSCutoverCASUpdate,
|
||||
) -> None:
|
||||
if not repository.compare_and_set(update):
|
||||
raise KnowledgeFSCleanupConflictError("Cutover ledger changed during cleanup authorization")
|
||||
|
||||
@classmethod
|
||||
def _fence_cleanup_cohort(
|
||||
cls,
|
||||
repository: SQLAlchemyKnowledgeFSCutoverRepository,
|
||||
*,
|
||||
started_at: datetime,
|
||||
) -> None:
|
||||
ledgers = repository.list_ledgers()
|
||||
if not ledgers:
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup Workspace cohort is empty")
|
||||
for candidate in ledgers:
|
||||
cls._cas_ledger(
|
||||
repository,
|
||||
KnowledgeFSCutoverCASUpdate(
|
||||
tenant_id=candidate.tenant_id,
|
||||
expected_phase=candidate.phase,
|
||||
expected_cas_version=candidate.cas_version,
|
||||
new_phase=candidate.phase,
|
||||
irreversible_cleanup_at=started_at,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _assert_request_replay(
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupReadinessEvidenceInput,
|
||||
) -> None:
|
||||
if (
|
||||
authorization.plan_digest != payload.plan_digest
|
||||
or authorization.targets != [target.value for target in payload.targets]
|
||||
or authorization.readiness_evidence != payload.model_dump(mode="json")
|
||||
):
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup request ID was reused with different evidence")
|
||||
|
||||
@staticmethod
|
||||
def _assert_approval_replay(
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupApprovalInput,
|
||||
) -> None:
|
||||
if (
|
||||
authorization.plan_digest != payload.plan_digest
|
||||
or authorization.approved_by_account_id != str(payload.approved_by_account_id)
|
||||
or authorization.approved_at != ensure_naive_utc(payload.approved_at)
|
||||
or authorization.approval_expires_at != ensure_naive_utc(payload.approval_expires_at)
|
||||
):
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup approval replay does not match persisted approval")
|
||||
|
||||
@staticmethod
|
||||
def _assert_start_replay(
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupStartInput,
|
||||
) -> None:
|
||||
if (
|
||||
authorization.plan_digest != payload.plan_digest
|
||||
or authorization.started_by_account_id != str(payload.started_by_account_id)
|
||||
or authorization.started_at != ensure_naive_utc(payload.started_at)
|
||||
):
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup start replay does not match persisted fence")
|
||||
|
||||
@staticmethod
|
||||
def _assert_completion_replay(
|
||||
authorization: KnowledgeFSCleanupAuthorization,
|
||||
payload: CleanupCompletionEvidenceInput,
|
||||
) -> None:
|
||||
if (
|
||||
authorization.plan_digest != payload.plan_digest
|
||||
or authorization.completed_by_account_id != str(payload.completed_by_account_id)
|
||||
or authorization.completed_at != ensure_naive_utc(payload.completed_at)
|
||||
or authorization.completion_evidence != payload.model_dump(mode="json")
|
||||
):
|
||||
raise KnowledgeFSCleanupConflictError("Cleanup completion replay does not match persisted evidence")
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return f"{value.isoformat()}Z" if value is not None else None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CleanupApprovalInput",
|
||||
"CleanupApprovalReport",
|
||||
"CleanupCompletionEvidenceInput",
|
||||
"CleanupCompletionReport",
|
||||
"CleanupReadinessEvidenceInput",
|
||||
"CleanupReadinessReport",
|
||||
"CleanupStartInput",
|
||||
"CleanupStartReport",
|
||||
"KnowledgeFSCleanupConflictError",
|
||||
"KnowledgeFSCleanupError",
|
||||
"KnowledgeFSCleanupGateBlockedError",
|
||||
"KnowledgeFSCleanupNotFoundError",
|
||||
"KnowledgeFSCleanupService",
|
||||
"cleanup_workspace_cohort_digest",
|
||||
]
|
||||
@@ -0,0 +1,430 @@
|
||||
"""Dify-owned KnowledgeFS members, visibility, and external-access mutations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models import TenantAccountJoin
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpacePermissionStatus,
|
||||
KnowledgeFSControlSpaceVisibility,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSExternalAccessPayload,
|
||||
KnowledgeFSExternalAccessResponse,
|
||||
KnowledgeFSMemberBindingPayload,
|
||||
KnowledgeFSPermissionListResponse,
|
||||
KnowledgeFSPermissionResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
|
||||
from services.knowledge_fs.product_service import KnowledgeFSProductService
|
||||
from services.knowledge_fs.revocation_commands import (
|
||||
KnowledgeFSRevocationCommandPort,
|
||||
KnowledgeFSRevocationCommandProducer,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSControlPlaneInvariantError(RuntimeError):
|
||||
"""Required authorization revision state is absent or inconsistent."""
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceMemberPort(Protocol):
|
||||
def are_active_members(self, *, session: Session, tenant_id: str, account_ids: Sequence[str]) -> bool: ...
|
||||
|
||||
|
||||
class SQLKnowledgeFSWorkspaceMemberPort:
|
||||
def are_active_members(self, *, session: Session, tenant_id: str, account_ids: Sequence[str]) -> bool:
|
||||
unique_ids = frozenset(account_ids)
|
||||
if not unique_ids:
|
||||
return True
|
||||
found = frozenset(
|
||||
session.scalars(
|
||||
sa.select(TenantAccountJoin.account_id).where(
|
||||
TenantAccountJoin.tenant_id == tenant_id,
|
||||
TenantAccountJoin.account_id.in_(unique_ids),
|
||||
)
|
||||
)
|
||||
)
|
||||
return found == unique_ids
|
||||
|
||||
|
||||
class KnowledgeFSControlPlaneService:
|
||||
"""Mutate Dify authorization state after product authorization succeeds."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
product: KnowledgeFSProductService,
|
||||
members: KnowledgeFSWorkspaceMemberPort,
|
||||
revocations: KnowledgeFSRevocationCommandPort | None = None,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._product = product
|
||||
self._members = members
|
||||
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
|
||||
|
||||
def list_permissions(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSPermissionListResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
with self._session_maker() as session:
|
||||
permissions = tuple(
|
||||
session.scalars(
|
||||
sa.select(KnowledgeFSControlSpacePermission)
|
||||
.where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
|
||||
)
|
||||
.order_by(KnowledgeFSControlSpacePermission.account_id)
|
||||
)
|
||||
)
|
||||
return KnowledgeFSPermissionListResponse(
|
||||
data=[
|
||||
KnowledgeFSPermissionResponse(
|
||||
account_id=permission.account_id,
|
||||
role=permission.role,
|
||||
status=permission.status.value,
|
||||
revision=permission.revision,
|
||||
)
|
||||
for permission in permissions
|
||||
]
|
||||
)
|
||||
|
||||
def replace_members(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
members: Sequence[KnowledgeFSMemberBindingPayload],
|
||||
) -> KnowledgeFSPermissionListResponse:
|
||||
authorized = self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
desired = {member.account_id: member.role for member in members}
|
||||
if len(desired) != len(members) or authorized.control_space.owner_account_id in desired:
|
||||
raise KnowledgeFSControlPlaneInvariantError("Member bindings must be unique and exclude the owner")
|
||||
with self._session_maker.begin() as session:
|
||||
control_space = session.scalar(
|
||||
sa.select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if control_space is None:
|
||||
raise KnowledgeFSControlPlaneInvariantError("Control-space disappeared during member update")
|
||||
if not self._members.are_active_members(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
account_ids=tuple(desired),
|
||||
):
|
||||
raise KnowledgeFSControlPlaneInvariantError("Every KnowledgeFS member must belong to the workspace")
|
||||
existing = {
|
||||
permission.account_id: permission
|
||||
for permission in session.scalars(
|
||||
sa.select(KnowledgeFSControlSpacePermission).where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
|
||||
KnowledgeFSControlSpacePermission.account_id != control_space.owner_account_id,
|
||||
)
|
||||
)
|
||||
}
|
||||
now = naive_utc_now()
|
||||
revoke_reasons_by_account: dict[str, str] = {}
|
||||
for account_id, permission in existing.items():
|
||||
role = desired.pop(account_id, None)
|
||||
if role is None:
|
||||
if permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE:
|
||||
permission.status = KnowledgeFSControlSpacePermissionStatus.REVOKED
|
||||
permission.revision += 1
|
||||
permission.revoked_at = now
|
||||
permission.revoked_by_account_id = actor_account_id
|
||||
revoke_reasons_by_account[account_id] = "permission_revoked"
|
||||
elif (
|
||||
permission.role is not role
|
||||
or permission.status is not KnowledgeFSControlSpacePermissionStatus.ACTIVE
|
||||
):
|
||||
previous_role = permission.role
|
||||
was_active = permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE
|
||||
permission.role = role
|
||||
permission.status = KnowledgeFSControlSpacePermissionStatus.ACTIVE
|
||||
permission.revision += 1
|
||||
permission.revoked_at = None
|
||||
permission.revoked_by_account_id = None
|
||||
if was_active and _permission_role_is_narrower(previous_role, role):
|
||||
revoke_reasons_by_account[account_id] = "permission_role_narrowed"
|
||||
for account_id, role in desired.items():
|
||||
session.add(
|
||||
KnowledgeFSControlSpacePermission(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
account_id=account_id,
|
||||
role=role,
|
||||
granted_by_account_id=actor_account_id,
|
||||
)
|
||||
)
|
||||
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
revision.space_acl_epoch += 1
|
||||
for account_id, reason_code in revoke_reasons_by_account.items():
|
||||
self._revocations.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
subject=f"dify-account:{account_id}",
|
||||
reason_code=reason_code,
|
||||
caller_kinds=("interactive",),
|
||||
)
|
||||
return self.list_permissions(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
|
||||
def update_visibility(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
visibility: KnowledgeFSControlSpaceVisibility,
|
||||
) -> None:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
control_space = session.scalar(
|
||||
sa.select(KnowledgeFSControlSpace)
|
||||
.where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if control_space is None:
|
||||
raise KnowledgeFSControlPlaneInvariantError("Control-space disappeared during visibility update")
|
||||
if control_space.visibility is visibility:
|
||||
return
|
||||
previous_visibility = control_space.visibility
|
||||
control_space.visibility = visibility
|
||||
control_space.resource_version += 1
|
||||
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
revision.space_acl_epoch += 1
|
||||
if _visibility_is_narrower(previous_visibility, visibility):
|
||||
active_member_subjects = {
|
||||
f"dify-account:{account_id}"
|
||||
for account_id in session.scalars(
|
||||
sa.select(KnowledgeFSControlSpacePermission.account_id).where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == control_space_id,
|
||||
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
KnowledgeFSControlSpacePermission.account_id != control_space.owner_account_id,
|
||||
)
|
||||
)
|
||||
}
|
||||
if previous_visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS:
|
||||
allowed_subjects = {f"dify-account:{control_space.owner_account_id}"}
|
||||
if visibility is KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS:
|
||||
allowed_subjects.update(active_member_subjects)
|
||||
self._revocations.enqueue_control_space_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
reason_code="visibility_narrowed",
|
||||
caller_kinds=("interactive",),
|
||||
excluded_subjects=tuple(sorted(allowed_subjects)),
|
||||
)
|
||||
else:
|
||||
for subject in sorted(active_member_subjects):
|
||||
self._revocations.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
subject=subject,
|
||||
reason_code="visibility_narrowed",
|
||||
caller_kinds=("interactive",),
|
||||
)
|
||||
|
||||
def get_external_access(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSExternalAccessResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
with self._session_maker() as session:
|
||||
policy = session.scalar(
|
||||
sa.select(KnowledgeFSExternalAccessPolicy).where(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == control_space_id,
|
||||
)
|
||||
)
|
||||
return _external_access_response(policy)
|
||||
|
||||
def update_external_access(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSExternalAccessPayload,
|
||||
) -> KnowledgeFSExternalAccessResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.ACCESS_CONFIG,
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
policy = session.scalar(
|
||||
sa.select(KnowledgeFSExternalAccessPolicy)
|
||||
.where(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if policy is None:
|
||||
policy = KnowledgeFSExternalAccessPolicy(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
session.add(policy)
|
||||
session.flush()
|
||||
previous_channels = {
|
||||
"service": policy.service_api_enabled,
|
||||
"agent": policy.agent_enabled,
|
||||
"workflow": policy.workflow_enabled,
|
||||
"mcp": policy.mcp_enabled,
|
||||
}
|
||||
requested_channels = {
|
||||
"service": payload.service_api_enabled,
|
||||
"agent": payload.agent_enabled,
|
||||
"workflow": payload.workflow_enabled,
|
||||
"mcp": payload.mcp_enabled,
|
||||
}
|
||||
if previous_channels == requested_channels:
|
||||
return _external_access_response(policy)
|
||||
policy.service_api_enabled = payload.service_api_enabled
|
||||
policy.agent_enabled = payload.agent_enabled
|
||||
policy.workflow_enabled = payload.workflow_enabled
|
||||
policy.mcp_enabled = payload.mcp_enabled
|
||||
policy.revision += 1
|
||||
policy.updated_by_account_id = actor_account_id
|
||||
revision = _authorization_revision(session, tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
revision.external_access_epoch += 1
|
||||
disabled_callers = tuple(
|
||||
caller_kind
|
||||
for caller_kind, was_enabled in previous_channels.items()
|
||||
if was_enabled and not requested_channels[caller_kind]
|
||||
)
|
||||
if disabled_callers:
|
||||
self._revocations.enqueue_control_space_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
reason_code="external_access_revoked",
|
||||
caller_kinds=disabled_callers,
|
||||
)
|
||||
session.flush()
|
||||
response = _external_access_response(policy)
|
||||
return response
|
||||
|
||||
|
||||
def _permission_role_is_narrower(
|
||||
previous: KnowledgeFSControlSpacePermissionRole,
|
||||
current: KnowledgeFSControlSpacePermissionRole,
|
||||
) -> bool:
|
||||
rank = {
|
||||
KnowledgeFSControlSpacePermissionRole.VIEWER: 0,
|
||||
KnowledgeFSControlSpacePermissionRole.EDITOR: 1,
|
||||
KnowledgeFSControlSpacePermissionRole.OWNER: 2,
|
||||
}
|
||||
return rank[current] < rank[previous]
|
||||
|
||||
|
||||
def _authorization_revision(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSAuthorizationRevision:
|
||||
revision = session.scalar(
|
||||
sa.select(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == control_space_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if revision is None:
|
||||
raise KnowledgeFSControlPlaneInvariantError("KnowledgeFS authorization revision is missing")
|
||||
return revision
|
||||
|
||||
|
||||
def _visibility_is_narrower(
|
||||
previous: KnowledgeFSControlSpaceVisibility,
|
||||
requested: KnowledgeFSControlSpaceVisibility,
|
||||
) -> bool:
|
||||
return (
|
||||
previous is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS
|
||||
and requested
|
||||
in {
|
||||
KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS,
|
||||
KnowledgeFSControlSpaceVisibility.ONLY_ME,
|
||||
}
|
||||
) or (
|
||||
previous is KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS
|
||||
and requested is KnowledgeFSControlSpaceVisibility.ONLY_ME
|
||||
)
|
||||
|
||||
|
||||
def _external_access_response(policy: KnowledgeFSExternalAccessPolicy | None) -> KnowledgeFSExternalAccessResponse:
|
||||
return KnowledgeFSExternalAccessResponse(
|
||||
service_api_enabled=policy.service_api_enabled if policy else False,
|
||||
agent_enabled=policy.agent_enabled if policy else False,
|
||||
workflow_enabled=policy.workflow_enabled if policy else False,
|
||||
mcp_enabled=policy.mcp_enabled if policy else False,
|
||||
revision=policy.revision if policy else 0,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSControlPlaneInvariantError",
|
||||
"KnowledgeFSControlPlaneService",
|
||||
"KnowledgeFSWorkspaceMemberPort",
|
||||
"SQLKnowledgeFSWorkspaceMemberPort",
|
||||
]
|
||||
@@ -0,0 +1,318 @@
|
||||
"""Durable product entrypoints for KnowledgeFS provision and deletion intent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import NamedTuple
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import (
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSCapabilityIssuanceAudit,
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSDeleteCommandPayload,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSLifecycleOperation,
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSLifecycleOutboxStatus,
|
||||
KnowledgeFSModelSelectionIntentPayload,
|
||||
KnowledgeFSProvisionCommandPayload,
|
||||
KnowledgeFSRetrievalProfileIntentPayload,
|
||||
)
|
||||
from repositories.knowledge_fs_control_space_repository import KnowledgeFSControlSpaceCASUpdate
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
from repositories.sqlalchemy_knowledge_fs_lifecycle_outbox_repository import (
|
||||
SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
|
||||
)
|
||||
from services.knowledge_fs.control_space_lifecycle import (
|
||||
KnowledgeFSControlSpaceLifecycleService,
|
||||
KnowledgeFSControlSpaceNotFoundError,
|
||||
KnowledgeFSControlSpaceVersionConflictError,
|
||||
KnowledgeFSWorkspaceDeletionBlockedError,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSProvisionIntent(NamedTuple):
|
||||
tenant_id: str
|
||||
owner_account_id: str
|
||||
provisioning_key: str
|
||||
operation_id: str
|
||||
idempotency_key: str
|
||||
name: str
|
||||
slug: str
|
||||
icon: str | None
|
||||
description: str | None
|
||||
model_intent: KnowledgeFSModelSelectionIntentPayload
|
||||
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
|
||||
|
||||
|
||||
class KnowledgeFSProvisionIntentResult(NamedTuple):
|
||||
control_space: KnowledgeFSControlSpace
|
||||
outbox: KnowledgeFSLifecycleOutbox
|
||||
|
||||
|
||||
class KnowledgeFSDeletionIntentResult(NamedTuple):
|
||||
control_space: KnowledgeFSControlSpace
|
||||
outbox: KnowledgeFSLifecycleOutbox | None
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceIntentConflictError(RuntimeError):
|
||||
"""An idempotency identity was reused for a different lifecycle intent."""
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceCommandService:
|
||||
"""Write control state and its outbox command in one database transaction."""
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session]):
|
||||
self._session_maker = session_maker
|
||||
|
||||
def create_provision_intent(self, intent: KnowledgeFSProvisionIntent) -> KnowledgeFSProvisionIntentResult:
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
existing = control_repository.find_by_provisioning_key(provisioning_key=intent.provisioning_key)
|
||||
if existing is not None:
|
||||
command = outbox_repository.get_by_operation_id(
|
||||
tenant_id=intent.tenant_id,
|
||||
operation_id=intent.operation_id,
|
||||
)
|
||||
if (
|
||||
existing.tenant_id != intent.tenant_id
|
||||
or existing.owner_account_id != intent.owner_account_id
|
||||
or command is None
|
||||
or command.control_space_id != existing.id
|
||||
or command.idempotency_key != intent.idempotency_key
|
||||
):
|
||||
raise KnowledgeFSControlSpaceIntentConflictError(
|
||||
"KnowledgeFS provisioning key was reused for a different intent"
|
||||
)
|
||||
return KnowledgeFSProvisionIntentResult(existing, command)
|
||||
|
||||
control_space = control_repository.add(
|
||||
KnowledgeFSControlSpace(
|
||||
tenant_id=intent.tenant_id,
|
||||
owner_account_id=intent.owner_account_id,
|
||||
provisioning_key=intent.provisioning_key,
|
||||
lifecycle_operation_id=intent.operation_id,
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
KnowledgeFSAuthorizationRevision(
|
||||
tenant_id=intent.tenant_id,
|
||||
control_space_id=control_space.id,
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
KnowledgeFSControlSpacePermission(
|
||||
tenant_id=intent.tenant_id,
|
||||
control_space_id=control_space.id,
|
||||
account_id=intent.owner_account_id,
|
||||
role=KnowledgeFSControlSpacePermissionRole.OWNER,
|
||||
granted_by_account_id=intent.owner_account_id,
|
||||
)
|
||||
)
|
||||
payload = KnowledgeFSProvisionCommandPayload(
|
||||
schema_version=1,
|
||||
idempotency_key=intent.idempotency_key,
|
||||
expected_revision=0,
|
||||
provisioning_key=intent.provisioning_key,
|
||||
name=intent.name,
|
||||
icon=intent.icon,
|
||||
description=intent.description,
|
||||
slug=intent.slug,
|
||||
model_intent=KnowledgeFSModelSelectionIntentPayload(**intent.model_intent),
|
||||
profile_intent=KnowledgeFSRetrievalProfileIntentPayload(**intent.profile_intent),
|
||||
)
|
||||
command = outbox_repository.add(
|
||||
KnowledgeFSLifecycleOutbox(
|
||||
tenant_id=intent.tenant_id,
|
||||
control_space_id=control_space.id,
|
||||
operation_id=intent.operation_id,
|
||||
idempotency_key=intent.idempotency_key,
|
||||
operation=KnowledgeFSLifecycleOperation.PROVISION,
|
||||
command_payload=payload,
|
||||
expected_control_space_version=control_space.resource_version,
|
||||
expected_knowledge_space_revision=0,
|
||||
)
|
||||
)
|
||||
return KnowledgeFSProvisionIntentResult(control_space, command)
|
||||
|
||||
def request_deletion(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
idempotency_key: str,
|
||||
) -> KnowledgeFSDeletionIntentResult:
|
||||
"""Use the same durable path for single, batch, and workspace cleanup."""
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
control_space = control_repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
if control_space is None:
|
||||
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space was not found in this tenant")
|
||||
if control_space.state is KnowledgeFSControlSpaceState.DELETED:
|
||||
return KnowledgeFSDeletionIntentResult(control_space, None)
|
||||
|
||||
replay = outbox_repository.get_by_operation_id(tenant_id=tenant_id, operation_id=operation_id)
|
||||
if replay is not None:
|
||||
if replay.control_space_id != control_space_id or replay.idempotency_key != idempotency_key:
|
||||
raise KnowledgeFSControlSpaceIntentConflictError(
|
||||
"KnowledgeFS deletion operation was reused for a different intent"
|
||||
)
|
||||
if (
|
||||
replay.operation is KnowledgeFSLifecycleOperation.DELETE
|
||||
and replay.status is KnowledgeFSLifecycleOutboxStatus.DEAD_LETTER
|
||||
):
|
||||
if not outbox_repository.reactivate_dead_letter(outbox_id=replay.id):
|
||||
raise KnowledgeFSControlSpaceVersionConflictError(
|
||||
"KnowledgeFS permanent cleanup command changed before reactivation"
|
||||
)
|
||||
replay = outbox_repository.get(outbox_id=replay.id)
|
||||
if replay is None:
|
||||
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS cleanup command disappeared")
|
||||
return KnowledgeFSDeletionIntentResult(control_space, replay)
|
||||
open_command = outbox_repository.find_open_for_control_space(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
operation=KnowledgeFSLifecycleOperation.DELETE,
|
||||
)
|
||||
if open_command is not None:
|
||||
return KnowledgeFSDeletionIntentResult(control_space, open_command)
|
||||
|
||||
if control_space.state is KnowledgeFSControlSpaceState.DELETING:
|
||||
changed = control_repository.compare_and_set_lifecycle(
|
||||
KnowledgeFSControlSpaceCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
expected_state=KnowledgeFSControlSpaceState.DELETING,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETING,
|
||||
lifecycle_operation_id=operation_id,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSControlSpaceVersionConflictError(
|
||||
"KnowledgeFS control-space changed while restoring permanent cleanup"
|
||||
)
|
||||
transitioned = control_repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
if transitioned is None:
|
||||
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space disappeared")
|
||||
else:
|
||||
transitioned = KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETING,
|
||||
lifecycle_operation_id=operation_id,
|
||||
)
|
||||
|
||||
payload = KnowledgeFSDeleteCommandPayload(
|
||||
schema_version=1,
|
||||
idempotency_key=idempotency_key,
|
||||
expected_revision=transitioned.knowledge_space_revision,
|
||||
knowledge_space_id=transitioned.knowledge_space_id,
|
||||
provisioning_key=transitioned.provisioning_key,
|
||||
)
|
||||
command = outbox_repository.add(
|
||||
KnowledgeFSLifecycleOutbox(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
idempotency_key=idempotency_key,
|
||||
operation=KnowledgeFSLifecycleOperation.DELETE,
|
||||
command_payload=payload,
|
||||
expected_control_space_version=transitioned.resource_version,
|
||||
expected_knowledge_space_revision=transitioned.knowledge_space_revision,
|
||||
)
|
||||
)
|
||||
return KnowledgeFSDeletionIntentResult(transitioned, command)
|
||||
|
||||
def request_workspace_cleanup(self, *, tenant_id: str) -> tuple[KnowledgeFSDeletionIntentResult, ...]:
|
||||
"""Enumerate independent control-spaces; never hard-delete them with the workspace."""
|
||||
|
||||
with self._session_maker() as session:
|
||||
control_spaces = SQLAlchemyKnowledgeFSControlSpaceRepository(session).list_for_tenant(tenant_id=tenant_id)
|
||||
control_space_ids = tuple(
|
||||
control_space.id
|
||||
for control_space in control_spaces
|
||||
if control_space.state is not KnowledgeFSControlSpaceState.DELETED
|
||||
)
|
||||
commands: list[KnowledgeFSDeletionIntentResult] = []
|
||||
for control_space_id in control_space_ids:
|
||||
operation_id = str(
|
||||
uuid.uuid5(uuid.NAMESPACE_URL, f"dify-kfs-workspace-cleanup:{tenant_id}:{control_space_id}")
|
||||
)
|
||||
commands.append(
|
||||
self.request_deletion(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
idempotency_key=f"workspace-cleanup:{tenant_id}:{control_space_id}",
|
||||
)
|
||||
)
|
||||
return tuple(commands)
|
||||
|
||||
def assert_workspace_deletion_allowed(self, *, tenant_id: str) -> None:
|
||||
with self._session_maker() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
KnowledgeFSControlSpaceLifecycleService(repository).assert_workspace_deletion_allowed(tenant_id=tenant_id)
|
||||
|
||||
def finalize_workspace_deletion(self, *, tenant_id: str) -> int:
|
||||
"""Remove local control-plane rows only after every remote Space is durably deleted.
|
||||
|
||||
The control-space Workspace FK remains ``RESTRICT`` until this explicit finalizer
|
||||
succeeds, so a caller cannot hard-delete the Workspace and strand remote resources.
|
||||
"""
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
control_spaces = tuple(
|
||||
session.scalars(
|
||||
sa.select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
|
||||
.order_by(KnowledgeFSControlSpace.created_at, KnowledgeFSControlSpace.id)
|
||||
.with_for_update()
|
||||
)
|
||||
)
|
||||
blocking_ids = tuple(
|
||||
control_space.id
|
||||
for control_space in control_spaces
|
||||
if control_space.state is not KnowledgeFSControlSpaceState.DELETED
|
||||
)
|
||||
if blocking_ids:
|
||||
raise KnowledgeFSWorkspaceDeletionBlockedError(blocking_ids)
|
||||
|
||||
for model in (
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSCapabilityIssuanceAudit,
|
||||
KnowledgeFSCapabilityIssuanceReservation,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
AppKnowledgeFSSpaceJoin,
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
):
|
||||
session.execute(sa.delete(model).where(model.tenant_id == tenant_id))
|
||||
session.execute(sa.delete(KnowledgeFSControlSpace).where(KnowledgeFSControlSpace.tenant_id == tenant_id))
|
||||
return len(control_spaces)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSControlSpaceCommandService",
|
||||
"KnowledgeFSControlSpaceIntentConflictError",
|
||||
"KnowledgeFSDeletionIntentResult",
|
||||
"KnowledgeFSProvisionIntent",
|
||||
"KnowledgeFSProvisionIntentResult",
|
||||
]
|
||||
@@ -0,0 +1,215 @@
|
||||
"""Local CAS lifecycle rules for KnowledgeFS control-spaces.
|
||||
|
||||
P1A persists lifecycle intent only. This service deliberately has no
|
||||
KnowledgeFS client or task dispatcher, so deploying it cannot emit P1B remote
|
||||
traffic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpace, KnowledgeFSControlSpaceState
|
||||
from repositories.knowledge_fs_control_space_repository import (
|
||||
KnowledgeFSControlSpaceCASUpdate,
|
||||
KnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
from services.knowledge_fs.observability import (
|
||||
KnowledgeFSControlSpaceStateMetric,
|
||||
KnowledgeFSOperationalMetricsPort,
|
||||
get_knowledge_fs_operational_metrics,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceLifecycleError(RuntimeError):
|
||||
"""Base error for a rejected local control-space lifecycle operation."""
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceNotFoundError(KnowledgeFSControlSpaceLifecycleError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceVersionConflictError(KnowledgeFSControlSpaceLifecycleError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSInvalidLifecycleTransitionError(KnowledgeFSControlSpaceLifecycleError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSSpaceRegistrationConflictError(KnowledgeFSControlSpaceLifecycleError):
|
||||
pass
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceDeletionBlockedError(KnowledgeFSControlSpaceLifecycleError):
|
||||
def __init__(self, control_space_ids: tuple[str, ...]):
|
||||
self.control_space_ids = control_space_ids
|
||||
super().__init__("Workspace deletion is blocked by non-terminal KnowledgeFS control-spaces")
|
||||
|
||||
|
||||
_ALLOWED_TRANSITIONS: Mapping[KnowledgeFSControlSpaceState, frozenset[KnowledgeFSControlSpaceState]] = {
|
||||
KnowledgeFSControlSpaceState.PROVISIONING: frozenset(
|
||||
{
|
||||
KnowledgeFSControlSpaceState.ACTIVE,
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
KnowledgeFSControlSpaceState.ERROR,
|
||||
}
|
||||
),
|
||||
KnowledgeFSControlSpaceState.ACTIVE: frozenset(
|
||||
{KnowledgeFSControlSpaceState.DELETING, KnowledgeFSControlSpaceState.ERROR}
|
||||
),
|
||||
KnowledgeFSControlSpaceState.DELETING: frozenset(
|
||||
{KnowledgeFSControlSpaceState.DELETED, KnowledgeFSControlSpaceState.ERROR}
|
||||
),
|
||||
KnowledgeFSControlSpaceState.ERROR: frozenset(
|
||||
{
|
||||
KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
KnowledgeFSControlSpaceState.ACTIVE,
|
||||
KnowledgeFSControlSpaceState.DELETING,
|
||||
}
|
||||
),
|
||||
KnowledgeFSControlSpaceState.DELETED: frozenset(),
|
||||
}
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceLifecycleService:
|
||||
"""Validate lifecycle invariants and persist transitions through a CAS."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository: KnowledgeFSControlSpaceRepository,
|
||||
*,
|
||||
metrics: KnowledgeFSOperationalMetricsPort | None = None,
|
||||
) -> None:
|
||||
self._repository = repository
|
||||
self._metrics = metrics or get_knowledge_fs_operational_metrics()
|
||||
|
||||
def transition(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
expected_resource_version: int,
|
||||
new_state: KnowledgeFSControlSpaceState,
|
||||
lifecycle_operation_id: str,
|
||||
knowledge_space_id: str | None = None,
|
||||
knowledge_space_revision: int | None = None,
|
||||
last_error_code: str | None = None,
|
||||
last_error_message: str | None = None,
|
||||
) -> KnowledgeFSControlSpace:
|
||||
control_space = self._repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
if control_space is None:
|
||||
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space was not found in this tenant")
|
||||
if control_space.resource_version != expected_resource_version:
|
||||
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS control-space resource version changed")
|
||||
if control_space.deletion_irreversible_at is not None and new_state in {
|
||||
KnowledgeFSControlSpaceState.ACTIVE,
|
||||
KnowledgeFSControlSpaceState.PROVISIONING,
|
||||
}:
|
||||
raise KnowledgeFSInvalidLifecycleTransitionError(
|
||||
"KnowledgeFS control-space deletion is irreversible and cannot be recovered"
|
||||
)
|
||||
if new_state not in _ALLOWED_TRANSITIONS[control_space.state]:
|
||||
raise KnowledgeFSInvalidLifecycleTransitionError(
|
||||
f"Cannot transition KnowledgeFS control-space from {control_space.state} to {new_state}"
|
||||
)
|
||||
|
||||
registered_space_id = control_space.knowledge_space_id
|
||||
if registered_space_id is not None and knowledge_space_id not in {None, registered_space_id}:
|
||||
raise KnowledgeFSSpaceRegistrationConflictError("KnowledgeFS Space registration is immutable")
|
||||
effective_space_id = knowledge_space_id or registered_space_id
|
||||
if new_state is KnowledgeFSControlSpaceState.ACTIVE and effective_space_id is None:
|
||||
raise KnowledgeFSSpaceRegistrationConflictError("An active control-space must have a KnowledgeFS Space ID")
|
||||
if knowledge_space_revision is not None and knowledge_space_revision < control_space.knowledge_space_revision:
|
||||
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS Space revision moved backwards")
|
||||
previous_state = control_space.state
|
||||
previous_updated_at = control_space.updated_at
|
||||
|
||||
changed = self._repository.compare_and_set_lifecycle(
|
||||
KnowledgeFSControlSpaceCASUpdate(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=expected_resource_version,
|
||||
expected_state=control_space.state,
|
||||
new_state=new_state,
|
||||
lifecycle_operation_id=lifecycle_operation_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
last_error_code=last_error_code,
|
||||
last_error_message=last_error_message,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSControlSpaceVersionConflictError("KnowledgeFS control-space changed during transition")
|
||||
|
||||
transitioned = self._repository.get(tenant_id=tenant_id, control_space_id=control_space_id)
|
||||
if transitioned is None:
|
||||
raise KnowledgeFSControlSpaceNotFoundError("KnowledgeFS control-space disappeared after transition")
|
||||
self._record_state_metric(
|
||||
previous_state=previous_state,
|
||||
previous_updated_at=previous_updated_at,
|
||||
transitioned=transitioned,
|
||||
)
|
||||
return transitioned
|
||||
|
||||
def _record_state_metric(
|
||||
self,
|
||||
*,
|
||||
previous_state: KnowledgeFSControlSpaceState,
|
||||
previous_updated_at: datetime,
|
||||
transitioned: KnowledgeFSControlSpace,
|
||||
) -> None:
|
||||
try:
|
||||
duration_seconds = max(0.0, (transitioned.updated_at - previous_updated_at).total_seconds())
|
||||
self._metrics.record_control_space_state(
|
||||
KnowledgeFSControlSpaceStateMetric(
|
||||
duration_seconds,
|
||||
previous_state.value,
|
||||
transitioned.state.value,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("KnowledgeFS control-space state metric export failed", exc_info=True)
|
||||
|
||||
def mark_deletion_irreversible(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
lifecycle_operation_id: str,
|
||||
irreversible_at: datetime,
|
||||
) -> None:
|
||||
"""Persist the monotonic point after which product recovery is forbidden."""
|
||||
|
||||
changed = self._repository.mark_deletion_irreversible(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
lifecycle_operation_id=lifecycle_operation_id,
|
||||
irreversible_at=irreversible_at,
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSControlSpaceVersionConflictError(
|
||||
"KnowledgeFS control-space changed before deletion became irreversible"
|
||||
)
|
||||
|
||||
def assert_workspace_deletion_allowed(self, *, tenant_id: str) -> None:
|
||||
"""Fence tenant deletion until every control-space reaches ``deleted``."""
|
||||
|
||||
blocking_ids = self._repository.list_workspace_deletion_blockers(tenant_id=tenant_id)
|
||||
if blocking_ids:
|
||||
raise KnowledgeFSWorkspaceDeletionBlockedError(blocking_ids)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSControlSpaceLifecycleError",
|
||||
"KnowledgeFSControlSpaceLifecycleService",
|
||||
"KnowledgeFSControlSpaceNotFoundError",
|
||||
"KnowledgeFSControlSpaceVersionConflictError",
|
||||
"KnowledgeFSInvalidLifecycleTransitionError",
|
||||
"KnowledgeFSSpaceRegistrationConflictError",
|
||||
"KnowledgeFSWorkspaceDeletionBlockedError",
|
||||
]
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Operator-only registration, backfill, repair, and dry-run helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from typing import NamedTuple
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpaceState,
|
||||
)
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceIntentConflictError
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceRegistration(NamedTuple):
|
||||
tenant_id: str
|
||||
owner_account_id: str
|
||||
provisioning_key: str
|
||||
knowledge_space_id: str
|
||||
knowledge_space_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceDryRunReport(NamedTuple):
|
||||
total: int
|
||||
by_state: dict[str, int]
|
||||
irreversible_deletions: int
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceBackfillReport(NamedTuple):
|
||||
candidates: int
|
||||
registered: int
|
||||
replays: int
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceManagementService:
|
||||
def __init__(self, session_maker: sessionmaker[Session]):
|
||||
self._session_maker = session_maker
|
||||
|
||||
def dry_run(self, *, tenant_id: str | None = None) -> KnowledgeFSControlSpaceDryRunReport:
|
||||
with self._session_maker() as session:
|
||||
if tenant_id is None:
|
||||
from sqlalchemy import select
|
||||
|
||||
control_spaces = tuple(session.scalars(select(KnowledgeFSControlSpace)))
|
||||
else:
|
||||
control_spaces = SQLAlchemyKnowledgeFSControlSpaceRepository(session).list_for_tenant(
|
||||
tenant_id=tenant_id
|
||||
)
|
||||
counts = Counter(control_space.state.value for control_space in control_spaces)
|
||||
return KnowledgeFSControlSpaceDryRunReport(
|
||||
total=len(control_spaces),
|
||||
by_state=dict(sorted(counts.items())),
|
||||
irreversible_deletions=sum(
|
||||
control_space.deletion_irreversible_at is not None for control_space in control_spaces
|
||||
),
|
||||
)
|
||||
|
||||
def register(self, registration: KnowledgeFSControlSpaceRegistration) -> tuple[KnowledgeFSControlSpace, bool]:
|
||||
with self._session_maker.begin() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
by_key = repository.find_by_provisioning_key(provisioning_key=registration.provisioning_key)
|
||||
by_space = repository.find_by_knowledge_space_id(
|
||||
tenant_id=registration.tenant_id,
|
||||
knowledge_space_id=registration.knowledge_space_id,
|
||||
)
|
||||
existing = by_key or by_space
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.tenant_id != registration.tenant_id
|
||||
or existing.owner_account_id != registration.owner_account_id
|
||||
or existing.provisioning_key != registration.provisioning_key
|
||||
or existing.knowledge_space_id != registration.knowledge_space_id
|
||||
):
|
||||
raise KnowledgeFSControlSpaceIntentConflictError(
|
||||
"KnowledgeFS registration conflicts with an existing local identity"
|
||||
)
|
||||
self._ensure_owner_permission(session, control_space=existing)
|
||||
return existing, True
|
||||
control_space = repository.add(
|
||||
KnowledgeFSControlSpace(
|
||||
tenant_id=registration.tenant_id,
|
||||
owner_account_id=registration.owner_account_id,
|
||||
provisioning_key=registration.provisioning_key,
|
||||
knowledge_space_id=registration.knowledge_space_id,
|
||||
knowledge_space_revision=registration.knowledge_space_revision,
|
||||
state=KnowledgeFSControlSpaceState.ACTIVE,
|
||||
lifecycle_operation_id="management-register",
|
||||
)
|
||||
)
|
||||
session.add(
|
||||
KnowledgeFSAuthorizationRevision(
|
||||
tenant_id=registration.tenant_id,
|
||||
control_space_id=control_space.id,
|
||||
)
|
||||
)
|
||||
self._ensure_owner_permission(session, control_space=control_space)
|
||||
return control_space, False
|
||||
|
||||
@staticmethod
|
||||
def _ensure_owner_permission(session: Session, *, control_space: KnowledgeFSControlSpace) -> None:
|
||||
permission = session.scalar(
|
||||
sa.select(KnowledgeFSControlSpacePermission).where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == control_space.tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == control_space.id,
|
||||
KnowledgeFSControlSpacePermission.account_id == control_space.owner_account_id,
|
||||
)
|
||||
)
|
||||
if permission is None:
|
||||
session.add(
|
||||
KnowledgeFSControlSpacePermission(
|
||||
tenant_id=control_space.tenant_id,
|
||||
control_space_id=control_space.id,
|
||||
account_id=control_space.owner_account_id,
|
||||
role=KnowledgeFSControlSpacePermissionRole.OWNER,
|
||||
granted_by_account_id=control_space.owner_account_id,
|
||||
)
|
||||
)
|
||||
|
||||
def backfill(
|
||||
self,
|
||||
registrations: Iterable[KnowledgeFSControlSpaceRegistration],
|
||||
*,
|
||||
apply: bool,
|
||||
) -> KnowledgeFSControlSpaceBackfillReport:
|
||||
candidates = tuple(registrations)
|
||||
if not apply:
|
||||
return KnowledgeFSControlSpaceBackfillReport(len(candidates), 0, 0)
|
||||
registered = 0
|
||||
replays = 0
|
||||
for registration in candidates:
|
||||
_, replayed = self.register(registration)
|
||||
registered += not replayed
|
||||
replays += replayed
|
||||
return KnowledgeFSControlSpaceBackfillReport(len(candidates), registered, replays)
|
||||
|
||||
def repair_registration(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
control_space_id: str,
|
||||
expected_resource_version: int,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
) -> KnowledgeFSControlSpace:
|
||||
with self._session_maker.begin() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
return KnowledgeFSControlSpaceLifecycleService(repository).transition(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
expected_resource_version=expected_resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.ACTIVE,
|
||||
lifecycle_operation_id="management-repair",
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
knowledge_space_revision=knowledge_space_revision,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSControlSpaceBackfillReport",
|
||||
"KnowledgeFSControlSpaceDryRunReport",
|
||||
"KnowledgeFSControlSpaceManagementService",
|
||||
"KnowledgeFSControlSpaceRegistration",
|
||||
]
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Resource-bound KnowledgeFS service credentials and fail-closed validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, datetime
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSApiCredentialStatus,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
)
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSCredentialCreatePayload,
|
||||
KnowledgeFSCredentialCreateResponse,
|
||||
KnowledgeFSCredentialItemResponse,
|
||||
KnowledgeFSCredentialListResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import (
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS,
|
||||
KnowledgeFSProductPermission,
|
||||
is_product_operation_ready,
|
||||
)
|
||||
from services.knowledge_fs.product_service import KnowledgeFSProductService
|
||||
from services.knowledge_fs.revocation_commands import (
|
||||
KnowledgeFSRevocationCommandPort,
|
||||
KnowledgeFSRevocationCommandProducer,
|
||||
)
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
_SECRET_PREFIX = "kfs_"
|
||||
_DISPLAY_PREFIX_LENGTH = 12
|
||||
|
||||
|
||||
class KnowledgeFSCredentialError(RuntimeError):
|
||||
"""Base class for stable credential failures."""
|
||||
|
||||
|
||||
class KnowledgeFSCredentialValidationError(KnowledgeFSCredentialError):
|
||||
"""A bearer credential is invalid, disabled, expired, or out of scope."""
|
||||
|
||||
|
||||
class KnowledgeFSCredentialPolicyError(KnowledgeFSCredentialError):
|
||||
"""A credential management request violates the product policy."""
|
||||
|
||||
|
||||
class KnowledgeFSServiceCredentialProfile(NamedTuple):
|
||||
tenant_id: str
|
||||
control_space_id: str
|
||||
credential_id: str
|
||||
principal_id: str
|
||||
allowed_actions: frozenset[str]
|
||||
knowledge_space_id: str
|
||||
knowledge_space_revision: int
|
||||
membership_epoch: int
|
||||
space_acl_epoch: int
|
||||
external_access_epoch: int
|
||||
content_policy_revision: int
|
||||
credential_revision: int
|
||||
expires_at: datetime | None
|
||||
|
||||
|
||||
class KnowledgeFSCredentialProfileCache(Protocol):
|
||||
def get(self, credential_hash: str) -> KnowledgeFSServiceCredentialProfile | None: ...
|
||||
|
||||
def set(self, credential_hash: str, profile: KnowledgeFSServiceCredentialProfile) -> None: ...
|
||||
|
||||
def invalidate(self, credential_hash: str) -> None: ...
|
||||
|
||||
|
||||
class NullKnowledgeFSCredentialProfileCache:
|
||||
"""Safe default: no cross-request authorization state is retained."""
|
||||
|
||||
def get(self, credential_hash: str) -> KnowledgeFSServiceCredentialProfile | None:
|
||||
_ = credential_hash
|
||||
return None
|
||||
|
||||
def set(self, credential_hash: str, profile: KnowledgeFSServiceCredentialProfile) -> None:
|
||||
_ = (credential_hash, profile)
|
||||
|
||||
def invalidate(self, credential_hash: str) -> None:
|
||||
_ = credential_hash
|
||||
|
||||
|
||||
def knowledge_fs_service_credential_actions() -> frozenset[str]:
|
||||
"""Return only actions backed by an exact P2 product-operation mapping."""
|
||||
|
||||
actions: set[str] = set()
|
||||
for product_operation_id, product_operation in KNOWLEDGE_FS_PRODUCT_OPERATIONS.items():
|
||||
if not is_product_operation_ready(product_operation_id):
|
||||
continue
|
||||
capability_operation_id = product_operation.capability_operation_id
|
||||
if capability_operation_id is None:
|
||||
continue
|
||||
actions.add(KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id].action)
|
||||
return frozenset(actions)
|
||||
|
||||
|
||||
class KnowledgeFSCredentialService:
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
*,
|
||||
product: KnowledgeFSProductService,
|
||||
cache: KnowledgeFSCredentialProfileCache | None = None,
|
||||
revocations: KnowledgeFSRevocationCommandPort | None = None,
|
||||
) -> None:
|
||||
self._session_maker = session_maker
|
||||
self._product = product
|
||||
self._cache = cache or NullKnowledgeFSCredentialProfileCache()
|
||||
self._revocations = revocations or KnowledgeFSRevocationCommandProducer()
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSCredentialCreatePayload,
|
||||
) -> KnowledgeFSCredentialCreateResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
|
||||
require_active=True,
|
||||
)
|
||||
allowed_actions = _validate_allowed_actions(payload.allowed_actions)
|
||||
expires_at = _normalize_expiry(payload.expires_at)
|
||||
if expires_at is not None and expires_at <= naive_utc_now():
|
||||
raise KnowledgeFSCredentialPolicyError("Credential expiry must be in the future")
|
||||
|
||||
raw_credential = f"{_SECRET_PREFIX}{secrets.token_urlsafe(32)}"
|
||||
credential_hash = hash_knowledge_fs_credential(raw_credential)
|
||||
credential = KnowledgeFSApiCredential(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
credential_hash=credential_hash,
|
||||
credential_prefix=raw_credential[:_DISPLAY_PREFIX_LENGTH],
|
||||
credential_last4=raw_credential[-4:],
|
||||
principal="pending",
|
||||
allowed_actions=list(allowed_actions),
|
||||
expires_at=expires_at,
|
||||
created_by_account_id=actor_account_id,
|
||||
)
|
||||
credential.principal = credential.id
|
||||
with self._session_maker.begin() as session:
|
||||
session.add(credential)
|
||||
return KnowledgeFSCredentialCreateResponse(
|
||||
id=credential.id,
|
||||
credential=raw_credential,
|
||||
credential_prefix=credential.credential_prefix,
|
||||
credential_last4=credential.credential_last4,
|
||||
principal=credential.principal,
|
||||
allowed_actions=list(allowed_actions),
|
||||
expires_at=credential.expires_at,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSCredentialListResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
|
||||
)
|
||||
with self._session_maker() as session:
|
||||
credentials = tuple(
|
||||
session.scalars(
|
||||
sa.select(KnowledgeFSApiCredential)
|
||||
.where(
|
||||
KnowledgeFSApiCredential.tenant_id == tenant_id,
|
||||
KnowledgeFSApiCredential.control_space_id == control_space_id,
|
||||
)
|
||||
.order_by(KnowledgeFSApiCredential.created_at.desc(), KnowledgeFSApiCredential.id.desc())
|
||||
)
|
||||
)
|
||||
return KnowledgeFSCredentialListResponse(data=[_credential_item(item) for item in credentials])
|
||||
|
||||
def revoke(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
control_space_id: str,
|
||||
credential_id: str,
|
||||
reason: str = "revoked_by_user",
|
||||
) -> None:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=actor_account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.API_KEY_MANAGE,
|
||||
)
|
||||
credential_hash: str | None = None
|
||||
with self._session_maker.begin() as session:
|
||||
credential = session.scalar(
|
||||
sa.select(KnowledgeFSApiCredential)
|
||||
.where(
|
||||
KnowledgeFSApiCredential.tenant_id == tenant_id,
|
||||
KnowledgeFSApiCredential.control_space_id == control_space_id,
|
||||
KnowledgeFSApiCredential.id == credential_id,
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if credential is None:
|
||||
raise KnowledgeFSCredentialPolicyError("KnowledgeFS credential was not found")
|
||||
credential_hash = credential.credential_hash
|
||||
if credential.status is KnowledgeFSApiCredentialStatus.ACTIVE:
|
||||
credential.status = KnowledgeFSApiCredentialStatus.REVOKED
|
||||
credential.revision += 1
|
||||
credential.revoked_at = naive_utc_now()
|
||||
credential.revoked_by_account_id = actor_account_id
|
||||
credential.revoke_reason = reason[:255]
|
||||
self._revocations.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
subject=f"dify-kfs-credential:{credential.principal}",
|
||||
reason_code="credential_revoked",
|
||||
caller_kinds=("service",),
|
||||
)
|
||||
if credential_hash is not None:
|
||||
self._cache.invalidate(credential_hash)
|
||||
|
||||
def validate_service_credential(
|
||||
self,
|
||||
*,
|
||||
raw_credential: str,
|
||||
required_action: str,
|
||||
) -> KnowledgeFSServiceCredentialProfile:
|
||||
if not raw_credential.startswith(_SECRET_PREFIX) or len(raw_credential) < 32:
|
||||
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
|
||||
credential_hash = hash_knowledge_fs_credential(raw_credential)
|
||||
cached = self._cache.get(credential_hash)
|
||||
if cached is not None:
|
||||
if cached.expires_at is not None and cached.expires_at <= naive_utc_now():
|
||||
self._cache.invalidate(credential_hash)
|
||||
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
|
||||
if required_action not in cached.allowed_actions:
|
||||
raise KnowledgeFSCredentialValidationError("Credential action is not allowed")
|
||||
return cached
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
row = session.execute(
|
||||
sa.select(
|
||||
KnowledgeFSApiCredential,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSControlSpace,
|
||||
sa.and_(
|
||||
KnowledgeFSControlSpace.tenant_id == KnowledgeFSApiCredential.tenant_id,
|
||||
KnowledgeFSControlSpace.id == KnowledgeFSApiCredential.control_space_id,
|
||||
),
|
||||
)
|
||||
.outerjoin(
|
||||
KnowledgeFSExternalAccessPolicy,
|
||||
sa.and_(
|
||||
KnowledgeFSExternalAccessPolicy.tenant_id == KnowledgeFSApiCredential.tenant_id,
|
||||
KnowledgeFSExternalAccessPolicy.control_space_id == KnowledgeFSApiCredential.control_space_id,
|
||||
),
|
||||
)
|
||||
.join(
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
sa.and_(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == KnowledgeFSApiCredential.tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id == KnowledgeFSApiCredential.control_space_id,
|
||||
),
|
||||
)
|
||||
.where(KnowledgeFSApiCredential.credential_hash == credential_hash)
|
||||
.with_for_update(of=KnowledgeFSApiCredential)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
|
||||
credential, control_space, policy, revision = row._t
|
||||
now = naive_utc_now()
|
||||
if credential.expires_at is not None and credential.expires_at <= now:
|
||||
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
|
||||
if (
|
||||
credential.status is not KnowledgeFSApiCredentialStatus.ACTIVE
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.ACTIVE
|
||||
or control_space.knowledge_space_id is None
|
||||
or policy is None
|
||||
or not policy.service_api_enabled
|
||||
or required_action not in credential.allowed_actions
|
||||
):
|
||||
raise KnowledgeFSCredentialValidationError("Invalid KnowledgeFS service credential")
|
||||
credential.last_used_at = now
|
||||
profile = KnowledgeFSServiceCredentialProfile(
|
||||
tenant_id=credential.tenant_id,
|
||||
control_space_id=credential.control_space_id,
|
||||
credential_id=credential.id,
|
||||
principal_id=credential.principal,
|
||||
allowed_actions=frozenset(credential.allowed_actions),
|
||||
knowledge_space_id=control_space.knowledge_space_id,
|
||||
knowledge_space_revision=control_space.knowledge_space_revision,
|
||||
membership_epoch=revision.membership_epoch,
|
||||
space_acl_epoch=revision.space_acl_epoch,
|
||||
external_access_epoch=revision.external_access_epoch,
|
||||
content_policy_revision=revision.content_policy_revision,
|
||||
credential_revision=credential.revision,
|
||||
expires_at=credential.expires_at,
|
||||
)
|
||||
self._cache.set(credential_hash, profile)
|
||||
return profile
|
||||
|
||||
|
||||
def hash_knowledge_fs_credential(raw_credential: str) -> str:
|
||||
return f"sha256:{hashlib.sha256(raw_credential.encode()).hexdigest()}"
|
||||
|
||||
|
||||
def _validate_allowed_actions(actions: Sequence[str]) -> tuple[str, ...]:
|
||||
normalized = tuple(dict.fromkeys(action.strip() for action in actions if action.strip()))
|
||||
if not normalized or len(normalized) != len(actions):
|
||||
raise KnowledgeFSCredentialPolicyError("Credential actions must be unique and non-empty")
|
||||
supported = knowledge_fs_service_credential_actions()
|
||||
if not set(normalized).issubset(supported):
|
||||
raise KnowledgeFSCredentialPolicyError("Credential action is not registered for the service profile")
|
||||
return tuple(sorted(normalized))
|
||||
|
||||
|
||||
def _normalize_expiry(value: datetime | None) -> datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
return value
|
||||
return value.astimezone(UTC).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _credential_item(credential: KnowledgeFSApiCredential) -> KnowledgeFSCredentialItemResponse:
|
||||
status = credential.status
|
||||
if (
|
||||
status is KnowledgeFSApiCredentialStatus.ACTIVE
|
||||
and credential.expires_at is not None
|
||||
and credential.expires_at <= naive_utc_now()
|
||||
):
|
||||
status = KnowledgeFSApiCredentialStatus.EXPIRED
|
||||
return KnowledgeFSCredentialItemResponse(
|
||||
id=credential.id,
|
||||
credential_prefix=credential.credential_prefix,
|
||||
credential_last4=credential.credential_last4,
|
||||
principal=credential.principal,
|
||||
allowed_actions=list(credential.allowed_actions),
|
||||
status=status.value,
|
||||
revision=credential.revision,
|
||||
expires_at=credential.expires_at,
|
||||
last_used_at=credential.last_used_at,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCredentialError",
|
||||
"KnowledgeFSCredentialPolicyError",
|
||||
"KnowledgeFSCredentialProfileCache",
|
||||
"KnowledgeFSCredentialService",
|
||||
"KnowledgeFSCredentialValidationError",
|
||||
"KnowledgeFSServiceCredentialProfile",
|
||||
"NullKnowledgeFSCredentialProfileCache",
|
||||
"hash_knowledge_fs_credential",
|
||||
"knowledge_fs_service_credential_actions",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
"""Fail-closed runtime admission for per-Workspace KnowledgeFS cutover."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs_cutover import (
|
||||
KnowledgeFSWorkspaceCutoverLedger,
|
||||
KnowledgeFSWorkspaceCutoverPhase,
|
||||
)
|
||||
from services.knowledge_fs.product_remote import KnowledgeFSOperationUnavailableError
|
||||
|
||||
_TRAFFIC_PHASES = frozenset(
|
||||
{
|
||||
KnowledgeFSWorkspaceCutoverPhase.CUTOVER,
|
||||
KnowledgeFSWorkspaceCutoverPhase.OBSERVING,
|
||||
KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceRuntimeGatePort(Protocol):
|
||||
"""Workspace-scoped admission checked before product traffic or token issuance."""
|
||||
|
||||
def require_product_routes(self, *, tenant_id: str) -> None: ...
|
||||
|
||||
def require_capability_v2(self, *, tenant_id: str) -> None: ...
|
||||
|
||||
|
||||
class SQLKnowledgeFSWorkspaceRuntimeGate:
|
||||
"""Read the atomic cutover row and reject missing or partial rollout state."""
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session]) -> None:
|
||||
self._session_maker = session_maker
|
||||
|
||||
def require_product_routes(self, *, tenant_id: str) -> None:
|
||||
self._require_complete_cutover(tenant_id=tenant_id)
|
||||
|
||||
def require_capability_v2(self, *, tenant_id: str) -> None:
|
||||
self._require_complete_cutover(tenant_id=tenant_id)
|
||||
|
||||
def _require_complete_cutover(self, *, tenant_id: str) -> None:
|
||||
with self._session_maker() as session:
|
||||
ledger = session.scalar(
|
||||
sa.select(KnowledgeFSWorkspaceCutoverLedger).where(
|
||||
KnowledgeFSWorkspaceCutoverLedger.tenant_id == tenant_id
|
||||
)
|
||||
)
|
||||
if (
|
||||
ledger is None
|
||||
or ledger.phase not in _TRAFFIC_PHASES
|
||||
or ledger.cutover_at is None
|
||||
or ledger.rolled_back_at is not None
|
||||
or not ledger.product_routes_enabled
|
||||
or not ledger.capability_v2_enabled
|
||||
or not ledger.integrated_mode_enabled
|
||||
or not ledger.legacy_acl_read_only
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS Workspace is not cut over for product traffic")
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSWorkspaceRuntimeGatePort", "SQLKnowledgeFSWorkspaceRuntimeGate"]
|
||||
@@ -0,0 +1,998 @@
|
||||
"""Typed KnowledgeFS BFF facade over operation capabilities and bounded transports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, JsonValue
|
||||
|
||||
from services.knowledge_fs.capability_broker import KnowledgeFSCapabilityBroker
|
||||
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
|
||||
from services.knowledge_fs.operation_admission import KnowledgeFSOperationAdmissionService
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSAnswerTraceResponse,
|
||||
KnowledgeFSBulkDeletionAcceptedResponse,
|
||||
KnowledgeFSBulkDocumentDeletePayload,
|
||||
KnowledgeFSBulkJobResponse,
|
||||
KnowledgeFSDocumentChunkListResponse,
|
||||
KnowledgeFSDocumentChunkResponse,
|
||||
KnowledgeFSDocumentCompilationJobResponse,
|
||||
KnowledgeFSDocumentCreatePayload,
|
||||
KnowledgeFSDocumentDeletePayload,
|
||||
KnowledgeFSDocumentListResponse,
|
||||
KnowledgeFSDocumentMetadataPayload,
|
||||
KnowledgeFSDocumentOutlineResponse,
|
||||
KnowledgeFSDocumentReindexPayload,
|
||||
KnowledgeFSDocumentReindexResponse,
|
||||
KnowledgeFSDocumentResponse,
|
||||
KnowledgeFSDocumentRevisionListResponse,
|
||||
KnowledgeFSDurableDeletionAcceptedResponse,
|
||||
KnowledgeFSLogicalDocumentResponse,
|
||||
KnowledgeFSQueryCreatePayload,
|
||||
KnowledgeFSQueryResponse,
|
||||
KnowledgeFSResearchTaskCreatePayload,
|
||||
KnowledgeFSResearchTaskListResponse,
|
||||
KnowledgeFSResearchTaskPartialListResponse,
|
||||
KnowledgeFSResearchTaskPlanPayload,
|
||||
KnowledgeFSResearchTaskPlanResponse,
|
||||
KnowledgeFSResearchTaskResponse,
|
||||
KnowledgeFSSettingsPayload,
|
||||
KnowledgeFSSettingsResponse,
|
||||
KnowledgeFSSmallFileUploadResponse,
|
||||
KnowledgeFSSourceCrawlResponse,
|
||||
KnowledgeFSSourceCreatePayload,
|
||||
KnowledgeFSSourceCredentialTestResponse,
|
||||
KnowledgeFSSourceDeletePayload,
|
||||
KnowledgeFSSourceFilesResponse,
|
||||
KnowledgeFSSourceImportFilesPayload,
|
||||
KnowledgeFSSourceImportPagesPayload,
|
||||
KnowledgeFSSourceImportResponse,
|
||||
KnowledgeFSSourceListResponse,
|
||||
KnowledgeFSSourcePagesResponse,
|
||||
KnowledgeFSSourceResponse,
|
||||
KnowledgeFSSourceUpdatePayload,
|
||||
KnowledgeFSSpaceUpdatePayload,
|
||||
KnowledgeFSTraceEntryListResponse,
|
||||
KnowledgeFSTraceListResponse,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
|
||||
from services.knowledge_fs.product_remote import (
|
||||
KnowledgeFSOperationUnavailableError,
|
||||
KnowledgeFSProductRemotePort,
|
||||
KnowledgeFSProductRequestRejectedError,
|
||||
KnowledgeFSRemoteBinaryRequest,
|
||||
KnowledgeFSRemoteJSONRequest,
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSDataFacade:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
admission: KnowledgeFSOperationAdmissionService,
|
||||
broker: KnowledgeFSCapabilityBroker,
|
||||
remote: KnowledgeFSProductRemotePort,
|
||||
) -> None:
|
||||
self._admission = admission
|
||||
self._broker = broker
|
||||
self._remote = remote
|
||||
|
||||
def get_settings(self, *, tenant_id: str, account_id: str, control_space_id: str) -> KnowledgeFSSettingsResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getSettings",
|
||||
)
|
||||
return KnowledgeFSSettingsResponse.model_validate(raw)
|
||||
|
||||
def update_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSSpaceUpdatePayload,
|
||||
) -> JsonValue:
|
||||
return self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="updateSpace",
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
def update_settings(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSSettingsPayload,
|
||||
) -> KnowledgeFSSettingsResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="updateSettings",
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSSettingsResponse.model_validate(raw)
|
||||
|
||||
def list_documents(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
cursor: str | None,
|
||||
) -> KnowledgeFSDocumentListResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listDocuments",
|
||||
query=(("cursor", cursor),) if cursor else (),
|
||||
)
|
||||
return KnowledgeFSDocumentListResponse.model_validate(raw)
|
||||
|
||||
def create_document(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSDocumentCreatePayload,
|
||||
) -> KnowledgeFSDocumentResponse:
|
||||
_ = (tenant_id, account_id, control_space_id, payload)
|
||||
raise KnowledgeFSOperationUnavailableError(
|
||||
"Buffered KnowledgeFS document creation is deprecated; use the P6 direct-upload capability flow"
|
||||
)
|
||||
|
||||
def upload_small_file(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
upload_session_id: str,
|
||||
body_reader: Callable[[int], bytes],
|
||||
) -> KnowledgeFSSmallFileUploadResponse:
|
||||
operation_id = "uploadSmallFile"
|
||||
_assert_binary_bff_ready(operation_id)
|
||||
with self._admitted(tenant_id=tenant_id, operation_id=operation_id):
|
||||
issued = self._broker.issue_interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
resource_id=upload_session_id,
|
||||
)
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
|
||||
body = body_reader(operation.max_request_bytes)
|
||||
if not isinstance(body, bytes) or not body:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=422)
|
||||
if len(body) > operation.max_request_bytes:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=413)
|
||||
if operation.kfs_path is None:
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
|
||||
path = _resolve_product_path(
|
||||
template=operation.kfs_path,
|
||||
knowledge_space_id=issued.knowledge_space_id,
|
||||
resource_id=upload_session_id,
|
||||
resource_resolver=operation.resource_resolver,
|
||||
path_parameters=(),
|
||||
)
|
||||
raw = self._remote.execute_binary(
|
||||
KnowledgeFSRemoteBinaryRequest(
|
||||
operation_id=operation_id,
|
||||
method=operation.method,
|
||||
path=path,
|
||||
namespace_id=tenant_id,
|
||||
knowledge_space_id=issued.knowledge_space_id,
|
||||
capability_token=issued.token,
|
||||
trace_id=issued.trace_id,
|
||||
body=body,
|
||||
query=(("knowledgeSpaceId", issued.knowledge_space_id),),
|
||||
)
|
||||
)
|
||||
return KnowledgeFSSmallFileUploadResponse.model_validate(raw)
|
||||
|
||||
def get_document(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, document_id: str
|
||||
) -> KnowledgeFSDocumentResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getDocument",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id),),
|
||||
)
|
||||
return KnowledgeFSDocumentResponse.model_validate(raw)
|
||||
|
||||
def get_document_outline(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, document_id: str
|
||||
) -> KnowledgeFSDocumentOutlineResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getDocumentOutline",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id),),
|
||||
)
|
||||
return KnowledgeFSDocumentOutlineResponse.model_validate(raw)
|
||||
|
||||
def list_document_revisions(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
document_id: str,
|
||||
cursor: str | None = None,
|
||||
) -> KnowledgeFSDocumentRevisionListResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listDocumentRevisions",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id),),
|
||||
query=(("cursor", cursor),) if cursor else (),
|
||||
)
|
||||
return KnowledgeFSDocumentRevisionListResponse.model_validate(raw)
|
||||
|
||||
def update_document_metadata(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
document_id: str,
|
||||
payload: KnowledgeFSDocumentMetadataPayload,
|
||||
) -> KnowledgeFSLogicalDocumentResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="updateDocumentMetadata",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id),),
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSLogicalDocumentResponse.model_validate(raw)
|
||||
|
||||
def list_document_chunks(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
document_id: str,
|
||||
revision: int,
|
||||
cursor: str | None = None,
|
||||
query_text: str | None = None,
|
||||
) -> KnowledgeFSDocumentChunkListResponse:
|
||||
query = tuple((name, value) for name, value in (("cursor", cursor), ("query", query_text)) if value)
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listDocumentChunks",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id), ("revision", str(revision))),
|
||||
query=query,
|
||||
)
|
||||
return KnowledgeFSDocumentChunkListResponse.model_validate(raw)
|
||||
|
||||
def get_document_chunk(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
document_id: str,
|
||||
revision: int,
|
||||
chunk_id: str,
|
||||
) -> KnowledgeFSDocumentChunkResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getDocumentChunk",
|
||||
resource_id=document_id,
|
||||
path_parameters=(
|
||||
("documentId", document_id),
|
||||
("revision", str(revision)),
|
||||
("chunkId", chunk_id),
|
||||
),
|
||||
)
|
||||
return KnowledgeFSDocumentChunkResponse.model_validate(raw)
|
||||
|
||||
def delete_document(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
document_id: str,
|
||||
payload: KnowledgeFSDocumentDeletePayload,
|
||||
idempotency_key: str,
|
||||
) -> KnowledgeFSDurableDeletionAcceptedResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="deleteDocument",
|
||||
resource_id=document_id,
|
||||
path_parameters=(("documentId", document_id),),
|
||||
payload=payload,
|
||||
headers=(("Idempotency-Key", idempotency_key),),
|
||||
)
|
||||
return KnowledgeFSDurableDeletionAcceptedResponse.model_validate(raw)
|
||||
|
||||
def bulk_delete_documents(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSBulkDocumentDeletePayload,
|
||||
idempotency_key: str,
|
||||
) -> KnowledgeFSBulkDeletionAcceptedResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="bulkDeleteDocuments",
|
||||
payload=payload,
|
||||
headers=(("Idempotency-Key", idempotency_key),),
|
||||
)
|
||||
return KnowledgeFSBulkDeletionAcceptedResponse.model_validate(raw)
|
||||
|
||||
def reindex_documents(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSDocumentReindexPayload,
|
||||
) -> KnowledgeFSDocumentReindexResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="reindexDocuments",
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSDocumentReindexResponse.model_validate(raw)
|
||||
|
||||
def get_compilation_job(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
|
||||
) -> KnowledgeFSDocumentCompilationJobResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getCompilationJob",
|
||||
resource_id=job_id,
|
||||
)
|
||||
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
|
||||
|
||||
def cancel_compilation_job(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
|
||||
) -> KnowledgeFSDocumentCompilationJobResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="cancelCompilationJob",
|
||||
resource_id=job_id,
|
||||
)
|
||||
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
|
||||
|
||||
def retry_compilation_job(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
|
||||
) -> KnowledgeFSDocumentCompilationJobResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="retryCompilationJob",
|
||||
resource_id=job_id,
|
||||
)
|
||||
return KnowledgeFSDocumentCompilationJobResponse.model_validate(raw)
|
||||
|
||||
def get_bulk_job(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, job_id: str
|
||||
) -> KnowledgeFSBulkJobResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getBulkJob",
|
||||
resource_id=job_id,
|
||||
)
|
||||
return KnowledgeFSBulkJobResponse.model_validate(raw)
|
||||
|
||||
def list_sources(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
|
||||
) -> KnowledgeFSSourceListResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listSources",
|
||||
query=(("cursor", cursor),) if cursor else (),
|
||||
)
|
||||
return KnowledgeFSSourceListResponse.model_validate(raw)
|
||||
|
||||
def create_source(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSSourceCreatePayload,
|
||||
) -> KnowledgeFSSourceResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="createSource",
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSSourceResponse.model_validate(raw)
|
||||
|
||||
def get_source(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
|
||||
) -> KnowledgeFSSourceResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getSource",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
)
|
||||
return KnowledgeFSSourceResponse.model_validate(raw)
|
||||
|
||||
def update_source(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
payload: KnowledgeFSSourceUpdatePayload,
|
||||
) -> KnowledgeFSSourceResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="updateSource",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSSourceResponse.model_validate(raw)
|
||||
|
||||
def delete_source(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
payload: KnowledgeFSSourceDeletePayload,
|
||||
documents: Literal["cascade", "keep"],
|
||||
idempotency_key: str,
|
||||
) -> KnowledgeFSDurableDeletionAcceptedResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="deleteSource",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
payload=payload,
|
||||
query=(("documents", documents),),
|
||||
headers=(("Idempotency-Key", idempotency_key),),
|
||||
)
|
||||
return KnowledgeFSDurableDeletionAcceptedResponse.model_validate(raw)
|
||||
|
||||
def test_source(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
|
||||
) -> KnowledgeFSSourceCredentialTestResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="testSource",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
)
|
||||
return KnowledgeFSSourceCredentialTestResponse.model_validate(raw)
|
||||
|
||||
def crawl_source(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, source_id: str
|
||||
) -> KnowledgeFSSourceCrawlResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="crawlSource",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
)
|
||||
return KnowledgeFSSourceCrawlResponse.model_validate(raw)
|
||||
|
||||
def list_source_pages(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
cursor: str | None = None,
|
||||
limit: int = 50,
|
||||
) -> KnowledgeFSSourcePagesResponse:
|
||||
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listSourcePages",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
query=query,
|
||||
)
|
||||
return KnowledgeFSSourcePagesResponse.model_validate(raw)
|
||||
|
||||
def import_source_pages(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
payload: KnowledgeFSSourceImportPagesPayload,
|
||||
) -> KnowledgeFSSourceImportResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="importSourcePages",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSSourceImportResponse.model_validate(raw)
|
||||
|
||||
def list_source_files(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
query: tuple[tuple[str, str], ...] = (),
|
||||
) -> KnowledgeFSSourceFilesResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listSourceFiles",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
query=query,
|
||||
)
|
||||
return KnowledgeFSSourceFilesResponse.model_validate(raw)
|
||||
|
||||
def import_source_files(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
source_id: str,
|
||||
payload: KnowledgeFSSourceImportFilesPayload,
|
||||
) -> KnowledgeFSSourceImportResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="importSourceFiles",
|
||||
resource_id=source_id,
|
||||
path_parameters=(("sourceId", source_id),),
|
||||
payload=payload,
|
||||
)
|
||||
return KnowledgeFSSourceImportResponse.model_validate(raw)
|
||||
|
||||
def create_query(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSQueryCreatePayload,
|
||||
) -> KnowledgeFSQueryResponse:
|
||||
_ = (tenant_id, account_id, control_space_id, payload)
|
||||
raise KnowledgeFSOperationUnavailableError(
|
||||
"Buffered KnowledgeFS query creation is deprecated; use the queries/admission direct flow"
|
||||
)
|
||||
|
||||
def create_research_task(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSResearchTaskCreatePayload,
|
||||
) -> KnowledgeFSResearchTaskResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="createResearchTask",
|
||||
payload=payload,
|
||||
bind_space_in_body=True,
|
||||
)
|
||||
return KnowledgeFSResearchTaskResponse.model_validate(raw)
|
||||
|
||||
def plan_research_task(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSResearchTaskPlanPayload,
|
||||
) -> KnowledgeFSResearchTaskPlanResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="planResearchTask",
|
||||
payload=payload,
|
||||
bind_space_in_body=True,
|
||||
)
|
||||
return KnowledgeFSResearchTaskPlanResponse.model_validate(raw)
|
||||
|
||||
def get_research_task(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, task_id: str
|
||||
) -> KnowledgeFSResearchTaskResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getResearchTask",
|
||||
resource_id=task_id,
|
||||
)
|
||||
return KnowledgeFSResearchTaskResponse.model_validate(raw)
|
||||
|
||||
def list_research_task_partials(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
task_id: str,
|
||||
cursor: str | None = None,
|
||||
limit: int = 25,
|
||||
) -> KnowledgeFSResearchTaskPartialListResponse:
|
||||
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listResearchTaskPartials",
|
||||
resource_id=task_id,
|
||||
query=query,
|
||||
)
|
||||
return KnowledgeFSResearchTaskPartialListResponse.model_validate(raw)
|
||||
|
||||
def cancel_research_task(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, task_id: str
|
||||
) -> KnowledgeFSResearchTaskResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="cancelResearchTask",
|
||||
resource_id=task_id,
|
||||
)
|
||||
return KnowledgeFSResearchTaskResponse.model_validate(raw)
|
||||
|
||||
def list_research_tasks(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
|
||||
) -> KnowledgeFSResearchTaskListResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listResearchTasks",
|
||||
query=(("cursor", cursor),) if cursor else (),
|
||||
)
|
||||
return KnowledgeFSResearchTaskListResponse.model_validate(raw)
|
||||
|
||||
def list_traces(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, cursor: str | None = None
|
||||
) -> KnowledgeFSTraceListResponse:
|
||||
raw = self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="listTraces",
|
||||
query=(("cursor", cursor),) if cursor else (),
|
||||
)
|
||||
return KnowledgeFSTraceListResponse.model_validate(raw)
|
||||
|
||||
def get_trace(
|
||||
self, *, tenant_id: str, account_id: str, control_space_id: str, trace_id: str
|
||||
) -> KnowledgeFSAnswerTraceResponse:
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id="getTrace",
|
||||
resource_id=trace_id,
|
||||
path_parameters=(("traceId", trace_id),),
|
||||
)
|
||||
return KnowledgeFSAnswerTraceResponse.model_validate(raw)
|
||||
|
||||
def list_trace_entries(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
trace_id: str,
|
||||
kind: Literal["conflicts", "evidence", "missing"],
|
||||
cursor: str | None = None,
|
||||
limit: int = 100,
|
||||
) -> KnowledgeFSTraceEntryListResponse:
|
||||
operation_id = {
|
||||
"conflicts": "listTraceConflicts",
|
||||
"evidence": "listTraceEvidence",
|
||||
"missing": "listTraceMissing",
|
||||
}[kind]
|
||||
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
resource_id=trace_id,
|
||||
path_parameters=(("traceId", trace_id),),
|
||||
query=query,
|
||||
)
|
||||
return KnowledgeFSTraceEntryListResponse.model_validate(raw)
|
||||
|
||||
def execute_service(
|
||||
self,
|
||||
*,
|
||||
profile: KnowledgeFSServiceCredentialProfile,
|
||||
operation_id: str,
|
||||
payload: BaseModel | None = None,
|
||||
query: tuple[tuple[str, str], ...] = (),
|
||||
bind_space_in_body: bool = False,
|
||||
resource_id: str | None = None,
|
||||
path_parameters: tuple[tuple[str, str], ...] = (),
|
||||
headers: tuple[tuple[str, str], ...] = (),
|
||||
) -> JsonValue:
|
||||
_assert_json_bff_ready(operation_id)
|
||||
with self._admitted(tenant_id=profile.tenant_id, operation_id=operation_id):
|
||||
issued = self._broker.issue_service(profile=profile, operation_id=operation_id, resource_id=resource_id)
|
||||
return self._execute(
|
||||
operation_id=operation_id,
|
||||
namespace_id=profile.tenant_id,
|
||||
knowledge_space_id=profile.knowledge_space_id,
|
||||
knowledge_space_revision=profile.knowledge_space_revision,
|
||||
capability_token=issued.token,
|
||||
trace_id=issued.trace_id,
|
||||
payload=payload,
|
||||
query=query,
|
||||
bind_space_in_body=bind_space_in_body,
|
||||
resource_id=resource_id,
|
||||
path_parameters=path_parameters,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def _interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
payload: BaseModel | None = None,
|
||||
query: tuple[tuple[str, str], ...] = (),
|
||||
bind_space_in_body: bool = False,
|
||||
resource_id: str | None = None,
|
||||
path_parameters: tuple[tuple[str, str], ...] = (),
|
||||
headers: tuple[tuple[str, str], ...] = (),
|
||||
) -> JsonValue:
|
||||
_assert_json_bff_ready(operation_id)
|
||||
with self._admitted(tenant_id=tenant_id, operation_id=operation_id):
|
||||
issued = self._broker.issue_interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
resource_id=resource_id,
|
||||
)
|
||||
return self._execute(
|
||||
operation_id=operation_id,
|
||||
namespace_id=tenant_id,
|
||||
knowledge_space_id=issued.knowledge_space_id,
|
||||
knowledge_space_revision=issued.knowledge_space_revision,
|
||||
capability_token=issued.token,
|
||||
trace_id=issued.trace_id,
|
||||
payload=payload,
|
||||
query=query,
|
||||
bind_space_in_body=bind_space_in_body,
|
||||
resource_id=resource_id,
|
||||
path_parameters=path_parameters,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _admitted(self, *, tenant_id: str, operation_id: str) -> Generator[None]:
|
||||
charge = self._admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
|
||||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
charge.refund()
|
||||
raise
|
||||
else:
|
||||
charge.commit()
|
||||
|
||||
def _interactive_child(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
resource_id: str,
|
||||
payload: BaseModel | None = None,
|
||||
query: tuple[tuple[str, str], ...] = (),
|
||||
path_parameters: tuple[tuple[str, str], ...] = (),
|
||||
headers: tuple[tuple[str, str], ...] = (),
|
||||
) -> JsonValue:
|
||||
return self._interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
resource_id=resource_id,
|
||||
payload=payload,
|
||||
query=query,
|
||||
path_parameters=path_parameters,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def _execute(
|
||||
self,
|
||||
*,
|
||||
operation_id: str,
|
||||
namespace_id: str,
|
||||
knowledge_space_id: str,
|
||||
knowledge_space_revision: int,
|
||||
capability_token: str,
|
||||
trace_id: str,
|
||||
payload: BaseModel | None,
|
||||
query: tuple[tuple[str, str], ...],
|
||||
bind_space_in_body: bool,
|
||||
resource_id: str | None,
|
||||
path_parameters: tuple[tuple[str, str], ...],
|
||||
headers: tuple[tuple[str, str], ...],
|
||||
) -> JsonValue:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
|
||||
if operation.kfs_path is None:
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
|
||||
path = _resolve_product_path(
|
||||
template=operation.kfs_path,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
resource_id=resource_id,
|
||||
resource_resolver=operation.resource_resolver,
|
||||
path_parameters=path_parameters,
|
||||
)
|
||||
remote_payload: JsonValue | None = (
|
||||
payload.model_dump(mode="json", exclude_none=True, by_alias=True) if payload is not None else None
|
||||
)
|
||||
if operation_id == "updateSpace":
|
||||
if knowledge_space_revision <= 0 or not isinstance(payload, KnowledgeFSSpaceUpdatePayload):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS metadata revision is not available")
|
||||
remote_payload = {"expectedRevision": knowledge_space_revision}
|
||||
if payload.name is not None:
|
||||
remote_payload["name"] = payload.name
|
||||
if payload.icon is not None:
|
||||
remote_payload["iconRef"] = payload.icon
|
||||
if payload.description is not None:
|
||||
remote_payload["description"] = payload.description
|
||||
if bind_space_in_body:
|
||||
if not isinstance(remote_payload, dict):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request body binding is unavailable")
|
||||
remote_payload["knowledgeSpaceId"] = knowledge_space_id
|
||||
if operation.resource_resolver in {"job", "query", "research_task"} and not path.startswith(
|
||||
"/knowledge-spaces/"
|
||||
):
|
||||
query = (*query, ("knowledgeSpaceId", knowledge_space_id))
|
||||
return self._remote.execute_json(
|
||||
KnowledgeFSRemoteJSONRequest(
|
||||
operation_id=operation_id,
|
||||
method=operation.method,
|
||||
path=path,
|
||||
namespace_id=namespace_id,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
capability_token=capability_token,
|
||||
trace_id=trace_id,
|
||||
payload=remote_payload,
|
||||
query=query,
|
||||
headers=headers,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_product_path(
|
||||
*,
|
||||
template: str,
|
||||
knowledge_space_id: str,
|
||||
resource_id: str | None,
|
||||
resource_resolver: str,
|
||||
path_parameters: tuple[tuple[str, str], ...],
|
||||
) -> str:
|
||||
bindings = {name: _path_segment(value) for name, value in path_parameters}
|
||||
if template.startswith("/knowledge-spaces/{id}") or resource_resolver in {"knowledge_space", "namespace"}:
|
||||
bindings["id"] = _path_segment(knowledge_space_id)
|
||||
elif "{id}" in template:
|
||||
if resource_id is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS child resource is required")
|
||||
bindings["id"] = _path_segment(resource_id)
|
||||
path = template
|
||||
for name, value in bindings.items():
|
||||
path = path.replace(f"{{{name}}}", value)
|
||||
if "{" in path or "}" in path:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product path resolver is not registered")
|
||||
return path
|
||||
|
||||
|
||||
def _path_segment(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if (
|
||||
not normalized
|
||||
or len(normalized) > 1_000
|
||||
or normalized in {".", ".."}
|
||||
or any(character in normalized for character in ("/", "%", "?", "#", "\\"))
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product path parameter is invalid")
|
||||
return normalized
|
||||
|
||||
|
||||
def _assert_ready(operation_id: str) -> None:
|
||||
if not is_product_operation_ready(operation_id):
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {operation_id}")
|
||||
|
||||
|
||||
def _assert_json_bff_ready(operation_id: str) -> None:
|
||||
_assert_ready(operation_id)
|
||||
if KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].transport != "json":
|
||||
raise KnowledgeFSOperationUnavailableError(
|
||||
f"KnowledgeFS operation requires a direct transport and cannot use the buffered BFF: {operation_id}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_binary_bff_ready(operation_id: str) -> None:
|
||||
_assert_ready(operation_id)
|
||||
if KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].transport != "binary":
|
||||
raise KnowledgeFSOperationUnavailableError(
|
||||
f"KnowledgeFS operation does not allow the bounded binary BFF: {operation_id}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSDataFacade"]
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Remote port for the KnowledgeFS lifecycle data-plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Literal, NamedTuple, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSModelSelectionIntentPayload, KnowledgeFSRetrievalProfileIntentPayload
|
||||
|
||||
|
||||
class KnowledgeFSRemoteSpace(NamedTuple):
|
||||
namespace_id: str
|
||||
knowledge_space_id: str
|
||||
provisioning_key: str
|
||||
revision: int
|
||||
|
||||
|
||||
class KnowledgeFSIntegratedProvisionRequest(NamedTuple):
|
||||
namespace_id: str
|
||||
control_space_id: str
|
||||
operation_id: str
|
||||
idempotency_key: str
|
||||
provisioning_key: str
|
||||
name: str
|
||||
slug: str
|
||||
icon: str | None
|
||||
description: str | None
|
||||
model_intent: KnowledgeFSModelSelectionIntentPayload
|
||||
profile_intent: KnowledgeFSRetrievalProfileIntentPayload
|
||||
|
||||
|
||||
class KnowledgeFSIntegratedDeletionRequest(NamedTuple):
|
||||
namespace_id: str
|
||||
control_space_id: str
|
||||
operation_id: str
|
||||
idempotency_key: str
|
||||
knowledge_space_id: str | None
|
||||
provisioning_key: str
|
||||
expected_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityGrantRevokeRequest(NamedTuple):
|
||||
namespace_id: str
|
||||
control_space_id: str
|
||||
operation_id: str
|
||||
idempotency_key: str
|
||||
knowledge_space_id: str
|
||||
grant_id: str
|
||||
event_id: str
|
||||
reason_code: str
|
||||
revoke_sequence: int
|
||||
expected_revision: int
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityGrantRevokeAck(NamedTuple):
|
||||
applied: bool
|
||||
highest_revoke_sequence: int
|
||||
state: Literal["active", "revoked"]
|
||||
|
||||
|
||||
class _StrictLifecycleModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
|
||||
class KnowledgeFSDifyIntegrationFreezeRequest(_StrictLifecycleModel):
|
||||
"""Tenant-scoped monotonic freeze command sent before final-delta capture."""
|
||||
|
||||
namespace_id: str = Field(min_length=1, max_length=255)
|
||||
control_space_id: str = Field(min_length=1, max_length=255)
|
||||
freeze_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991)
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991)
|
||||
|
||||
@field_validator("namespace_id", "control_space_id", "freeze_id")
|
||||
@classmethod
|
||||
def validate_normalized_identifier(cls, value: str) -> str:
|
||||
if value.strip() != value:
|
||||
raise ValueError("freeze identifiers must be normalized")
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeFSDifyIntegrationFreezeAck(_StrictLifecycleModel):
|
||||
"""Exact durable KnowledgeFS acknowledgement required before local freeze publication."""
|
||||
|
||||
namespace_id: str = Field(min_length=1, max_length=255)
|
||||
freeze_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991)
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991)
|
||||
frozen_at: datetime
|
||||
updated_at: datetime
|
||||
frozen: Literal[True]
|
||||
applied: bool
|
||||
replayed: bool
|
||||
|
||||
@field_validator("namespace_id", "freeze_id")
|
||||
@classmethod
|
||||
def validate_normalized_identifier(cls, value: str) -> str:
|
||||
if value.strip() != value:
|
||||
raise ValueError("freeze identifiers must be normalized")
|
||||
return value
|
||||
|
||||
@field_validator("frozen_at", "updated_at")
|
||||
@classmethod
|
||||
def validate_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("freeze timestamps must include a timezone")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_outcome(self) -> KnowledgeFSDifyIntegrationFreezeAck:
|
||||
if self.applied is self.replayed:
|
||||
raise ValueError("freeze acknowledgement must be either applied or replayed")
|
||||
return self
|
||||
|
||||
|
||||
class KnowledgeFSDifyIntegrationActivationRequest(_StrictLifecycleModel):
|
||||
"""Tenant-scoped monotonic activation command sent only after local cutover gates pass."""
|
||||
|
||||
namespace_id: str = Field(min_length=1, max_length=255)
|
||||
control_space_id: str = Field(min_length=1, max_length=255)
|
||||
activation_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991)
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
|
||||
@field_validator("namespace_id", "control_space_id", "activation_id")
|
||||
@classmethod
|
||||
def validate_normalized_identifier(cls, value: str) -> str:
|
||||
if value.strip() != value:
|
||||
raise ValueError("activation identifiers must be normalized")
|
||||
return value
|
||||
|
||||
|
||||
class KnowledgeFSDifyIntegrationActivationAck(_StrictLifecycleModel):
|
||||
"""Exact durable KnowledgeFS acknowledgement required before local routes are enabled."""
|
||||
|
||||
namespace_id: str = Field(min_length=1, max_length=255)
|
||||
activation_id: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991)
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$")
|
||||
activated_at: datetime
|
||||
updated_at: datetime
|
||||
active: Literal[True]
|
||||
applied: bool
|
||||
replayed: bool
|
||||
|
||||
@field_validator("namespace_id", "activation_id")
|
||||
@classmethod
|
||||
def validate_normalized_identifier(cls, value: str) -> str:
|
||||
if value.strip() != value:
|
||||
raise ValueError("activation identifiers must be normalized")
|
||||
return value
|
||||
|
||||
@field_validator("activated_at", "updated_at")
|
||||
@classmethod
|
||||
def validate_timezone(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None or value.utcoffset() is None:
|
||||
raise ValueError("activation timestamps must include a timezone")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_outcome(self) -> KnowledgeFSDifyIntegrationActivationAck:
|
||||
if self.applied is self.replayed:
|
||||
raise ValueError("activation acknowledgement must be either applied or replayed")
|
||||
return self
|
||||
|
||||
|
||||
class KnowledgeFSDeletionPhase(StrEnum):
|
||||
ACCEPTED = "accepted"
|
||||
IRREVERSIBLE = "irreversible"
|
||||
COMPLETED = "completed"
|
||||
|
||||
|
||||
class KnowledgeFSDeletionProgress(NamedTuple):
|
||||
phase: KnowledgeFSDeletionPhase
|
||||
revision: int
|
||||
irreversible_at: datetime | None
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleRemotePort(Protocol):
|
||||
"""Explicit hexagonal boundary; tests inject a fake with no HTTP dependency."""
|
||||
|
||||
def provision_integrated_space(self, request: KnowledgeFSIntegratedProvisionRequest) -> KnowledgeFSRemoteSpace: ...
|
||||
|
||||
def request_integrated_deletion(
|
||||
self, request: KnowledgeFSIntegratedDeletionRequest
|
||||
) -> KnowledgeFSDeletionProgress: ...
|
||||
|
||||
def revoke_capability_grant(
|
||||
self, request: KnowledgeFSCapabilityGrantRevokeRequest
|
||||
) -> KnowledgeFSCapabilityGrantRevokeAck: ...
|
||||
|
||||
def activate_dify_workspace_integration(
|
||||
self, request: KnowledgeFSDifyIntegrationActivationRequest
|
||||
) -> KnowledgeFSDifyIntegrationActivationAck: ...
|
||||
|
||||
def freeze_dify_workspace_integration(
|
||||
self, request: KnowledgeFSDifyIntegrationFreezeRequest
|
||||
) -> KnowledgeFSDifyIntegrationFreezeAck: ...
|
||||
|
||||
def find_by_provisioning_key(
|
||||
self,
|
||||
*,
|
||||
provisioning_key: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSRemoteSpace | None: ...
|
||||
|
||||
def list_spaces(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str,
|
||||
control_space_id: str,
|
||||
) -> tuple[KnowledgeFSRemoteSpace, ...]: ...
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleRemoteError(RuntimeError):
|
||||
def __init__(self, code: str, message: str):
|
||||
self.code = code
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSCapabilityGrantRevokeAck",
|
||||
"KnowledgeFSCapabilityGrantRevokeRequest",
|
||||
"KnowledgeFSDeletionPhase",
|
||||
"KnowledgeFSDeletionProgress",
|
||||
"KnowledgeFSDifyIntegrationActivationAck",
|
||||
"KnowledgeFSDifyIntegrationActivationRequest",
|
||||
"KnowledgeFSDifyIntegrationFreezeAck",
|
||||
"KnowledgeFSDifyIntegrationFreezeRequest",
|
||||
"KnowledgeFSIntegratedDeletionRequest",
|
||||
"KnowledgeFSIntegratedProvisionRequest",
|
||||
"KnowledgeFSLifecycleRemoteError",
|
||||
"KnowledgeFSLifecycleRemotePort",
|
||||
"KnowledgeFSRemoteSpace",
|
||||
]
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Fail-closed rollout gates for the KnowledgeFS lifecycle worker."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleWorkerReadiness(NamedTuple):
|
||||
ready: bool
|
||||
blockers: tuple[str, ...]
|
||||
|
||||
|
||||
def evaluate_knowledge_fs_lifecycle_worker_readiness(
|
||||
*,
|
||||
worker_enabled: bool,
|
||||
capability_v2_enabled: bool,
|
||||
integrated_provision_ready: bool,
|
||||
legacy_acl_freeze_ready: bool,
|
||||
) -> KnowledgeFSLifecycleWorkerReadiness:
|
||||
"""Require every security dependency without coupling cleanup to the product flag."""
|
||||
|
||||
gates = (
|
||||
("worker", worker_enabled),
|
||||
("capability_v2", capability_v2_enabled),
|
||||
("integrated_provision", integrated_provision_ready),
|
||||
("legacy_acl_freeze", legacy_acl_freeze_ready),
|
||||
)
|
||||
blockers = tuple(name for name, enabled in gates if not enabled)
|
||||
return KnowledgeFSLifecycleWorkerReadiness(ready=not blockers, blockers=blockers)
|
||||
|
||||
|
||||
def get_configured_knowledge_fs_lifecycle_worker_readiness() -> KnowledgeFSLifecycleWorkerReadiness:
|
||||
from configs import dify_config
|
||||
|
||||
return evaluate_knowledge_fs_lifecycle_worker_readiness(
|
||||
worker_enabled=dify_config.KNOWLEDGE_FS_LIFECYCLE_WORKER_ENABLED,
|
||||
capability_v2_enabled=dify_config.KNOWLEDGE_FS_CAPABILITY_V2_ENABLED,
|
||||
integrated_provision_ready=dify_config.KNOWLEDGE_FS_INTEGRATED_PROVISION_READY,
|
||||
legacy_acl_freeze_ready=dify_config.KNOWLEDGE_FS_LEGACY_ACL_FREEZE_READY,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSLifecycleWorkerReadiness",
|
||||
"evaluate_knowledge_fs_lifecycle_worker_readiness",
|
||||
"get_configured_knowledge_fs_lifecycle_worker_readiness",
|
||||
]
|
||||
@@ -0,0 +1,578 @@
|
||||
"""Capability-v2-only HTTP adapter for the KnowledgeFS lifecycle data plane."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from http import HTTPStatus
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from core.helper import ssrf_proxy
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from services.knowledge_fs.lifecycle_port import (
|
||||
KnowledgeFSCapabilityGrantRevokeAck,
|
||||
KnowledgeFSCapabilityGrantRevokeRequest,
|
||||
KnowledgeFSDeletionPhase,
|
||||
KnowledgeFSDeletionProgress,
|
||||
KnowledgeFSDifyIntegrationActivationAck,
|
||||
KnowledgeFSDifyIntegrationActivationRequest,
|
||||
KnowledgeFSDifyIntegrationFreezeAck,
|
||||
KnowledgeFSDifyIntegrationFreezeRequest,
|
||||
KnowledgeFSIntegratedDeletionRequest,
|
||||
KnowledgeFSIntegratedProvisionRequest,
|
||||
KnowledgeFSLifecycleRemoteError,
|
||||
KnowledgeFSRemoteSpace,
|
||||
)
|
||||
from services.knowledge_fs_capability import (
|
||||
CapabilityAuthzRevision,
|
||||
CapabilityIssueRequest,
|
||||
CapabilityResource,
|
||||
KnowledgeFSCapabilityIssuer,
|
||||
)
|
||||
|
||||
_JSON_ADAPTER: TypeAdapter[JsonValue] = TypeAdapter(JsonValue)
|
||||
_MAX_RESPONSE_BYTES = 1024 * 1024
|
||||
_MAX_RECONCILIATION_SPACES = 10_000
|
||||
_PAGE_SIZE = 100
|
||||
_WORKER_PRINCIPAL = "knowledge-fs-lifecycle"
|
||||
|
||||
|
||||
class _KnowledgeSpaceResponse(BaseModel):
|
||||
id: str
|
||||
revision: int = Field(ge=1)
|
||||
slug: str = Field(min_length=1, max_length=160)
|
||||
tenant_id: str = Field(min_length=1, max_length=255, alias="tenantId")
|
||||
|
||||
model_config = ConfigDict(extra="ignore", populate_by_name=True)
|
||||
|
||||
|
||||
class _KnowledgeSpaceListResponse(BaseModel):
|
||||
items: list[_KnowledgeSpaceResponse]
|
||||
next_cursor: str | None = Field(default=None, alias="nextCursor")
|
||||
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
|
||||
class _DeletionProgressResponse(BaseModel):
|
||||
phase: Literal["accepted", "irreversible", "completed"]
|
||||
revision: int = Field(ge=1)
|
||||
irreversible_at: datetime | None = Field(default=None, alias="irreversibleAt")
|
||||
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
|
||||
class _CapabilityGrantRevokeResponse(BaseModel):
|
||||
applied: bool
|
||||
highest_revoke_sequence: int = Field(ge=0, alias="highestRevokeSequence")
|
||||
state: Literal["active", "revoked"]
|
||||
|
||||
model_config = ConfigDict(extra="forbid", populate_by_name=True)
|
||||
|
||||
|
||||
class _DifyIntegrationActivationBody(BaseModel):
|
||||
activation_id: str = Field(alias="activationId")
|
||||
activation_revision: int = Field(ge=1, le=9_007_199_254_740_991, alias="activationRevision")
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$", alias="sourceRevisionDigest")
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True)
|
||||
|
||||
|
||||
class _DifyIntegrationActivationResponse(_DifyIntegrationActivationBody):
|
||||
namespace_id: str = Field(min_length=1, max_length=255, alias="namespaceId")
|
||||
activated_at: datetime = Field(alias="activatedAt")
|
||||
updated_at: datetime = Field(alias="updatedAt")
|
||||
active: Literal[True]
|
||||
applied: bool
|
||||
replayed: bool
|
||||
|
||||
|
||||
class _DifyIntegrationFreezeBody(BaseModel):
|
||||
freeze_id: str = Field(alias="freezeId")
|
||||
freeze_revision: int = Field(ge=1, le=9_007_199_254_740_991, alias="freezeRevision")
|
||||
source_revision_digest: str = Field(pattern=r"^sha256:[a-f0-9]{64}$", alias="sourceRevisionDigest")
|
||||
source_task_watermark: int = Field(ge=0, le=9_007_199_254_740_991, alias="sourceTaskWatermark")
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, populate_by_name=True)
|
||||
|
||||
|
||||
class _DifyIntegrationFreezeResponse(_DifyIntegrationFreezeBody):
|
||||
namespace_id: str = Field(min_length=1, max_length=255, alias="namespaceId")
|
||||
frozen_at: datetime = Field(alias="frozenAt")
|
||||
updated_at: datetime = Field(alias="updatedAt")
|
||||
frozen: Literal[True]
|
||||
applied: bool
|
||||
replayed: bool
|
||||
|
||||
|
||||
class HTTPKnowledgeFSLifecycleRemoteClient:
|
||||
"""Deliver durable lifecycle commands with freshly issued internal-worker capabilities."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_url: str,
|
||||
issuer: KnowledgeFSCapabilityIssuer,
|
||||
timeout_seconds: float,
|
||||
max_response_bytes: int = _MAX_RESPONSE_BYTES,
|
||||
) -> None:
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._issuer = issuer
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._max_response_bytes = max_response_bytes
|
||||
|
||||
def provision_integrated_space(self, request: KnowledgeFSIntegratedProvisionRequest) -> KnowledgeFSRemoteSpace:
|
||||
token = self._issue_capability(
|
||||
namespace_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
operation_id="provisionIntegratedKnowledgeSpace",
|
||||
operation_identity=request.operation_id,
|
||||
resource=CapabilityResource(type="namespace", id=request.namespace_id),
|
||||
)
|
||||
payload: dict[str, JsonValue] = {
|
||||
"embeddingProfile": _json_object(request.model_intent, "embedding profile"),
|
||||
"idempotencyKey": request.provisioning_key,
|
||||
"name": request.name,
|
||||
"retrievalProfile": _json_object(request.profile_intent, "retrieval profile"),
|
||||
"slug": request.slug,
|
||||
}
|
||||
if request.icon is not None:
|
||||
payload["iconRef"] = request.icon
|
||||
if request.description is not None:
|
||||
payload["description"] = request.description
|
||||
response = self._request_json(
|
||||
method="POST",
|
||||
path="/internal/knowledge-spaces/provision",
|
||||
token=token,
|
||||
trace_id=request.operation_id,
|
||||
payload=payload,
|
||||
expected_statuses=(HTTPStatus.CREATED,),
|
||||
)
|
||||
try:
|
||||
space = _KnowledgeSpaceResponse.model_validate(response)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid provision response"
|
||||
) from exc
|
||||
if space.tenant_id != request.namespace_id:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_SCOPE_MISMATCH", "KnowledgeFS provision response crossed its namespace"
|
||||
)
|
||||
return _remote_space(space, provisioning_key=request.provisioning_key)
|
||||
|
||||
def request_integrated_deletion(self, request: KnowledgeFSIntegratedDeletionRequest) -> KnowledgeFSDeletionProgress:
|
||||
knowledge_space_id = request.knowledge_space_id
|
||||
expected_revision = request.expected_revision
|
||||
if knowledge_space_id is None:
|
||||
recovered = self.find_by_provisioning_key(
|
||||
provisioning_key=request.provisioning_key,
|
||||
control_space_id=request.control_space_id,
|
||||
)
|
||||
if recovered is None:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_SPACE_NOT_FOUND",
|
||||
"KnowledgeFS deletion identity could not be recovered",
|
||||
)
|
||||
knowledge_space_id = recovered.knowledge_space_id
|
||||
expected_revision = recovered.revision
|
||||
token = self._issue_capability(
|
||||
namespace_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
operation_id="deleteIntegratedKnowledgeSpace",
|
||||
operation_identity=request.operation_id,
|
||||
resource=CapabilityResource(type="knowledge_space", id=knowledge_space_id),
|
||||
)
|
||||
response = self._request_json(
|
||||
method="POST",
|
||||
path=f"/internal/knowledge-spaces/{knowledge_space_id}/delete",
|
||||
token=token,
|
||||
trace_id=request.operation_id,
|
||||
payload={
|
||||
"controlSpaceId": request.control_space_id,
|
||||
"expectedRevision": expected_revision,
|
||||
"idempotencyKey": request.idempotency_key,
|
||||
"operationId": request.operation_id,
|
||||
"provisioningKey": request.provisioning_key,
|
||||
},
|
||||
expected_statuses=(HTTPStatus.OK, HTTPStatus.ACCEPTED),
|
||||
)
|
||||
try:
|
||||
progress = _DeletionProgressResponse.model_validate(response)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid deletion response"
|
||||
) from exc
|
||||
irreversible_at = progress.irreversible_at
|
||||
if irreversible_at is not None:
|
||||
if irreversible_at.tzinfo is None or irreversible_at.utcoffset() is None:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID",
|
||||
"KnowledgeFS deletion timestamp must include a timezone",
|
||||
)
|
||||
irreversible_at = irreversible_at.astimezone(UTC).replace(tzinfo=None)
|
||||
return KnowledgeFSDeletionProgress(
|
||||
phase=KnowledgeFSDeletionPhase(progress.phase),
|
||||
revision=progress.revision,
|
||||
irreversible_at=irreversible_at,
|
||||
)
|
||||
|
||||
def revoke_capability_grant(
|
||||
self, request: KnowledgeFSCapabilityGrantRevokeRequest
|
||||
) -> KnowledgeFSCapabilityGrantRevokeAck:
|
||||
token = self._issue_capability(
|
||||
namespace_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
operation_id="revokeCapabilityGrant",
|
||||
operation_identity=request.operation_id,
|
||||
resource=CapabilityResource(type="knowledge_space", id=request.knowledge_space_id),
|
||||
)
|
||||
response = self._request_json(
|
||||
method="POST",
|
||||
path=f"/internal/capability-grants/{request.grant_id}/revoke",
|
||||
token=token,
|
||||
trace_id=request.operation_id,
|
||||
payload={
|
||||
"eventId": request.event_id,
|
||||
"knowledgeSpaceId": request.knowledge_space_id,
|
||||
"reasonCode": request.reason_code,
|
||||
"revokeSequence": request.revoke_sequence,
|
||||
},
|
||||
expected_statuses=(HTTPStatus.OK,),
|
||||
)
|
||||
try:
|
||||
ack = _CapabilityGrantRevokeResponse.model_validate(response)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid capability revoke response"
|
||||
) from exc
|
||||
return KnowledgeFSCapabilityGrantRevokeAck(
|
||||
applied=ack.applied,
|
||||
highest_revoke_sequence=ack.highest_revoke_sequence,
|
||||
state=ack.state,
|
||||
)
|
||||
|
||||
def activate_dify_workspace_integration(
|
||||
self, request: KnowledgeFSDifyIntegrationActivationRequest
|
||||
) -> KnowledgeFSDifyIntegrationActivationAck:
|
||||
"""Persist one monotonic namespace activation using a stable grant and trace identity."""
|
||||
|
||||
token = self._issue_capability(
|
||||
namespace_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
operation_id="activateDifyWorkspaceIntegration",
|
||||
operation_identity=request.activation_id,
|
||||
resource=CapabilityResource(type="namespace", id=request.namespace_id),
|
||||
)
|
||||
body = _DifyIntegrationActivationBody(
|
||||
activationId=request.activation_id,
|
||||
activationRevision=request.activation_revision,
|
||||
sourceRevisionDigest=request.source_revision_digest,
|
||||
)
|
||||
response = self._request_json(
|
||||
method="POST",
|
||||
path="/internal/dify-integration/activate",
|
||||
token=token,
|
||||
trace_id=request.activation_id,
|
||||
payload=_JSON_ADAPTER.validate_python(body.model_dump(mode="json", by_alias=True)),
|
||||
expected_statuses=(HTTPStatus.OK,),
|
||||
)
|
||||
try:
|
||||
parsed = _DifyIntegrationActivationResponse.model_validate(response)
|
||||
ack = KnowledgeFSDifyIntegrationActivationAck(
|
||||
namespace_id=parsed.namespace_id,
|
||||
activation_id=parsed.activation_id,
|
||||
activation_revision=parsed.activation_revision,
|
||||
source_revision_digest=parsed.source_revision_digest,
|
||||
activated_at=parsed.activated_at,
|
||||
updated_at=parsed.updated_at,
|
||||
active=parsed.active,
|
||||
applied=parsed.applied,
|
||||
replayed=parsed.replayed,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID",
|
||||
"KnowledgeFS returned an invalid integration activation acknowledgement",
|
||||
) from exc
|
||||
if (
|
||||
ack.namespace_id != request.namespace_id
|
||||
or ack.activation_id != request.activation_id
|
||||
or ack.activation_revision != request.activation_revision
|
||||
or ack.source_revision_digest != request.source_revision_digest
|
||||
):
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_ACTIVATION_ACK_MISMATCH",
|
||||
"KnowledgeFS integration activation acknowledgement did not match the request",
|
||||
)
|
||||
return ack
|
||||
|
||||
def freeze_dify_workspace_integration(
|
||||
self, request: KnowledgeFSDifyIntegrationFreezeRequest
|
||||
) -> KnowledgeFSDifyIntegrationFreezeAck:
|
||||
"""Persist the remote maintenance freeze before Dify captures the final delta."""
|
||||
|
||||
token = self._issue_capability(
|
||||
namespace_id=request.namespace_id,
|
||||
control_space_id=request.control_space_id,
|
||||
operation_id="freezeDifyWorkspaceIntegration",
|
||||
operation_identity=request.freeze_id,
|
||||
resource=CapabilityResource(type="namespace", id=request.namespace_id),
|
||||
)
|
||||
body = _DifyIntegrationFreezeBody(
|
||||
freezeId=request.freeze_id,
|
||||
freezeRevision=request.freeze_revision,
|
||||
sourceRevisionDigest=request.source_revision_digest,
|
||||
sourceTaskWatermark=request.source_task_watermark,
|
||||
)
|
||||
response = self._request_json(
|
||||
method="POST",
|
||||
path="/internal/dify-integration/freeze",
|
||||
token=token,
|
||||
trace_id=request.freeze_id,
|
||||
payload=_JSON_ADAPTER.validate_python(body.model_dump(mode="json", by_alias=True)),
|
||||
expected_statuses=(HTTPStatus.OK,),
|
||||
)
|
||||
try:
|
||||
parsed = _DifyIntegrationFreezeResponse.model_validate(response)
|
||||
ack = KnowledgeFSDifyIntegrationFreezeAck(
|
||||
namespace_id=parsed.namespace_id,
|
||||
freeze_id=parsed.freeze_id,
|
||||
freeze_revision=parsed.freeze_revision,
|
||||
source_revision_digest=parsed.source_revision_digest,
|
||||
source_task_watermark=parsed.source_task_watermark,
|
||||
frozen_at=parsed.frozen_at,
|
||||
updated_at=parsed.updated_at,
|
||||
frozen=parsed.frozen,
|
||||
applied=parsed.applied,
|
||||
replayed=parsed.replayed,
|
||||
)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID",
|
||||
"KnowledgeFS returned an invalid integration freeze acknowledgement",
|
||||
) from exc
|
||||
if (
|
||||
ack.namespace_id != request.namespace_id
|
||||
or ack.freeze_id != request.freeze_id
|
||||
or ack.freeze_revision != request.freeze_revision
|
||||
or ack.source_revision_digest != request.source_revision_digest
|
||||
or ack.source_task_watermark != request.source_task_watermark
|
||||
):
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_FREEZE_ACK_MISMATCH",
|
||||
"KnowledgeFS integration freeze acknowledgement did not match the request",
|
||||
)
|
||||
return ack
|
||||
|
||||
def find_by_provisioning_key(
|
||||
self,
|
||||
*,
|
||||
provisioning_key: str,
|
||||
control_space_id: str,
|
||||
) -> KnowledgeFSRemoteSpace | None:
|
||||
namespace_id = _namespace_from_provisioning_key(provisioning_key)
|
||||
return next(
|
||||
(
|
||||
space
|
||||
for space in self.list_spaces(
|
||||
namespace_id=namespace_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
if space.provisioning_key == provisioning_key
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def list_spaces(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str | None = None,
|
||||
control_space_id: str | None = None,
|
||||
) -> tuple[KnowledgeFSRemoteSpace, ...]:
|
||||
if namespace_id is None or not namespace_id.strip():
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_NAMESPACE_REQUIRED",
|
||||
"KnowledgeFS reconciliation requires an explicit namespace",
|
||||
)
|
||||
if control_space_id is None or not control_space_id.strip():
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_CONTROL_SPACE_REQUIRED",
|
||||
"KnowledgeFS reconciliation requires an auditable control-space",
|
||||
)
|
||||
namespace_id = namespace_id.strip()
|
||||
token = self._issue_capability(
|
||||
namespace_id=namespace_id,
|
||||
control_space_id=control_space_id.strip(),
|
||||
operation_id="listKnowledgeSpaces",
|
||||
operation_identity=f"reconcile:{namespace_id}",
|
||||
resource=CapabilityResource(type="namespace", id=namespace_id),
|
||||
)
|
||||
spaces: list[KnowledgeFSRemoteSpace] = []
|
||||
cursor: str | None = None
|
||||
consumed_cursors: set[str] = set()
|
||||
while True:
|
||||
query = (
|
||||
(("limit", str(_PAGE_SIZE)),) if cursor is None else (("limit", str(_PAGE_SIZE)), ("cursor", cursor))
|
||||
)
|
||||
response = self._request_json(
|
||||
method="GET",
|
||||
path="/knowledge-spaces",
|
||||
token=token,
|
||||
trace_id=f"reconcile:{namespace_id}",
|
||||
query=query,
|
||||
payload=None,
|
||||
expected_statuses=(HTTPStatus.OK,),
|
||||
)
|
||||
try:
|
||||
page = _KnowledgeSpaceListResponse.model_validate(response)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned an invalid Space list"
|
||||
) from exc
|
||||
for item in page.items:
|
||||
if item.tenant_id != namespace_id:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_SCOPE_MISMATCH", "KnowledgeFS list response crossed its namespace"
|
||||
)
|
||||
spaces.append(_remote_space(item, provisioning_key=f"dify:{namespace_id}:{item.slug}"))
|
||||
if len(spaces) > _MAX_RECONCILIATION_SPACES:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RECONCILIATION_LIMIT",
|
||||
"KnowledgeFS reconciliation exceeded its bounded Space limit",
|
||||
)
|
||||
cursor = page.next_cursor
|
||||
if cursor is None:
|
||||
break
|
||||
if cursor in consumed_cursors:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_CURSOR_LOOP", "KnowledgeFS repeated a reconciliation cursor"
|
||||
)
|
||||
consumed_cursors.add(cursor)
|
||||
return tuple(spaces)
|
||||
|
||||
def _issue_capability(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
operation_identity: str,
|
||||
resource: CapabilityResource,
|
||||
) -> str:
|
||||
issued = self._issuer.issue(
|
||||
CapabilityIssueRequest(
|
||||
actor=f"dify-worker:{_WORKER_PRINCIPAL}",
|
||||
authz_revision=CapabilityAuthzRevision(
|
||||
credential_revision=None,
|
||||
external_access_epoch=0,
|
||||
membership_epoch=0,
|
||||
space_acl_epoch=0,
|
||||
),
|
||||
caller_kind="internal_worker",
|
||||
content_policy_revision=0,
|
||||
control_space_id=control_space_id,
|
||||
grant_id=operation_identity,
|
||||
namespace_id=namespace_id,
|
||||
operation_id=operation_id,
|
||||
principal_id=_WORKER_PRINCIPAL,
|
||||
resource=resource,
|
||||
trace_id=operation_identity,
|
||||
)
|
||||
)
|
||||
return issued.token
|
||||
|
||||
def _request_json(
|
||||
self,
|
||||
*,
|
||||
method: Literal["GET", "POST"],
|
||||
path: str,
|
||||
token: str,
|
||||
trace_id: str,
|
||||
payload: JsonValue | None,
|
||||
expected_statuses: tuple[HTTPStatus, ...],
|
||||
query: tuple[tuple[str, str], ...] = (),
|
||||
) -> JsonValue:
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Encoding": "identity",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-Trace-Id": trace_id,
|
||||
}
|
||||
request_kwargs: dict[str, object] = {
|
||||
"follow_redirects": False,
|
||||
"headers": headers,
|
||||
"params": query,
|
||||
"timeout": self._timeout_seconds,
|
||||
}
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
request_kwargs["json"] = payload
|
||||
try:
|
||||
upstream_url = httpx.URL(f"{self._base_url}/").join(path.lstrip("/"))
|
||||
response = ssrf_proxy.make_request(
|
||||
method=method,
|
||||
url=str(upstream_url),
|
||||
max_retries=0,
|
||||
stream_response=True,
|
||||
**request_kwargs,
|
||||
)
|
||||
response = ssrf_proxy.buffer_response(response, max_response_bytes=self._max_response_bytes)
|
||||
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_REQUEST_FAILED", "KnowledgeFS lifecycle request failed"
|
||||
) from exc
|
||||
try:
|
||||
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
|
||||
if content_type != "application/json" and not content_type.endswith("+json"):
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_MEDIA_TYPE_INVALID",
|
||||
"KnowledgeFS returned an unsupported lifecycle media type",
|
||||
)
|
||||
if response.status_code not in {int(status) for status in expected_statuses}:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
f"KNOWLEDGE_FS_HTTP_{response.status_code}",
|
||||
f"KnowledgeFS lifecycle request returned HTTP {response.status_code}",
|
||||
)
|
||||
try:
|
||||
return _JSON_ADAPTER.validate_python(response.json())
|
||||
except (ValueError, ValidationError) as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_RESPONSE_INVALID", "KnowledgeFS returned invalid lifecycle JSON"
|
||||
) from exc
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def _json_object(value: object, label: str) -> dict[str, JsonValue]:
|
||||
try:
|
||||
parsed = _JSON_ADAPTER.validate_python(value)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_REQUEST_INVALID", f"KnowledgeFS {label} is not JSON-compatible"
|
||||
) from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise KnowledgeFSLifecycleRemoteError("KNOWLEDGE_FS_REQUEST_INVALID", f"KnowledgeFS {label} must be an object")
|
||||
return parsed
|
||||
|
||||
|
||||
def _namespace_from_provisioning_key(provisioning_key: str) -> str:
|
||||
parts = provisioning_key.split(":", 2)
|
||||
if len(parts) != 3 or parts[0] != "dify" or not parts[1] or not parts[2]:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_PROVISIONING_KEY_INVALID",
|
||||
"KnowledgeFS provisioning key does not identify a namespace",
|
||||
)
|
||||
return parts[1]
|
||||
|
||||
|
||||
def _remote_space(space: _KnowledgeSpaceResponse, *, provisioning_key: str) -> KnowledgeFSRemoteSpace:
|
||||
return KnowledgeFSRemoteSpace(
|
||||
namespace_id=space.tenant_id,
|
||||
knowledge_space_id=space.id,
|
||||
provisioning_key=provisioning_key,
|
||||
revision=space.revision,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["HTTPKnowledgeFSLifecycleRemoteClient"]
|
||||
@@ -0,0 +1,627 @@
|
||||
"""Lease-driven KnowledgeFS integrated provision and durable deletion saga."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Literal, NamedTuple, cast
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSDeleteCommandPayload,
|
||||
KnowledgeFSLifecycleOperation,
|
||||
KnowledgeFSLifecycleOutbox,
|
||||
KnowledgeFSLifecycleOutboxStatus,
|
||||
KnowledgeFSProvisionCommandPayload,
|
||||
KnowledgeFSRevokeCommandPayload,
|
||||
)
|
||||
from repositories.knowledge_fs_control_space_repository import KnowledgeFSControlSpaceCASUpdate
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
from repositories.sqlalchemy_knowledge_fs_lifecycle_outbox_repository import (
|
||||
SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
|
||||
)
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
|
||||
from services.knowledge_fs.lifecycle_port import (
|
||||
KnowledgeFSCapabilityGrantRevokeAck,
|
||||
KnowledgeFSCapabilityGrantRevokeRequest,
|
||||
KnowledgeFSDeletionPhase,
|
||||
KnowledgeFSIntegratedDeletionRequest,
|
||||
KnowledgeFSIntegratedProvisionRequest,
|
||||
KnowledgeFSLifecycleRemoteError,
|
||||
KnowledgeFSLifecycleRemotePort,
|
||||
KnowledgeFSRemoteSpace,
|
||||
)
|
||||
from services.knowledge_fs.observability import (
|
||||
KnowledgeFSLifecycleTaskMetric,
|
||||
KnowledgeFSOperationalMetricsPort,
|
||||
get_knowledge_fs_operational_metrics,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleDispatchResult(NamedTuple):
|
||||
claimed: bool
|
||||
completed: bool
|
||||
outbox_id: str | None
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleLeaseLostError(RuntimeError):
|
||||
"""The command lease changed before the worker could settle its result."""
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleSagaRunner:
|
||||
"""Claim, deliver, and settle one command without holding a DB transaction over HTTP."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_maker: sessionmaker[Session],
|
||||
remote: KnowledgeFSLifecycleRemotePort,
|
||||
metrics: KnowledgeFSOperationalMetricsPort | None = None,
|
||||
):
|
||||
self._session_maker = session_maker
|
||||
self._remote = remote
|
||||
self._metrics = metrics or get_knowledge_fs_operational_metrics()
|
||||
|
||||
def dispatch_one(
|
||||
self,
|
||||
*,
|
||||
worker_id: str,
|
||||
now: datetime,
|
||||
lease_duration: timedelta,
|
||||
product_enabled: bool,
|
||||
) -> KnowledgeFSLifecycleDispatchResult:
|
||||
allowed_operations = tuple(KnowledgeFSLifecycleOperation)
|
||||
if not product_enabled:
|
||||
allowed_operations = tuple(
|
||||
operation
|
||||
for operation in allowed_operations
|
||||
if operation is not KnowledgeFSLifecycleOperation.PROVISION
|
||||
)
|
||||
with self._session_maker.begin() as session:
|
||||
command = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session).claim_next(
|
||||
lease_owner=worker_id,
|
||||
now=now,
|
||||
lease_duration=lease_duration,
|
||||
allowed_operations=allowed_operations,
|
||||
)
|
||||
if command is None:
|
||||
return KnowledgeFSLifecycleDispatchResult(False, False, None)
|
||||
claimed = _ClaimedCommand.from_model(command)
|
||||
|
||||
self._record_task(claimed, status="running")
|
||||
try:
|
||||
return self._dispatch_claimed(claimed=claimed, now=now, product_enabled=product_enabled)
|
||||
except Exception:
|
||||
self._record_task(claimed, status="dispatch_error")
|
||||
raise
|
||||
|
||||
def _dispatch_claimed(
|
||||
self,
|
||||
*,
|
||||
claimed: _ClaimedCommand,
|
||||
now: datetime,
|
||||
product_enabled: bool,
|
||||
) -> KnowledgeFSLifecycleDispatchResult:
|
||||
try:
|
||||
if claimed.operation is KnowledgeFSLifecycleOperation.PROVISION:
|
||||
if self._settle_provision_without_delivery(claimed=claimed, completed_at=now):
|
||||
return self._result(claimed, completed=True, settled_at=now)
|
||||
remote_space = self._remote.provision_integrated_space(_provision_request(claimed))
|
||||
self._complete_provision(claimed=claimed, remote_space=remote_space, completed_at=now)
|
||||
return self._result(claimed, completed=True, settled_at=now)
|
||||
if claimed.operation is KnowledgeFSLifecycleOperation.DELETE:
|
||||
if self._already_deleted(claimed=claimed, completed_at=now):
|
||||
return self._result(claimed, completed=True, settled_at=now)
|
||||
if not product_enabled and self._complete_never_provisioned_deletion(
|
||||
claimed=claimed,
|
||||
completed_at=now,
|
||||
):
|
||||
return self._result(claimed, completed=True, settled_at=now)
|
||||
request = self._prepare_deletion_request(claimed)
|
||||
if request is None:
|
||||
if not product_enabled and self._complete_remote_absent_deletion(
|
||||
claimed=claimed,
|
||||
observed_at=now,
|
||||
):
|
||||
return self._result(claimed, completed=True, settled_at=now)
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_SPACE_NOT_FOUND",
|
||||
"KnowledgeFS deletion identity could not be recovered",
|
||||
)
|
||||
progress = self._remote.request_integrated_deletion(request)
|
||||
completed = self._settle_deletion(claimed=claimed, progress=progress, settled_at=now)
|
||||
return self._result(claimed, completed=completed, settled_at=now)
|
||||
if claimed.operation is KnowledgeFSLifecycleOperation.REVOKE:
|
||||
acknowledgment = self._remote.revoke_capability_grant(_revoke_request(claimed))
|
||||
completed = self._settle_revoke(
|
||||
claimed=claimed,
|
||||
acknowledgment=acknowledgment,
|
||||
settled_at=now,
|
||||
)
|
||||
return self._result(claimed, completed=completed, settled_at=now)
|
||||
self._retry(
|
||||
claimed=claimed,
|
||||
now=now,
|
||||
error_code="UNSUPPORTED_OPERATION",
|
||||
error_message=f"Lifecycle operation {claimed.operation} is not dispatched yet",
|
||||
)
|
||||
except KnowledgeFSLifecycleRemoteError as exc:
|
||||
self._retry(claimed=claimed, now=now, error_code=exc.code, error_message=str(exc))
|
||||
return self._result(claimed, completed=False, settled_at=now)
|
||||
|
||||
def _result(
|
||||
self,
|
||||
claimed: _ClaimedCommand,
|
||||
*,
|
||||
completed: bool,
|
||||
settled_at: datetime,
|
||||
) -> KnowledgeFSLifecycleDispatchResult:
|
||||
self._record_task(
|
||||
claimed,
|
||||
status="succeeded" if completed else "retry",
|
||||
settled_at=settled_at if completed else None,
|
||||
)
|
||||
return KnowledgeFSLifecycleDispatchResult(True, completed, claimed.outbox_id)
|
||||
|
||||
def _record_task(
|
||||
self,
|
||||
claimed: _ClaimedCommand,
|
||||
*,
|
||||
status: Literal["dispatch_error", "retry", "running", "succeeded"],
|
||||
settled_at: datetime | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
duration_seconds = None
|
||||
if settled_at is not None:
|
||||
duration_seconds = max(0.0, (settled_at - claimed.created_at).total_seconds())
|
||||
self._metrics.record_lifecycle_task(
|
||||
KnowledgeFSLifecycleTaskMetric(duration_seconds, claimed.operation.value, status)
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"KnowledgeFS lifecycle metric export failed operation=%s status=%s",
|
||||
claimed.operation.value,
|
||||
status,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _settle_provision_without_delivery(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
|
||||
"""ACK a provision already applied or superseded before crossing the HTTP boundary."""
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
still_owned = (
|
||||
control_space is not None
|
||||
and control_space.lifecycle_operation_id == claimed.operation_id
|
||||
and control_space.state is KnowledgeFSControlSpaceState.PROVISIONING
|
||||
)
|
||||
if still_owned:
|
||||
return False
|
||||
self._ack(
|
||||
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
|
||||
claimed=claimed,
|
||||
completed_at=completed_at,
|
||||
)
|
||||
return True
|
||||
|
||||
def _complete_never_provisioned_deletion(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
|
||||
"""Terminally delete a local intent proven never to have crossed the remote boundary."""
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
control_space = control_repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if (
|
||||
control_space is None
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
|
||||
or control_space.lifecycle_operation_id != claimed.operation_id
|
||||
or control_space.knowledge_space_id is not None
|
||||
):
|
||||
return False
|
||||
provision = outbox_repository.find_open_for_control_space(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
operation=KnowledgeFSLifecycleOperation.PROVISION,
|
||||
)
|
||||
if (
|
||||
provision is None
|
||||
or provision.status is not KnowledgeFSLifecycleOutboxStatus.PENDING
|
||||
or provision.attempt_count != 0
|
||||
):
|
||||
return False
|
||||
if not outbox_repository.supersede_unattempted(
|
||||
outbox_id=provision.id,
|
||||
completed_at=completed_at,
|
||||
error_code="SUPERSEDED_BY_DELETE",
|
||||
error_message="Provision was canceled before delivery by permanent cleanup",
|
||||
):
|
||||
return False
|
||||
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETED,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
)
|
||||
self._ack(outbox_repository, claimed=claimed, completed_at=completed_at)
|
||||
return True
|
||||
|
||||
def _complete_remote_absent_deletion(self, *, claimed: _ClaimedCommand, observed_at: datetime) -> bool:
|
||||
"""Finish cleanup after an authoritative remote listing proves the target absent."""
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
control_space = control_repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if (
|
||||
control_space is None
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
|
||||
or control_space.lifecycle_operation_id != claimed.operation_id
|
||||
or control_space.knowledge_space_id is not None
|
||||
):
|
||||
return False
|
||||
provision = outbox_repository.find_latest_for_control_space(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
operation=KnowledgeFSLifecycleOperation.PROVISION,
|
||||
)
|
||||
if provision is None or not outbox_repository.supersede_after_remote_absence(
|
||||
outbox_id=provision.id,
|
||||
observed_at=observed_at,
|
||||
error_code="SUPERSEDED_BY_DELETE",
|
||||
error_message="Remote absence was confirmed by permanent cleanup",
|
||||
):
|
||||
return False
|
||||
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETED,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
)
|
||||
self._ack(outbox_repository, claimed=claimed, completed_at=observed_at)
|
||||
return True
|
||||
|
||||
def _prepare_deletion_request(self, claimed: _ClaimedCommand) -> KnowledgeFSIntegratedDeletionRequest | None:
|
||||
"""Resolve and durably register a lost-ACK remote identity before deletion admission."""
|
||||
|
||||
with self._session_maker() as session:
|
||||
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if (
|
||||
control_space is None
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
|
||||
or control_space.lifecycle_operation_id != claimed.operation_id
|
||||
):
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Delete command no longer owns the control-space")
|
||||
if control_space.knowledge_space_id is not None:
|
||||
return _resolved_deletion_request(
|
||||
claimed,
|
||||
knowledge_space_id=control_space.knowledge_space_id,
|
||||
expected_revision=control_space.knowledge_space_revision,
|
||||
)
|
||||
|
||||
payload = cast(KnowledgeFSDeleteCommandPayload, claimed.command_payload)
|
||||
recovered = self._remote.find_by_provisioning_key(
|
||||
provisioning_key=payload["provisioning_key"],
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if recovered is None:
|
||||
return None
|
||||
if recovered.namespace_id != claimed.tenant_id:
|
||||
raise KnowledgeFSLifecycleRemoteError(
|
||||
"KNOWLEDGE_FS_SCOPE_MISMATCH",
|
||||
"KnowledgeFS recovered deletion identity crossed its namespace",
|
||||
)
|
||||
|
||||
with self._session_maker.begin() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
control_space = repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if (
|
||||
control_space is None
|
||||
or control_space.state is not KnowledgeFSControlSpaceState.DELETING
|
||||
or control_space.lifecycle_operation_id != claimed.operation_id
|
||||
):
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Delete command lost ownership during identity recovery")
|
||||
if control_space.knowledge_space_id is None:
|
||||
changed = repository.compare_and_set_lifecycle(
|
||||
KnowledgeFSControlSpaceCASUpdate(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
expected_state=KnowledgeFSControlSpaceState.DELETING,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETING,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
knowledge_space_id=recovered.knowledge_space_id,
|
||||
knowledge_space_revision=recovered.revision,
|
||||
)
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSLifecycleLeaseLostError(
|
||||
"Delete command lost ownership while registering its remote identity"
|
||||
)
|
||||
knowledge_space_id = recovered.knowledge_space_id
|
||||
expected_revision = recovered.revision
|
||||
else:
|
||||
if control_space.knowledge_space_id != recovered.knowledge_space_id:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Recovered deletion identity conflicts with registration")
|
||||
knowledge_space_id = control_space.knowledge_space_id
|
||||
expected_revision = control_space.knowledge_space_revision
|
||||
return _resolved_deletion_request(
|
||||
claimed,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
|
||||
def _complete_provision(
|
||||
self,
|
||||
*,
|
||||
claimed: _ClaimedCommand,
|
||||
remote_space: KnowledgeFSRemoteSpace,
|
||||
completed_at: datetime,
|
||||
) -> None:
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
control_space = control_repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if control_space is None or remote_space.namespace_id != claimed.tenant_id:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Provision response did not match its Dify control-space")
|
||||
already_applied = (
|
||||
control_space.state is KnowledgeFSControlSpaceState.ACTIVE
|
||||
and control_space.lifecycle_operation_id == claimed.operation_id
|
||||
and control_space.knowledge_space_id == remote_space.knowledge_space_id
|
||||
and control_space.knowledge_space_revision >= remote_space.revision
|
||||
)
|
||||
if not already_applied:
|
||||
if control_space.lifecycle_operation_id != claimed.operation_id:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Provision command no longer owns the control-space")
|
||||
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.ACTIVE,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
knowledge_space_id=remote_space.knowledge_space_id,
|
||||
knowledge_space_revision=remote_space.revision,
|
||||
)
|
||||
self._ack(outbox_repository, claimed=claimed, completed_at=completed_at)
|
||||
|
||||
def _already_deleted(self, *, claimed: _ClaimedCommand, completed_at: datetime) -> bool:
|
||||
with self._session_maker.begin() as session:
|
||||
control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if control_space is None or control_space.state is not KnowledgeFSControlSpaceState.DELETED:
|
||||
return False
|
||||
self._ack(
|
||||
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
|
||||
claimed=claimed,
|
||||
completed_at=completed_at,
|
||||
)
|
||||
return True
|
||||
|
||||
def _settle_deletion(self, *, claimed: _ClaimedCommand, progress: object, settled_at: datetime) -> bool:
|
||||
from services.knowledge_fs.lifecycle_port import KnowledgeFSDeletionProgress
|
||||
|
||||
if not isinstance(progress, KnowledgeFSDeletionProgress):
|
||||
raise TypeError("Deletion port returned an invalid progress value")
|
||||
with self._session_maker.begin() as session:
|
||||
control_repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
|
||||
control_space = control_repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if control_space is None or control_space.lifecycle_operation_id != claimed.operation_id:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Delete command no longer owns the control-space")
|
||||
if progress.phase in {KnowledgeFSDeletionPhase.IRREVERSIBLE, KnowledgeFSDeletionPhase.COMPLETED}:
|
||||
irreversible_at = progress.irreversible_at or settled_at
|
||||
if control_space.deletion_irreversible_at is None:
|
||||
KnowledgeFSControlSpaceLifecycleService(control_repository).mark_deletion_irreversible(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
irreversible_at=irreversible_at,
|
||||
)
|
||||
control_space = control_repository.get(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
)
|
||||
if control_space is None:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Control-space disappeared during deletion")
|
||||
if progress.phase is KnowledgeFSDeletionPhase.COMPLETED:
|
||||
KnowledgeFSControlSpaceLifecycleService(control_repository).transition(
|
||||
tenant_id=claimed.tenant_id,
|
||||
control_space_id=claimed.control_space_id,
|
||||
expected_resource_version=control_space.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.DELETED,
|
||||
lifecycle_operation_id=claimed.operation_id,
|
||||
knowledge_space_revision=max(control_space.knowledge_space_revision, progress.revision),
|
||||
)
|
||||
self._ack(outbox_repository, claimed=claimed, completed_at=settled_at)
|
||||
return True
|
||||
if not outbox_repository.schedule_retry(
|
||||
outbox_id=claimed.outbox_id,
|
||||
lease_owner=claimed.lease_owner,
|
||||
expected_lease_expires_at=claimed.lease_expires_at,
|
||||
next_attempt_at=settled_at + _retry_delay(claimed.attempt_count),
|
||||
error_code="REMOTE_DELETION_PENDING",
|
||||
error_message=f"KnowledgeFS deletion is {progress.phase}",
|
||||
):
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Delete command lease was lost before retry scheduling")
|
||||
return False
|
||||
|
||||
def _settle_revoke(
|
||||
self,
|
||||
*,
|
||||
claimed: _ClaimedCommand,
|
||||
acknowledgment: object,
|
||||
settled_at: datetime,
|
||||
) -> bool:
|
||||
if not isinstance(acknowledgment, KnowledgeFSCapabilityGrantRevokeAck):
|
||||
raise TypeError("Capability revoke port returned an invalid acknowledgment")
|
||||
payload = cast(KnowledgeFSRevokeCommandPayload, claimed.command_payload)
|
||||
if acknowledgment.state == "revoked" and acknowledgment.highest_revoke_sequence >= payload["revoke_sequence"]:
|
||||
with self._session_maker.begin() as session:
|
||||
self._ack(
|
||||
SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session),
|
||||
claimed=claimed,
|
||||
completed_at=settled_at,
|
||||
)
|
||||
return True
|
||||
self._retry(
|
||||
claimed=claimed,
|
||||
now=settled_at,
|
||||
error_code="REMOTE_REVOKE_WATERMARK_LAG",
|
||||
error_message="KnowledgeFS grant revoke acknowledgment is below the command watermark",
|
||||
)
|
||||
return False
|
||||
|
||||
def _retry(self, *, claimed: _ClaimedCommand, now: datetime, error_code: str, error_message: str) -> None:
|
||||
with self._session_maker.begin() as session:
|
||||
changed = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session).schedule_retry(
|
||||
outbox_id=claimed.outbox_id,
|
||||
lease_owner=claimed.lease_owner,
|
||||
expected_lease_expires_at=claimed.lease_expires_at,
|
||||
next_attempt_at=now + _retry_delay(claimed.attempt_count),
|
||||
error_code=error_code,
|
||||
error_message=error_message,
|
||||
)
|
||||
if not changed:
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Lifecycle command lease was lost before retry scheduling")
|
||||
|
||||
@staticmethod
|
||||
def _ack(
|
||||
repository: SQLAlchemyKnowledgeFSLifecycleOutboxRepository,
|
||||
*,
|
||||
claimed: _ClaimedCommand,
|
||||
completed_at: datetime,
|
||||
) -> None:
|
||||
if not repository.acknowledge(
|
||||
outbox_id=claimed.outbox_id,
|
||||
lease_owner=claimed.lease_owner,
|
||||
expected_lease_expires_at=claimed.lease_expires_at,
|
||||
completed_at=completed_at,
|
||||
):
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Lifecycle command lease was lost before ACK")
|
||||
|
||||
|
||||
class _ClaimedCommand(NamedTuple):
|
||||
outbox_id: str
|
||||
tenant_id: str
|
||||
control_space_id: str
|
||||
operation_id: str
|
||||
idempotency_key: str
|
||||
operation: KnowledgeFSLifecycleOperation
|
||||
command_payload: dict[str, object]
|
||||
attempt_count: int
|
||||
lease_owner: str
|
||||
lease_expires_at: datetime
|
||||
created_at: datetime
|
||||
|
||||
@classmethod
|
||||
def from_model(cls, command: KnowledgeFSLifecycleOutbox) -> _ClaimedCommand:
|
||||
if (
|
||||
command.status is not KnowledgeFSLifecycleOutboxStatus.PROCESSING
|
||||
or command.lease_owner is None
|
||||
or command.lease_expires_at is None
|
||||
):
|
||||
raise KnowledgeFSLifecycleLeaseLostError("Claimed command did not carry a complete lease")
|
||||
return cls(
|
||||
command.id,
|
||||
command.tenant_id,
|
||||
command.control_space_id,
|
||||
command.operation_id,
|
||||
command.idempotency_key,
|
||||
command.operation,
|
||||
cast(dict[str, object], command.command_payload),
|
||||
command.attempt_count,
|
||||
command.lease_owner,
|
||||
command.lease_expires_at,
|
||||
command.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _provision_request(command: _ClaimedCommand) -> KnowledgeFSIntegratedProvisionRequest:
|
||||
payload = cast(KnowledgeFSProvisionCommandPayload, command.command_payload)
|
||||
return KnowledgeFSIntegratedProvisionRequest(
|
||||
namespace_id=command.tenant_id,
|
||||
control_space_id=command.control_space_id,
|
||||
operation_id=command.operation_id,
|
||||
idempotency_key=command.idempotency_key,
|
||||
provisioning_key=payload["provisioning_key"],
|
||||
name=payload["name"],
|
||||
slug=payload["slug"],
|
||||
icon=payload["icon"],
|
||||
description=payload["description"],
|
||||
model_intent=payload["model_intent"],
|
||||
profile_intent=payload["profile_intent"],
|
||||
)
|
||||
|
||||
|
||||
def _resolved_deletion_request(
|
||||
command: _ClaimedCommand,
|
||||
*,
|
||||
knowledge_space_id: str,
|
||||
expected_revision: int,
|
||||
) -> KnowledgeFSIntegratedDeletionRequest:
|
||||
payload = cast(KnowledgeFSDeleteCommandPayload, command.command_payload)
|
||||
return KnowledgeFSIntegratedDeletionRequest(
|
||||
namespace_id=command.tenant_id,
|
||||
control_space_id=command.control_space_id,
|
||||
operation_id=command.operation_id,
|
||||
idempotency_key=command.idempotency_key,
|
||||
knowledge_space_id=knowledge_space_id,
|
||||
provisioning_key=payload["provisioning_key"],
|
||||
expected_revision=expected_revision,
|
||||
)
|
||||
|
||||
|
||||
def _revoke_request(command: _ClaimedCommand) -> KnowledgeFSCapabilityGrantRevokeRequest:
|
||||
payload = cast(KnowledgeFSRevokeCommandPayload, command.command_payload)
|
||||
return KnowledgeFSCapabilityGrantRevokeRequest(
|
||||
namespace_id=command.tenant_id,
|
||||
control_space_id=command.control_space_id,
|
||||
operation_id=command.operation_id,
|
||||
idempotency_key=command.idempotency_key,
|
||||
knowledge_space_id=payload["knowledge_space_id"],
|
||||
grant_id=payload["grant_id"],
|
||||
event_id=payload["event_id"],
|
||||
reason_code=payload["reason_code"],
|
||||
revoke_sequence=payload["revoke_sequence"],
|
||||
expected_revision=payload["expected_revision"],
|
||||
)
|
||||
|
||||
|
||||
def _retry_delay(attempt_count: int) -> timedelta:
|
||||
return timedelta(seconds=min(300, 2 ** min(attempt_count, 8)))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSLifecycleDispatchResult",
|
||||
"KnowledgeFSLifecycleLeaseLostError",
|
||||
"KnowledgeFSLifecycleSagaRunner",
|
||||
]
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Transactional KnowledgeFS authorization effects of Dify Workspace membership changes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from enum import StrEnum
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSAuthorizationRevision,
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpacePermissionStatus,
|
||||
)
|
||||
from services.knowledge_fs.revocation_commands import KnowledgeFSRevocationCommandProducer
|
||||
|
||||
_MAX_SAFE_INTEGER = 2**53 - 1
|
||||
|
||||
|
||||
class KnowledgeFSWorkspaceMembershipChange(StrEnum):
|
||||
MEMBER_ADDED = "member_added"
|
||||
MEMBER_REMOVED = "member_removed"
|
||||
ROLE_CHANGED = "role_changed"
|
||||
|
||||
|
||||
class KnowledgeFSMembershipChangeInvariantError(RuntimeError):
|
||||
"""Workspace membership state cannot be applied safely to KnowledgeFS authorization."""
|
||||
|
||||
|
||||
def knowledge_fs_membership_schema_installed(session: Session) -> bool:
|
||||
"""Support additive rollout and lightweight tests before the control-plane table exists."""
|
||||
return sa.inspect(session.connection()).has_table(KnowledgeFSControlSpace.__tablename__)
|
||||
|
||||
|
||||
def apply_workspace_membership_change(
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
actor_account_id: str,
|
||||
account_ids: Sequence[str],
|
||||
change: KnowledgeFSWorkspaceMembershipChange,
|
||||
removed_account_id: str | None = None,
|
||||
replacement_owner_account_id: str | None = None,
|
||||
) -> None:
|
||||
"""Advance every Space epoch and append revokes in the caller's membership transaction."""
|
||||
normalized_tenant_id = _identifier(tenant_id, "tenant_id")
|
||||
normalized_actor_id = _identifier(actor_account_id, "actor_account_id")
|
||||
normalized_accounts = tuple(dict.fromkeys(_identifier(value, "account_id") for value in account_ids))
|
||||
if not normalized_accounts:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("Membership change requires an affected account")
|
||||
normalized_removed_id = _optional_identifier(removed_account_id, "removed_account_id")
|
||||
normalized_replacement_id = _optional_identifier(
|
||||
replacement_owner_account_id,
|
||||
"replacement_owner_account_id",
|
||||
)
|
||||
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED:
|
||||
if normalized_removed_id not in normalized_accounts or normalized_replacement_id is None:
|
||||
raise KnowledgeFSMembershipChangeInvariantError(
|
||||
"Member removal requires the removed account and replacement Workspace owner"
|
||||
)
|
||||
if normalized_removed_id == normalized_replacement_id:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("Removed member cannot remain the replacement owner")
|
||||
elif normalized_removed_id is not None or normalized_replacement_id is not None:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("Owner replacement is valid only for member removal")
|
||||
|
||||
if not knowledge_fs_membership_schema_installed(session):
|
||||
return
|
||||
spaces, revisions = _lock_workspace_authorization(session, normalized_tenant_id)
|
||||
if not spaces:
|
||||
return
|
||||
|
||||
producer = KnowledgeFSRevocationCommandProducer()
|
||||
for space in spaces:
|
||||
revision = revisions[space.id]
|
||||
if revision.membership_epoch >= _MAX_SAFE_INTEGER:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS membership epoch is exhausted")
|
||||
revision.membership_epoch += 1
|
||||
|
||||
acl_changed = False
|
||||
if normalized_removed_id is not None and normalized_replacement_id is not None:
|
||||
acl_changed = _remove_space_member(
|
||||
session=session,
|
||||
space=space,
|
||||
removed_account_id=normalized_removed_id,
|
||||
replacement_owner_account_id=normalized_replacement_id,
|
||||
actor_account_id=normalized_actor_id,
|
||||
)
|
||||
if acl_changed:
|
||||
if revision.space_acl_epoch >= _MAX_SAFE_INTEGER:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS Space ACL epoch is exhausted")
|
||||
revision.space_acl_epoch += 1
|
||||
|
||||
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_ADDED:
|
||||
continue
|
||||
reason_code = (
|
||||
"workspace_membership_removed"
|
||||
if change is KnowledgeFSWorkspaceMembershipChange.MEMBER_REMOVED
|
||||
else "workspace_role_changed"
|
||||
)
|
||||
for account_id in normalized_accounts:
|
||||
producer.enqueue_principal_grants(
|
||||
session=session,
|
||||
tenant_id=normalized_tenant_id,
|
||||
control_space_id=space.id,
|
||||
subject=f"dify-account:{account_id}",
|
||||
reason_code=reason_code,
|
||||
caller_kinds=("interactive",),
|
||||
)
|
||||
|
||||
|
||||
def apply_workspace_rbac_role_change(*, session: Session, tenant_id: str) -> None:
|
||||
"""Advance Workspace membership epochs and revoke every interactive grant.
|
||||
|
||||
Custom Enterprise RBAC role edits can affect any member bound to the role,
|
||||
while Dify deliberately does not mirror those bindings locally. The safe
|
||||
invalidation therefore scans durable grant provenance per control-space
|
||||
instead of relying on a potentially incomplete member list.
|
||||
"""
|
||||
normalized_tenant_id = _identifier(tenant_id, "tenant_id")
|
||||
if not knowledge_fs_membership_schema_installed(session):
|
||||
return
|
||||
spaces, revisions = _lock_workspace_authorization(session, normalized_tenant_id)
|
||||
if not spaces:
|
||||
return
|
||||
|
||||
producer = KnowledgeFSRevocationCommandProducer()
|
||||
for space in spaces:
|
||||
revision = revisions[space.id]
|
||||
if revision.membership_epoch >= _MAX_SAFE_INTEGER:
|
||||
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS membership epoch is exhausted")
|
||||
revision.membership_epoch += 1
|
||||
producer.enqueue_control_space_grants(
|
||||
session=session,
|
||||
tenant_id=normalized_tenant_id,
|
||||
control_space_id=space.id,
|
||||
reason_code="workspace_rbac_role_changed",
|
||||
caller_kinds=("interactive",),
|
||||
)
|
||||
|
||||
|
||||
def _lock_workspace_authorization(
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
) -> tuple[tuple[KnowledgeFSControlSpace, ...], dict[str, KnowledgeFSAuthorizationRevision]]:
|
||||
spaces = tuple(
|
||||
session.scalars(
|
||||
sa.select(KnowledgeFSControlSpace)
|
||||
.where(KnowledgeFSControlSpace.tenant_id == tenant_id)
|
||||
.order_by(KnowledgeFSControlSpace.id)
|
||||
.with_for_update()
|
||||
)
|
||||
)
|
||||
if not spaces:
|
||||
return (), {}
|
||||
revisions = {
|
||||
revision.control_space_id: revision
|
||||
for revision in session.scalars(
|
||||
sa.select(KnowledgeFSAuthorizationRevision)
|
||||
.where(
|
||||
KnowledgeFSAuthorizationRevision.tenant_id == tenant_id,
|
||||
KnowledgeFSAuthorizationRevision.control_space_id.in_(space.id for space in spaces),
|
||||
)
|
||||
.order_by(KnowledgeFSAuthorizationRevision.control_space_id)
|
||||
.with_for_update()
|
||||
)
|
||||
}
|
||||
if len(revisions) != len(spaces):
|
||||
raise KnowledgeFSMembershipChangeInvariantError("KnowledgeFS authorization revision is missing")
|
||||
return spaces, revisions
|
||||
|
||||
|
||||
def _remove_space_member(
|
||||
*,
|
||||
session: Session,
|
||||
space: KnowledgeFSControlSpace,
|
||||
removed_account_id: str,
|
||||
replacement_owner_account_id: str,
|
||||
actor_account_id: str,
|
||||
) -> bool:
|
||||
permissions = {
|
||||
permission.account_id: permission
|
||||
for permission in session.scalars(
|
||||
sa.select(KnowledgeFSControlSpacePermission)
|
||||
.where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == space.tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == space.id,
|
||||
KnowledgeFSControlSpacePermission.account_id.in_((removed_account_id, replacement_owner_account_id)),
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
}
|
||||
changed = False
|
||||
now = naive_utc_now()
|
||||
removed_permission = permissions.get(removed_account_id)
|
||||
if removed_permission is not None and (removed_permission.status is KnowledgeFSControlSpacePermissionStatus.ACTIVE):
|
||||
removed_permission.status = KnowledgeFSControlSpacePermissionStatus.REVOKED
|
||||
removed_permission.revision += 1
|
||||
removed_permission.revoked_at = now
|
||||
removed_permission.revoked_by_account_id = actor_account_id
|
||||
changed = True
|
||||
|
||||
if space.owner_account_id != removed_account_id:
|
||||
return changed
|
||||
space.owner_account_id = replacement_owner_account_id
|
||||
space.resource_version += 1
|
||||
replacement_permission = permissions.get(replacement_owner_account_id)
|
||||
if replacement_permission is None:
|
||||
session.add(
|
||||
KnowledgeFSControlSpacePermission(
|
||||
tenant_id=space.tenant_id,
|
||||
control_space_id=space.id,
|
||||
account_id=replacement_owner_account_id,
|
||||
role=KnowledgeFSControlSpacePermissionRole.OWNER,
|
||||
granted_by_account_id=actor_account_id,
|
||||
)
|
||||
)
|
||||
elif (
|
||||
replacement_permission.role is not KnowledgeFSControlSpacePermissionRole.OWNER
|
||||
or replacement_permission.status is not KnowledgeFSControlSpacePermissionStatus.ACTIVE
|
||||
):
|
||||
replacement_permission.role = KnowledgeFSControlSpacePermissionRole.OWNER
|
||||
replacement_permission.status = KnowledgeFSControlSpacePermissionStatus.ACTIVE
|
||||
replacement_permission.revision += 1
|
||||
replacement_permission.revoked_at = None
|
||||
replacement_permission.revoked_by_account_id = None
|
||||
return True
|
||||
|
||||
|
||||
def _identifier(value: str, field: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized or normalized != value or len(normalized) > 255:
|
||||
raise KnowledgeFSMembershipChangeInvariantError(f"{field} must be a normalized identifier")
|
||||
return normalized
|
||||
|
||||
|
||||
def _optional_identifier(value: str | None, field: str) -> str | None:
|
||||
return None if value is None else _identifier(value, field)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSMembershipChangeInvariantError",
|
||||
"KnowledgeFSWorkspaceMembershipChange",
|
||||
"apply_workspace_membership_change",
|
||||
"apply_workspace_rbac_role_change",
|
||||
"knowledge_fs_membership_schema_installed",
|
||||
]
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Low-cardinality operational metrics for the Dify side of KnowledgeFS.
|
||||
|
||||
The metric contracts intentionally exclude tenant, resource, principal, token, URL, and free-form
|
||||
error values. Callers own best-effort failure isolation so telemetry can never change product state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from functools import lru_cache
|
||||
from typing import Literal, NamedTuple, Protocol
|
||||
|
||||
from opentelemetry.metrics import Observation, get_meter
|
||||
|
||||
|
||||
class KnowledgeFSCapabilityIssuanceMetric(NamedTuple):
|
||||
caller_kind: str
|
||||
operation_id: str
|
||||
outcome: Literal["denied", "failed", "issued"]
|
||||
reason: str
|
||||
|
||||
|
||||
class KnowledgeFSBatchStatusMetric(NamedTuple):
|
||||
duration_seconds: float
|
||||
missing_spaces: int
|
||||
outcome: Literal["degraded", "failed", "success"]
|
||||
requested_spaces: int
|
||||
returned_spaces: int
|
||||
|
||||
|
||||
class KnowledgeFSControlSpaceStateMetric(NamedTuple):
|
||||
duration_seconds: float
|
||||
from_state: str
|
||||
to_state: str
|
||||
|
||||
|
||||
class KnowledgeFSLifecycleTaskMetric(NamedTuple):
|
||||
duration_seconds: float | None
|
||||
operation: str
|
||||
status: Literal["dispatch_error", "queued", "retry", "running", "succeeded"]
|
||||
|
||||
|
||||
class KnowledgeFSOperationAdmissionMetric(NamedTuple):
|
||||
operation_id: str
|
||||
bucket: str
|
||||
phase: Literal["commit", "refund", "reserve"]
|
||||
outcome: Literal["failure", "success"]
|
||||
|
||||
|
||||
class KnowledgeFSOperationalMetricsPort(Protocol):
|
||||
def record_batch_status(self, event: KnowledgeFSBatchStatusMetric) -> None: ...
|
||||
|
||||
def record_capability_issuance(self, event: KnowledgeFSCapabilityIssuanceMetric) -> None: ...
|
||||
|
||||
def record_control_space_state(self, event: KnowledgeFSControlSpaceStateMetric) -> None: ...
|
||||
|
||||
def record_lifecycle_task(self, event: KnowledgeFSLifecycleTaskMetric) -> None: ...
|
||||
|
||||
def record_operation_admission(self, event: KnowledgeFSOperationAdmissionMetric) -> None: ...
|
||||
|
||||
def register_control_space_state_gauge(self, read_counts: Callable[[], Mapping[str, int]]) -> None: ...
|
||||
|
||||
|
||||
class _CounterPort(Protocol):
|
||||
def add(self, amount: int, *, attributes: dict[str, str]) -> None: ...
|
||||
|
||||
|
||||
class _HistogramPort(Protocol):
|
||||
def record(self, amount: float, *, attributes: dict[str, str]) -> None: ...
|
||||
|
||||
|
||||
class _MeterPort(Protocol):
|
||||
def create_counter(self, name: str, *, description: str, unit: str) -> _CounterPort: ...
|
||||
|
||||
def create_histogram(self, name: str, *, description: str, unit: str) -> _HistogramPort: ...
|
||||
|
||||
def create_observable_gauge(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
callbacks: tuple[Callable[[object], Iterable[Observation]], ...],
|
||||
description: str,
|
||||
unit: str,
|
||||
) -> object: ...
|
||||
|
||||
|
||||
class OpenTelemetryKnowledgeFSOperationalMetrics:
|
||||
"""Map sanitized events to OpenTelemetry counters and duration histograms."""
|
||||
|
||||
def __init__(self, *, meter: _MeterPort | None = None) -> None:
|
||||
resolved_meter = meter or get_meter("dify.knowledge_fs")
|
||||
self._meter = resolved_meter
|
||||
self._control_space_state_gauge: object | None = None
|
||||
self._capability_issuance = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.capability_issuance",
|
||||
description="KnowledgeFS capability issuance outcomes",
|
||||
unit="{issuance}",
|
||||
)
|
||||
self._batch_requests = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.batch_status_requests",
|
||||
description="KnowledgeFS batch status request outcomes",
|
||||
unit="{request}",
|
||||
)
|
||||
self._batch_spaces = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.batch_status_spaces",
|
||||
description="KnowledgeFS batch status returned and missing spaces",
|
||||
unit="{space}",
|
||||
)
|
||||
self._batch_latency = resolved_meter.create_histogram(
|
||||
"dify.knowledge_fs.batch_status_latency",
|
||||
description="KnowledgeFS batch status latency",
|
||||
unit="s",
|
||||
)
|
||||
self._control_space_transitions = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.control_space_transitions",
|
||||
description="KnowledgeFS control-space lifecycle transitions",
|
||||
unit="{transition}",
|
||||
)
|
||||
self._control_space_state_duration = resolved_meter.create_histogram(
|
||||
"dify.knowledge_fs.control_space_state_duration",
|
||||
description="Time spent in a KnowledgeFS control-space state before transition",
|
||||
unit="s",
|
||||
)
|
||||
self._lifecycle_tasks = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.lifecycle_tasks",
|
||||
description="KnowledgeFS durable lifecycle task state observations",
|
||||
unit="{task}",
|
||||
)
|
||||
self._lifecycle_task_latency = resolved_meter.create_histogram(
|
||||
"dify.knowledge_fs.lifecycle_task_latency",
|
||||
description="KnowledgeFS durable lifecycle task terminal latency",
|
||||
unit="s",
|
||||
)
|
||||
self._revoke_latency = resolved_meter.create_histogram(
|
||||
"dify.knowledge_fs.revoke_latency",
|
||||
description="KnowledgeFS capability revoke enqueue-to-ack latency",
|
||||
unit="s",
|
||||
)
|
||||
self._operation_admission = resolved_meter.create_counter(
|
||||
"dify.knowledge_fs.operation_admission",
|
||||
description="KnowledgeFS direct-operation reserve and finalization outcomes",
|
||||
unit="{operation}",
|
||||
)
|
||||
|
||||
def record_capability_issuance(self, event: KnowledgeFSCapabilityIssuanceMetric) -> None:
|
||||
self._capability_issuance.add(
|
||||
1,
|
||||
attributes={
|
||||
"caller_kind": event.caller_kind,
|
||||
"operation_id": event.operation_id,
|
||||
"outcome": event.outcome,
|
||||
"reason": event.reason,
|
||||
},
|
||||
)
|
||||
|
||||
def record_batch_status(self, event: KnowledgeFSBatchStatusMetric) -> None:
|
||||
outcome: dict[str, str] = {"outcome": event.outcome}
|
||||
self._batch_requests.add(1, attributes=outcome)
|
||||
self._batch_latency.record(event.duration_seconds, attributes=outcome)
|
||||
self._batch_spaces.add(event.returned_spaces, attributes={"result": "returned"})
|
||||
self._batch_spaces.add(event.missing_spaces, attributes={"result": "missing"})
|
||||
|
||||
def record_control_space_state(self, event: KnowledgeFSControlSpaceStateMetric) -> None:
|
||||
attributes = {"from_state": event.from_state, "to_state": event.to_state}
|
||||
self._control_space_transitions.add(1, attributes=attributes)
|
||||
self._control_space_state_duration.record(event.duration_seconds, attributes=attributes)
|
||||
|
||||
def record_lifecycle_task(self, event: KnowledgeFSLifecycleTaskMetric) -> None:
|
||||
attributes = {"operation": event.operation, "status": event.status}
|
||||
self._lifecycle_tasks.add(1, attributes=attributes)
|
||||
if event.duration_seconds is None:
|
||||
return
|
||||
self._lifecycle_task_latency.record(event.duration_seconds, attributes=attributes)
|
||||
if event.operation == "revoke" and event.status == "succeeded":
|
||||
self._revoke_latency.record(event.duration_seconds, attributes={"operation": "revoke"})
|
||||
|
||||
def record_operation_admission(self, event: KnowledgeFSOperationAdmissionMetric) -> None:
|
||||
self._operation_admission.add(
|
||||
1,
|
||||
attributes={
|
||||
"bucket": event.bucket,
|
||||
"operation_id": event.operation_id,
|
||||
"outcome": event.outcome,
|
||||
"phase": event.phase,
|
||||
},
|
||||
)
|
||||
|
||||
def register_control_space_state_gauge(self, read_counts: Callable[[], Mapping[str, int]]) -> None:
|
||||
"""Register one DB-backed current-state instrument per process."""
|
||||
|
||||
if self._control_space_state_gauge is not None:
|
||||
return
|
||||
|
||||
def observe(_: object) -> Iterable[Observation]:
|
||||
try:
|
||||
counts = read_counts()
|
||||
return tuple(
|
||||
Observation(
|
||||
max(0, counts.get(state, 0)),
|
||||
attributes={"aggregation_scope": "global_database_snapshot", "state": state},
|
||||
)
|
||||
for state in ("provisioning", "deleting", "error")
|
||||
)
|
||||
except Exception:
|
||||
return ()
|
||||
|
||||
self._control_space_state_gauge = self._meter.create_observable_gauge(
|
||||
"dify.knowledge_fs.control_spaces",
|
||||
callbacks=(observe,),
|
||||
description=(
|
||||
"Global KnowledgeFS control-space database snapshot by operational state; "
|
||||
"aggregate replica series with max, never sum"
|
||||
),
|
||||
unit="{space}",
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_knowledge_fs_operational_metrics() -> KnowledgeFSOperationalMetricsPort:
|
||||
"""Return one process-wide instrument set to avoid duplicate OpenTelemetry registration."""
|
||||
|
||||
return OpenTelemetryKnowledgeFSOperationalMetrics()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSBatchStatusMetric",
|
||||
"KnowledgeFSCapabilityIssuanceMetric",
|
||||
"KnowledgeFSControlSpaceStateMetric",
|
||||
"KnowledgeFSLifecycleTaskMetric",
|
||||
"KnowledgeFSOperationAdmissionMetric",
|
||||
"KnowledgeFSOperationalMetricsPort",
|
||||
"OpenTelemetryKnowledgeFSOperationalMetrics",
|
||||
"get_knowledge_fs_operational_metrics",
|
||||
]
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Weighted admission for BFF calls and browser-executed KnowledgeFS operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import Literal, NamedTuple, Protocol, cast
|
||||
|
||||
from configs import dify_config
|
||||
from extensions.ext_redis import redis_client
|
||||
from services.billing_service import BillingService
|
||||
from services.feature_service import FeatureService, KnowledgeRateLimitModel
|
||||
from services.knowledge_fs.capability_broker import KnowledgeFSIssuedProductCapability
|
||||
from services.knowledge_fs.credential_service import KnowledgeFSServiceCredentialProfile
|
||||
from services.knowledge_fs.observability import (
|
||||
KnowledgeFSOperationAdmissionMetric,
|
||||
KnowledgeFSOperationalMetricsPort,
|
||||
get_knowledge_fs_operational_metrics,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_BILLING_FEATURE_KEY = "knowledge_fs_operations"
|
||||
_RATE_LIMIT_WINDOW_MS = 60_000
|
||||
_WEIGHTED_RATE_LIMIT_SCRIPT = """
|
||||
redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[2])
|
||||
local current = redis.call('ZCARD', KEYS[1])
|
||||
local requested = tonumber(ARGV[4])
|
||||
if current + requested > tonumber(ARGV[3]) then
|
||||
return 0
|
||||
end
|
||||
for index = 1, requested do
|
||||
redis.call('ZADD', KEYS[1], ARGV[1], ARGV[5] .. ':' .. index)
|
||||
end
|
||||
redis.call('PEXPIRE', KEYS[1], 61000)
|
||||
return 1
|
||||
"""
|
||||
|
||||
|
||||
class KnowledgeFSOperationRateLimitExceededError(RuntimeError):
|
||||
"""The operation's weighted per-minute bucket is exhausted."""
|
||||
|
||||
|
||||
class KnowledgeFSOperationQuotaExceededError(RuntimeError):
|
||||
"""The billing service explicitly rejected an operation reservation."""
|
||||
|
||||
|
||||
class KnowledgeFSOperationUsage(NamedTuple):
|
||||
tenant_id: str
|
||||
operation_id: str
|
||||
bucket: str
|
||||
billing_cost: int
|
||||
rate_limit_cost: int
|
||||
|
||||
|
||||
class KnowledgeFSOperationChargePort(Protocol):
|
||||
def commit(self) -> None: ...
|
||||
|
||||
def refund(self) -> None: ...
|
||||
|
||||
|
||||
class KnowledgeFSOperationRateLimitPort(Protocol):
|
||||
def admit(self, usage: KnowledgeFSOperationUsage) -> None: ...
|
||||
|
||||
|
||||
class KnowledgeFSOperationBillingPort(Protocol):
|
||||
def reserve(self, usage: KnowledgeFSOperationUsage) -> KnowledgeFSOperationChargePort: ...
|
||||
|
||||
|
||||
class KnowledgeFSRateLimitAuditPort(Protocol):
|
||||
def record_rejection(self, usage: KnowledgeFSOperationUsage, *, subscription_plan: str) -> None: ...
|
||||
|
||||
|
||||
class KnowledgeFSBillingGateway(Protocol):
|
||||
def quota_reserve(self, **kwargs: object) -> dict[str, object]: ...
|
||||
|
||||
def quota_commit(self, **kwargs: object) -> dict[str, object]: ...
|
||||
|
||||
def quota_release(self, **kwargs: object) -> dict[str, object]: ...
|
||||
|
||||
|
||||
class KnowledgeFSRedisEvalPort(Protocol):
|
||||
def eval(self, *args: object) -> int: ...
|
||||
|
||||
|
||||
class _NoopCharge:
|
||||
def commit(self) -> None:
|
||||
return
|
||||
|
||||
def refund(self) -> None:
|
||||
return
|
||||
|
||||
|
||||
class _DifyBillingCharge:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
gateway: KnowledgeFSBillingGateway,
|
||||
usage: KnowledgeFSOperationUsage,
|
||||
reservation_id: str,
|
||||
) -> None:
|
||||
self._gateway = gateway
|
||||
self._usage = usage
|
||||
self._reservation_id = reservation_id
|
||||
self._finalized = False
|
||||
|
||||
def commit(self) -> None:
|
||||
if self._finalized:
|
||||
return
|
||||
self._finalized = True
|
||||
try:
|
||||
self._gateway.quota_commit(
|
||||
tenant_id=self._usage.tenant_id,
|
||||
feature_key=_BILLING_FEATURE_KEY,
|
||||
bucket=self._usage.bucket,
|
||||
reservation_id=self._reservation_id,
|
||||
actual_amount=self._usage.billing_cost,
|
||||
meta={"operation_id": self._usage.operation_id, "source": "knowledge_fs"},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"KnowledgeFS billing commit failed for tenant_id=%s operation_id=%s",
|
||||
self._usage.tenant_id,
|
||||
self._usage.operation_id,
|
||||
)
|
||||
|
||||
def refund(self) -> None:
|
||||
if self._finalized:
|
||||
return
|
||||
self._finalized = True
|
||||
try:
|
||||
self._gateway.quota_release(
|
||||
tenant_id=self._usage.tenant_id,
|
||||
feature_key=_BILLING_FEATURE_KEY,
|
||||
bucket=self._usage.bucket,
|
||||
reservation_id=self._reservation_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"KnowledgeFS billing release failed for tenant_id=%s operation_id=%s",
|
||||
self._usage.tenant_id,
|
||||
self._usage.operation_id,
|
||||
)
|
||||
|
||||
|
||||
class LoggingKnowledgeFSRateLimitAudit:
|
||||
def record_rejection(self, usage: KnowledgeFSOperationUsage, *, subscription_plan: str) -> None:
|
||||
logger.warning(
|
||||
"KnowledgeFS weighted rate limit rejected tenant_id=%s operation_id=%s bucket=%s cost=%s plan=%s",
|
||||
usage.tenant_id,
|
||||
usage.operation_id,
|
||||
usage.bucket,
|
||||
usage.rate_limit_cost,
|
||||
subscription_plan,
|
||||
)
|
||||
|
||||
|
||||
class DifyKnowledgeFSWeightedRateLimitPort:
|
||||
"""Consume weighted operation units atomically from one Redis bucket."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
redis: KnowledgeFSRedisEvalPort | None = None,
|
||||
audit: KnowledgeFSRateLimitAuditPort,
|
||||
rate_limit_lookup: Callable[[str], KnowledgeRateLimitModel] = FeatureService.get_knowledge_rate_limit,
|
||||
clock_ms: Callable[[], int] = lambda: int(time.time() * 1000),
|
||||
member_id: Callable[[], str] = lambda: str(uuid.uuid4()),
|
||||
) -> None:
|
||||
self._redis = redis or cast(KnowledgeFSRedisEvalPort, redis_client)
|
||||
self._audit = audit
|
||||
self._rate_limit_lookup = rate_limit_lookup
|
||||
self._clock_ms = clock_ms
|
||||
self._member_id = member_id
|
||||
|
||||
def admit(self, usage: KnowledgeFSOperationUsage) -> None:
|
||||
rate_limit = self._rate_limit_lookup(usage.tenant_id)
|
||||
if not rate_limit.enabled:
|
||||
return
|
||||
now = self._clock_ms()
|
||||
accepted = self._redis.eval(
|
||||
_WEIGHTED_RATE_LIMIT_SCRIPT,
|
||||
1,
|
||||
f"knowledge_fs:rate_limit:{usage.tenant_id}:{usage.bucket}",
|
||||
now,
|
||||
now - _RATE_LIMIT_WINDOW_MS,
|
||||
rate_limit.limit,
|
||||
usage.rate_limit_cost,
|
||||
self._member_id(),
|
||||
)
|
||||
if accepted == 1:
|
||||
return
|
||||
self._audit.record_rejection(usage, subscription_plan=rate_limit.subscription_plan)
|
||||
raise KnowledgeFSOperationRateLimitExceededError("KnowledgeFS operation rate limit exceeded")
|
||||
|
||||
|
||||
class DifyKnowledgeFSBillingPort:
|
||||
"""Reserve operation-specific billing units for the caller to commit or release."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
gateway: KnowledgeFSBillingGateway | None = None,
|
||||
billing_enabled: Callable[[], bool] = lambda: dify_config.BILLING_ENABLED,
|
||||
request_id: Callable[[], str] = lambda: str(uuid.uuid4()),
|
||||
) -> None:
|
||||
self._gateway = gateway or cast(KnowledgeFSBillingGateway, BillingService)
|
||||
self._billing_enabled = billing_enabled
|
||||
self._request_id = request_id
|
||||
|
||||
def reserve(self, usage: KnowledgeFSOperationUsage) -> KnowledgeFSOperationChargePort:
|
||||
if not self._billing_enabled():
|
||||
return _NoopCharge()
|
||||
try:
|
||||
result = self._gateway.quota_reserve(
|
||||
tenant_id=usage.tenant_id,
|
||||
feature_key=_BILLING_FEATURE_KEY,
|
||||
bucket=usage.bucket,
|
||||
request_id=self._request_id(),
|
||||
amount=usage.billing_cost,
|
||||
meta={"operation_id": usage.operation_id, "source": "knowledge_fs"},
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"KnowledgeFS billing reservation unavailable for tenant_id=%s operation_id=%s; allowing request",
|
||||
usage.tenant_id,
|
||||
usage.operation_id,
|
||||
)
|
||||
return _NoopCharge()
|
||||
reservation_id = result.get("reservation_id")
|
||||
if not isinstance(reservation_id, str) or not reservation_id:
|
||||
raise KnowledgeFSOperationQuotaExceededError("KnowledgeFS operation quota exceeded")
|
||||
return _DifyBillingCharge(gateway=self._gateway, usage=usage, reservation_id=reservation_id)
|
||||
|
||||
|
||||
class KnowledgeFSOperationAdmissionService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
rate_limit: KnowledgeFSOperationRateLimitPort,
|
||||
billing: KnowledgeFSOperationBillingPort,
|
||||
) -> None:
|
||||
self._rate_limit = rate_limit
|
||||
self._billing = billing
|
||||
|
||||
def reserve(self, *, tenant_id: str, operation_id: str) -> KnowledgeFSOperationChargePort:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id]
|
||||
usage = KnowledgeFSOperationUsage(
|
||||
tenant_id=tenant_id,
|
||||
operation_id=operation_id,
|
||||
bucket=operation.rate_limit_bucket,
|
||||
billing_cost=operation.billing_cost,
|
||||
rate_limit_cost=operation.rate_limit_cost,
|
||||
)
|
||||
self._rate_limit.admit(usage)
|
||||
return self._billing.reserve(usage)
|
||||
|
||||
|
||||
class KnowledgeFSDirectCapabilityBrokerPort(Protocol):
|
||||
def issue_interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability: ...
|
||||
|
||||
def issue_service(
|
||||
self,
|
||||
*,
|
||||
profile: KnowledgeFSServiceCredentialProfile,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability: ...
|
||||
|
||||
|
||||
class KnowledgeFSDirectOperationAdmissionService:
|
||||
"""Admit browser-executed operations at the Dify Capability issuance boundary.
|
||||
|
||||
A successful Capability issuance is the only outcome Dify can observe before the browser
|
||||
talks to KnowledgeFS, so it commits the operation charge immediately. Issuance failures
|
||||
release the reservation. Controllers must treat alternative endpoints for the same manifest
|
||||
operation as separate entry points and must never chain them for one user action.
|
||||
"""
|
||||
|
||||
_admission: KnowledgeFSOperationAdmissionService
|
||||
_broker: KnowledgeFSDirectCapabilityBrokerPort
|
||||
_metrics: KnowledgeFSOperationalMetricsPort
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
admission: KnowledgeFSOperationAdmissionService,
|
||||
broker: KnowledgeFSDirectCapabilityBrokerPort,
|
||||
metrics: KnowledgeFSOperationalMetricsPort | None = None,
|
||||
) -> None:
|
||||
self._admission = admission
|
||||
self._broker = broker
|
||||
self._metrics = metrics or get_knowledge_fs_operational_metrics()
|
||||
|
||||
def issue_interactive(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
return self._issue(
|
||||
tenant_id=tenant_id,
|
||||
operation_id=operation_id,
|
||||
issue=lambda: self._broker.issue_interactive(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
resource_id=resource_id,
|
||||
trace_id=trace_id,
|
||||
),
|
||||
)
|
||||
|
||||
def issue_service(
|
||||
self,
|
||||
*,
|
||||
profile: KnowledgeFSServiceCredentialProfile,
|
||||
operation_id: str,
|
||||
resource_id: str | None = None,
|
||||
trace_id: str | None = None,
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
return self._issue(
|
||||
tenant_id=profile.tenant_id,
|
||||
operation_id=operation_id,
|
||||
issue=lambda: self._broker.issue_service(
|
||||
profile=profile,
|
||||
operation_id=operation_id,
|
||||
resource_id=resource_id,
|
||||
trace_id=trace_id,
|
||||
),
|
||||
)
|
||||
|
||||
def _issue(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
operation_id: str,
|
||||
issue: Callable[[], KnowledgeFSIssuedProductCapability],
|
||||
) -> KnowledgeFSIssuedProductCapability:
|
||||
bucket = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].rate_limit_bucket
|
||||
try:
|
||||
charge = self._admission.reserve(tenant_id=tenant_id, operation_id=operation_id)
|
||||
except BaseException:
|
||||
self._record_metric(operation_id, bucket=bucket, phase="reserve", outcome="failure")
|
||||
raise
|
||||
self._record_metric(operation_id, bucket=bucket, phase="reserve", outcome="success")
|
||||
try:
|
||||
issued = issue()
|
||||
except BaseException:
|
||||
try:
|
||||
charge.refund()
|
||||
except BaseException:
|
||||
self._record_metric(operation_id, bucket=bucket, phase="refund", outcome="failure")
|
||||
raise
|
||||
self._record_metric(operation_id, bucket=bucket, phase="refund", outcome="success")
|
||||
raise
|
||||
try:
|
||||
charge.commit()
|
||||
except BaseException:
|
||||
self._record_metric(operation_id, bucket=bucket, phase="commit", outcome="failure")
|
||||
raise
|
||||
self._record_metric(operation_id, bucket=bucket, phase="commit", outcome="success")
|
||||
return issued
|
||||
|
||||
def _record_metric(
|
||||
self,
|
||||
operation_id: str,
|
||||
*,
|
||||
bucket: str,
|
||||
phase: Literal["commit", "refund", "reserve"],
|
||||
outcome: Literal["failure", "success"],
|
||||
) -> None:
|
||||
try:
|
||||
self._metrics.record_operation_admission(
|
||||
KnowledgeFSOperationAdmissionMetric(operation_id, bucket, phase, outcome)
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("KnowledgeFS direct-operation admission metric export failed", exc_info=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DifyKnowledgeFSBillingPort",
|
||||
"DifyKnowledgeFSWeightedRateLimitPort",
|
||||
"KnowledgeFSDirectOperationAdmissionService",
|
||||
"KnowledgeFSOperationAdmissionService",
|
||||
"KnowledgeFSOperationChargePort",
|
||||
"KnowledgeFSOperationQuotaExceededError",
|
||||
"KnowledgeFSOperationRateLimitExceededError",
|
||||
"KnowledgeFSOperationUsage",
|
||||
"LoggingKnowledgeFSRateLimitAudit",
|
||||
]
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Conservative local/remote reconciliation for KnowledgeFS control-spaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from typing import NamedTuple
|
||||
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from models.knowledge_fs import KnowledgeFSControlSpaceState
|
||||
from repositories.sqlalchemy_knowledge_fs_control_space_repository import (
|
||||
SQLAlchemyKnowledgeFSControlSpaceRepository,
|
||||
)
|
||||
from services.knowledge_fs.control_space_commands import KnowledgeFSControlSpaceCommandService
|
||||
from services.knowledge_fs.control_space_lifecycle import KnowledgeFSControlSpaceLifecycleService
|
||||
from services.knowledge_fs.lifecycle_port import KnowledgeFSLifecycleRemotePort
|
||||
|
||||
|
||||
class KnowledgeFSOrphanReport(NamedTuple):
|
||||
repaired_control_space_ids: tuple[str, ...]
|
||||
remote_orphan_space_ids: tuple[str, ...]
|
||||
local_missing_remote_ids: tuple[str, ...]
|
||||
cleanup_control_space_ids: tuple[str, ...]
|
||||
|
||||
|
||||
class KnowledgeFSOrphanReconciler:
|
||||
"""Repair identities proven by provisioning_key; report ambiguous drift fail-closed."""
|
||||
|
||||
def __init__(self, session_maker: sessionmaker[Session], remote: KnowledgeFSLifecycleRemotePort):
|
||||
self._session_maker = session_maker
|
||||
self._remote = remote
|
||||
|
||||
def reconcile(self, *, limit: int = 500, apply_repairs: bool = True) -> KnowledgeFSOrphanReport:
|
||||
with self._session_maker() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
local_spaces = repository.list_for_reconciliation(limit=limit)
|
||||
control_space_by_tenant: dict[str, str] = {}
|
||||
for space in local_spaces:
|
||||
control_space_by_tenant.setdefault(space.tenant_id, space.id)
|
||||
local_keys = {
|
||||
space.provisioning_key
|
||||
for tenant_id in control_space_by_tenant
|
||||
for space in repository.list_for_tenant(tenant_id=tenant_id)
|
||||
if space.state is not KnowledgeFSControlSpaceState.DELETED
|
||||
}
|
||||
remote_spaces = tuple(
|
||||
space
|
||||
for tenant_id, control_space_id in sorted(control_space_by_tenant.items())
|
||||
for space in self._remote.list_spaces(
|
||||
namespace_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
)
|
||||
remote_by_key = {space.provisioning_key: space for space in remote_spaces}
|
||||
repaired: list[str] = []
|
||||
local_missing: list[str] = []
|
||||
cleanup: list[str] = []
|
||||
|
||||
for snapshot in local_spaces:
|
||||
remote_space = remote_by_key.get(snapshot.provisioning_key)
|
||||
if apply_repairs and (
|
||||
remote_space is not None
|
||||
and remote_space.namespace_id == snapshot.tenant_id
|
||||
and snapshot.state is KnowledgeFSControlSpaceState.PROVISIONING
|
||||
and snapshot.deletion_irreversible_at is None
|
||||
):
|
||||
with self._session_maker.begin() as session:
|
||||
repository = SQLAlchemyKnowledgeFSControlSpaceRepository(session)
|
||||
current = repository.get(tenant_id=snapshot.tenant_id, control_space_id=snapshot.id)
|
||||
if current is not None and current.resource_version == snapshot.resource_version:
|
||||
KnowledgeFSControlSpaceLifecycleService(repository).transition(
|
||||
tenant_id=snapshot.tenant_id,
|
||||
control_space_id=snapshot.id,
|
||||
expected_resource_version=snapshot.resource_version,
|
||||
new_state=KnowledgeFSControlSpaceState.ACTIVE,
|
||||
lifecycle_operation_id=snapshot.lifecycle_operation_id or "orphan-reconcile",
|
||||
knowledge_space_id=remote_space.knowledge_space_id,
|
||||
knowledge_space_revision=remote_space.revision,
|
||||
)
|
||||
repaired.append(snapshot.id)
|
||||
continue
|
||||
if apply_repairs and snapshot.state is KnowledgeFSControlSpaceState.DELETING:
|
||||
operation_id = str(
|
||||
uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
f"dify-kfs-orphan-cleanup:{snapshot.tenant_id}:{snapshot.id}:{snapshot.resource_version}",
|
||||
)
|
||||
)
|
||||
result = KnowledgeFSControlSpaceCommandService(self._session_maker).request_deletion(
|
||||
tenant_id=snapshot.tenant_id,
|
||||
control_space_id=snapshot.id,
|
||||
operation_id=operation_id,
|
||||
idempotency_key=f"orphan-cleanup:{snapshot.tenant_id}:{snapshot.id}",
|
||||
)
|
||||
if result.outbox is not None:
|
||||
cleanup.append(snapshot.id)
|
||||
continue
|
||||
if snapshot.knowledge_space_id is not None and remote_space is None:
|
||||
local_missing.append(snapshot.id)
|
||||
|
||||
remote_orphans = tuple(
|
||||
sorted(space.knowledge_space_id for space in remote_spaces if space.provisioning_key not in local_keys)
|
||||
)
|
||||
return KnowledgeFSOrphanReport(
|
||||
repaired_control_space_ids=tuple(sorted(repaired)),
|
||||
remote_orphan_space_ids=remote_orphans,
|
||||
local_missing_remote_ids=tuple(sorted(local_missing)),
|
||||
cleanup_control_space_ids=tuple(sorted(cleanup)),
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSOrphanReconciler", "KnowledgeFSOrphanReport"]
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Product-level lifecycle and metadata use cases for KnowledgeFS control-spaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSControlSpaceVisibility,
|
||||
KnowledgeFSModelSelectionIntentPayload,
|
||||
KnowledgeFSRerankIntentPayload,
|
||||
KnowledgeFSRetrievalProfileIntentPayload,
|
||||
KnowledgeFSScoreThresholdIntentPayload,
|
||||
)
|
||||
from services.knowledge_fs.control_plane_service import KnowledgeFSControlPlaneService
|
||||
from services.knowledge_fs.control_space_commands import (
|
||||
KnowledgeFSControlSpaceCommandService,
|
||||
KnowledgeFSProvisionIntent,
|
||||
)
|
||||
from services.knowledge_fs.data_facade import KnowledgeFSDataFacade
|
||||
from services.knowledge_fs.product_authorization import KnowledgeFSProductRBACPort
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSModelIntent,
|
||||
KnowledgeFSRetrievalProfileIntent,
|
||||
KnowledgeFSSpaceCreatePayload,
|
||||
KnowledgeFSSpaceCreateResponse,
|
||||
KnowledgeFSSpaceDetailResponse,
|
||||
KnowledgeFSSpaceListResponse,
|
||||
KnowledgeFSSpaceUpdatePayload,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
|
||||
from services.knowledge_fs.product_service import KnowledgeFSProductService
|
||||
|
||||
|
||||
class KnowledgeFSProductApplicationService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
product: KnowledgeFSProductService,
|
||||
control_plane: KnowledgeFSControlPlaneService,
|
||||
commands: KnowledgeFSControlSpaceCommandService,
|
||||
facade: KnowledgeFSDataFacade,
|
||||
rbac: KnowledgeFSProductRBACPort,
|
||||
) -> None:
|
||||
self._product = product
|
||||
self._control_plane = control_plane
|
||||
self._commands = commands
|
||||
self._facade = facade
|
||||
self._rbac = rbac
|
||||
|
||||
def list_spaces(self, *, tenant_id: str, account_id: str, page: int, limit: int) -> KnowledgeFSSpaceListResponse:
|
||||
return self._product.list_spaces(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
page=page,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
def create_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
payload: KnowledgeFSSpaceCreatePayload,
|
||||
) -> KnowledgeFSSpaceCreateResponse:
|
||||
self._product.require_product_routes(tenant_id=tenant_id)
|
||||
if not self._rbac.workspace_permission_allowed(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
permission=KnowledgeFSProductPermission.CREATE,
|
||||
):
|
||||
raise PermissionError("KnowledgeFS space creation is not allowed")
|
||||
idempotency_key = payload.idempotency_key or str(uuid.uuid4())
|
||||
operation_id = str(
|
||||
uuid.uuid5(uuid.NAMESPACE_URL, f"dify-kfs-provision:{tenant_id}:{account_id}:{idempotency_key}")
|
||||
)
|
||||
result = self._commands.create_provision_intent(
|
||||
KnowledgeFSProvisionIntent(
|
||||
tenant_id=tenant_id,
|
||||
owner_account_id=account_id,
|
||||
provisioning_key=f"dify:{tenant_id}:{payload.slug}",
|
||||
operation_id=operation_id,
|
||||
idempotency_key=idempotency_key,
|
||||
name=payload.name,
|
||||
slug=payload.slug,
|
||||
icon=payload.icon,
|
||||
description=payload.description,
|
||||
model_intent=_model_intent(payload.embedding),
|
||||
profile_intent=_retrieval_profile_intent(payload.retrieval),
|
||||
)
|
||||
)
|
||||
if payload.visibility is not KnowledgeFSControlSpaceVisibility.ONLY_ME:
|
||||
self._control_plane.update_visibility(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=account_id,
|
||||
control_space_id=result.control_space.id,
|
||||
visibility=payload.visibility,
|
||||
)
|
||||
return KnowledgeFSSpaceCreateResponse(
|
||||
control_space_id=result.control_space.id,
|
||||
state=result.control_space.state,
|
||||
operation_id=operation_id,
|
||||
)
|
||||
|
||||
def get_space(self, *, tenant_id: str, account_id: str, control_space_id: str) -> KnowledgeFSSpaceDetailResponse:
|
||||
return self._product.get_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
|
||||
def update_space(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_id: str,
|
||||
payload: KnowledgeFSSpaceUpdatePayload,
|
||||
) -> KnowledgeFSSpaceDetailResponse:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.EDIT,
|
||||
)
|
||||
if payload.visibility is not None:
|
||||
self._control_plane.update_visibility(
|
||||
tenant_id=tenant_id,
|
||||
actor_account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
visibility=payload.visibility,
|
||||
)
|
||||
metadata = payload.model_copy(update={"visibility": None})
|
||||
if any(value is not None for value in (metadata.name, metadata.icon, metadata.description)):
|
||||
self._facade.update_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
payload=metadata,
|
||||
)
|
||||
return self.get_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
)
|
||||
|
||||
def delete_space(self, *, tenant_id: str, account_id: str, control_space_id: str) -> None:
|
||||
self._product.authorize_control_space(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_id=control_space_id,
|
||||
permission=KnowledgeFSProductPermission.DELETE,
|
||||
)
|
||||
operation_id = str(uuid.uuid4())
|
||||
self._commands.request_deletion(
|
||||
tenant_id=tenant_id,
|
||||
control_space_id=control_space_id,
|
||||
operation_id=operation_id,
|
||||
idempotency_key=f"delete:{operation_id}",
|
||||
)
|
||||
|
||||
|
||||
def _model_intent(model: KnowledgeFSModelIntent) -> KnowledgeFSModelSelectionIntentPayload:
|
||||
return {
|
||||
"pluginId": model.plugin_id,
|
||||
"provider": model.provider,
|
||||
"model": model.model,
|
||||
}
|
||||
|
||||
|
||||
def _retrieval_profile_intent(
|
||||
profile: KnowledgeFSRetrievalProfileIntent,
|
||||
) -> KnowledgeFSRetrievalProfileIntentPayload:
|
||||
rerank: KnowledgeFSRerankIntentPayload = {"enabled": profile.rerank.enabled}
|
||||
if profile.rerank.model is not None:
|
||||
rerank["model"] = _model_intent(profile.rerank.model)
|
||||
score_threshold: KnowledgeFSScoreThresholdIntentPayload = {
|
||||
"enabled": profile.score_threshold.enabled,
|
||||
"stage": profile.score_threshold.stage,
|
||||
}
|
||||
if profile.score_threshold.value is not None:
|
||||
score_threshold["value"] = profile.score_threshold.value
|
||||
return {
|
||||
"defaultMode": profile.default_mode,
|
||||
"reasoningModel": _model_intent(profile.reasoning_model),
|
||||
"rerank": rerank,
|
||||
"scoreThreshold": score_threshold,
|
||||
"topK": profile.top_k,
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["KnowledgeFSProductApplicationService"]
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Dify-owned visibility, role, and enterprise RBAC authorization for KnowledgeFS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from models.knowledge_fs import (
|
||||
KnowledgeFSControlSpace,
|
||||
KnowledgeFSControlSpacePermission,
|
||||
KnowledgeFSControlSpacePermissionRole,
|
||||
KnowledgeFSControlSpacePermissionStatus,
|
||||
KnowledgeFSControlSpaceState,
|
||||
KnowledgeFSControlSpaceVisibility,
|
||||
)
|
||||
from services.enterprise.rbac_service import RBACService
|
||||
from services.knowledge_fs.product_operations import KnowledgeFSProductPermission
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KnowledgeFSProductNotFoundError(RuntimeError):
|
||||
"""A missing and an unauthorized control-space deliberately share one error."""
|
||||
|
||||
|
||||
class KnowledgeFSProductRBACPort(Protocol):
|
||||
def permission_keys_by_control_space(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_ids: Sequence[str],
|
||||
) -> dict[str, frozenset[KnowledgeFSProductPermission]]: ...
|
||||
|
||||
def filter_authorized_control_space_ids(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_ids: Sequence[str],
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> frozenset[str]: ...
|
||||
|
||||
def workspace_permission_allowed(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class DifyKnowledgeFSProductRBACPort:
|
||||
"""Use one Enterprise batch request for a page; fail closed if it is unavailable."""
|
||||
|
||||
def permission_keys_by_control_space(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_ids: Sequence[str],
|
||||
) -> dict[str, frozenset[KnowledgeFSProductPermission]]:
|
||||
if not control_space_ids:
|
||||
return {}
|
||||
try:
|
||||
permissions = RBACService.KnowledgeFSPermissions.batch_get(
|
||||
tenant_id,
|
||||
account_id,
|
||||
list(control_space_ids),
|
||||
session=session,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("KnowledgeFS enterprise RBAC batch lookup failed for tenant_id=%s", tenant_id, exc_info=True)
|
||||
return {}
|
||||
return {
|
||||
control_space_id: frozenset(
|
||||
permission
|
||||
for permission in KnowledgeFSProductPermission
|
||||
if permission.value in permissions.get(control_space_id, ())
|
||||
)
|
||||
for control_space_id in control_space_ids
|
||||
}
|
||||
|
||||
def filter_authorized_control_space_ids(
|
||||
self,
|
||||
*,
|
||||
session: Session,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
control_space_ids: Sequence[str],
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> frozenset[str]:
|
||||
permissions = self.permission_keys_by_control_space(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
control_space_ids=control_space_ids,
|
||||
)
|
||||
return frozenset(
|
||||
control_space_id
|
||||
for control_space_id, permission_keys in permissions.items()
|
||||
if permission in permission_keys
|
||||
)
|
||||
|
||||
def workspace_permission_allowed(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
account_id: str,
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> bool:
|
||||
if not dify_config.RBAC_ENABLED:
|
||||
return True
|
||||
return RBACService.CheckAccess.check(
|
||||
tenant_id,
|
||||
account_id,
|
||||
scene=permission.value,
|
||||
resource_type=None,
|
||||
resource_id=None,
|
||||
)
|
||||
|
||||
|
||||
_ROLE_PERMISSIONS: dict[KnowledgeFSControlSpacePermissionRole, frozenset[KnowledgeFSProductPermission]] = {
|
||||
KnowledgeFSControlSpacePermissionRole.OWNER: frozenset(KnowledgeFSProductPermission),
|
||||
KnowledgeFSControlSpacePermissionRole.EDITOR: frozenset(
|
||||
{
|
||||
KnowledgeFSProductPermission.READ,
|
||||
KnowledgeFSProductPermission.EDIT,
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
}
|
||||
),
|
||||
KnowledgeFSControlSpacePermissionRole.VIEWER: frozenset(
|
||||
{KnowledgeFSProductPermission.READ, KnowledgeFSProductPermission.QUERY}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def visible_control_space_statement(*, tenant_id: str, account_id: str) -> sa.Select[tuple[KnowledgeFSControlSpace]]:
|
||||
"""Build the SQL visibility filter before pagination or any KFS request."""
|
||||
|
||||
active_permission = sa.exists().where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == KnowledgeFSControlSpace.tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == KnowledgeFSControlSpace.id,
|
||||
KnowledgeFSControlSpacePermission.account_id == account_id,
|
||||
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
)
|
||||
return sa.select(KnowledgeFSControlSpace).where(
|
||||
KnowledgeFSControlSpace.tenant_id == tenant_id,
|
||||
KnowledgeFSControlSpace.state != KnowledgeFSControlSpaceState.DELETED,
|
||||
sa.or_(
|
||||
KnowledgeFSControlSpace.owner_account_id == account_id,
|
||||
KnowledgeFSControlSpace.visibility == KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS,
|
||||
sa.and_(
|
||||
KnowledgeFSControlSpace.visibility == KnowledgeFSControlSpaceVisibility.PARTIAL_MEMBERS,
|
||||
active_permission,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def resolve_local_role(
|
||||
session: Session,
|
||||
*,
|
||||
control_space: KnowledgeFSControlSpace,
|
||||
account_id: str,
|
||||
) -> KnowledgeFSControlSpacePermissionRole | None:
|
||||
if control_space.owner_account_id == account_id:
|
||||
return KnowledgeFSControlSpacePermissionRole.OWNER
|
||||
permission = session.scalar(
|
||||
sa.select(KnowledgeFSControlSpacePermission).where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id == control_space.tenant_id,
|
||||
KnowledgeFSControlSpacePermission.control_space_id == control_space.id,
|
||||
KnowledgeFSControlSpacePermission.account_id == account_id,
|
||||
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
if permission is not None:
|
||||
return permission.role
|
||||
if control_space.visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS:
|
||||
return KnowledgeFSControlSpacePermissionRole.VIEWER
|
||||
return None
|
||||
|
||||
|
||||
def resolve_local_roles(
|
||||
session: Session,
|
||||
*,
|
||||
control_spaces: Sequence[KnowledgeFSControlSpace],
|
||||
account_id: str,
|
||||
) -> dict[str, KnowledgeFSControlSpacePermissionRole | None]:
|
||||
if not control_spaces:
|
||||
return {}
|
||||
control_space_ids = tuple(control_space.id for control_space in control_spaces)
|
||||
tenant_ids = tuple({control_space.tenant_id for control_space in control_spaces})
|
||||
explicit_roles = {
|
||||
(permission.tenant_id, permission.control_space_id): permission.role
|
||||
for permission in session.scalars(
|
||||
sa.select(KnowledgeFSControlSpacePermission).where(
|
||||
KnowledgeFSControlSpacePermission.tenant_id.in_(tenant_ids),
|
||||
KnowledgeFSControlSpacePermission.control_space_id.in_(control_space_ids),
|
||||
KnowledgeFSControlSpacePermission.account_id == account_id,
|
||||
KnowledgeFSControlSpacePermission.status == KnowledgeFSControlSpacePermissionStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
}
|
||||
return {
|
||||
control_space.id: (
|
||||
KnowledgeFSControlSpacePermissionRole.OWNER
|
||||
if control_space.owner_account_id == account_id
|
||||
else explicit_roles.get((control_space.tenant_id, control_space.id))
|
||||
or (
|
||||
KnowledgeFSControlSpacePermissionRole.VIEWER
|
||||
if control_space.visibility is KnowledgeFSControlSpaceVisibility.ALL_TEAM_MEMBERS
|
||||
else None
|
||||
)
|
||||
)
|
||||
for control_space in control_spaces
|
||||
}
|
||||
|
||||
|
||||
def local_role_allows(
|
||||
role: KnowledgeFSControlSpacePermissionRole | None,
|
||||
permission: KnowledgeFSProductPermission,
|
||||
) -> bool:
|
||||
return role is not None and permission in _ROLE_PERMISSIONS[role]
|
||||
|
||||
|
||||
def effective_product_permissions(
|
||||
role: KnowledgeFSControlSpacePermissionRole | None,
|
||||
rbac_permissions: frozenset[KnowledgeFSProductPermission],
|
||||
) -> tuple[KnowledgeFSProductPermission, ...]:
|
||||
if role is None:
|
||||
return ()
|
||||
return tuple(
|
||||
permission
|
||||
for permission in KnowledgeFSProductPermission
|
||||
if permission is not KnowledgeFSProductPermission.CREATE
|
||||
and permission in _ROLE_PERMISSIONS[role]
|
||||
and permission in rbac_permissions
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DifyKnowledgeFSProductRBACPort",
|
||||
"KnowledgeFSProductNotFoundError",
|
||||
"KnowledgeFSProductRBACPort",
|
||||
"effective_product_permissions",
|
||||
"local_role_allows",
|
||||
"resolve_local_role",
|
||||
"resolve_local_roles",
|
||||
"visible_control_space_statement",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,759 @@
|
||||
"""Typed Dify BFF manifest and explicit gaps against the Capability v2 registry.
|
||||
|
||||
An operation is executable only when ``capability_operation_id`` resolves to an
|
||||
exact method/path/action/resource entry in the P2 registry. Declared operations
|
||||
without an executable transport remain explicit gaps and fail before token
|
||||
issuance or external I/O.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, NamedTuple
|
||||
|
||||
from services.knowledge_fs_capability import KNOWLEDGE_FS_CAPABILITY_OPERATIONS
|
||||
|
||||
|
||||
class KnowledgeFSProductPermission(StrEnum):
|
||||
READ = "knowledge_space_read"
|
||||
CREATE = "knowledge_space_create"
|
||||
EDIT = "knowledge_space_edit"
|
||||
DELETE = "knowledge_space_delete"
|
||||
ACCESS_CONFIG = "knowledge_space_access_config"
|
||||
API_KEY_MANAGE = "knowledge_space_api_key_manage"
|
||||
DOCUMENT_WRITE = "knowledge_space_document_write"
|
||||
QUERY = "knowledge_space_query"
|
||||
|
||||
|
||||
class KnowledgeFSProductOperation(NamedTuple):
|
||||
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
|
||||
capability_operation_id: str | None
|
||||
permission: KnowledgeFSProductPermission
|
||||
kfs_path: str | None
|
||||
transport: Literal["binary", "direct", "json", "multipart", "sse", "unavailable"]
|
||||
resource_resolver: Literal[
|
||||
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
|
||||
]
|
||||
rbac_permission: KnowledgeFSProductPermission
|
||||
billing_cost: int
|
||||
max_request_bytes: int
|
||||
max_response_bytes: int
|
||||
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"]
|
||||
rate_limit_bucket: Literal["direct", "import", "job", "query", "read", "write"]
|
||||
rate_limit_cost: int
|
||||
|
||||
@property
|
||||
def action(self) -> str | None:
|
||||
if self.capability_operation_id is None:
|
||||
return None
|
||||
capability = KNOWLEDGE_FS_CAPABILITY_OPERATIONS.get(self.capability_operation_id)
|
||||
return capability.action if capability is not None else None
|
||||
|
||||
|
||||
def _operation(
|
||||
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"],
|
||||
capability_operation_id: str | None,
|
||||
permission: KnowledgeFSProductPermission,
|
||||
kfs_path: str | None,
|
||||
transport: Literal["binary", "direct", "json", "multipart", "sse", "unavailable"],
|
||||
*,
|
||||
resource_resolver: Literal[
|
||||
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
|
||||
],
|
||||
billing_cost: int,
|
||||
max_request_bytes: int,
|
||||
max_response_bytes: int,
|
||||
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"],
|
||||
rate_limit_bucket: Literal["direct", "import", "job", "query", "read", "write"] | None = None,
|
||||
rate_limit_cost: int | None = None,
|
||||
) -> KnowledgeFSProductOperation:
|
||||
resolved_rate_limit_bucket = rate_limit_bucket or _default_rate_limit_bucket(
|
||||
method=method,
|
||||
permission=permission,
|
||||
resource_resolver=resource_resolver,
|
||||
stream_kind=stream_kind,
|
||||
)
|
||||
return KnowledgeFSProductOperation(
|
||||
method=method,
|
||||
capability_operation_id=capability_operation_id,
|
||||
permission=permission,
|
||||
kfs_path=kfs_path,
|
||||
transport=transport,
|
||||
resource_resolver=resource_resolver,
|
||||
rbac_permission=permission,
|
||||
billing_cost=billing_cost,
|
||||
max_request_bytes=max_request_bytes,
|
||||
max_response_bytes=max_response_bytes,
|
||||
stream_kind=stream_kind,
|
||||
rate_limit_bucket=resolved_rate_limit_bucket,
|
||||
rate_limit_cost=rate_limit_cost if rate_limit_cost is not None else billing_cost,
|
||||
)
|
||||
|
||||
|
||||
def _default_rate_limit_bucket(
|
||||
*,
|
||||
method: Literal["DELETE", "GET", "PATCH", "POST", "PUT"],
|
||||
permission: KnowledgeFSProductPermission,
|
||||
resource_resolver: Literal[
|
||||
"document", "job", "knowledge_space", "namespace", "query", "research_task", "source", "upload_session"
|
||||
],
|
||||
stream_kind: Literal["buffered-multipart", "direct-upload", "json", "sse"],
|
||||
) -> Literal["direct", "import", "job", "query", "read", "write"]:
|
||||
if stream_kind in {"direct-upload", "sse"}:
|
||||
return "direct"
|
||||
if permission == KnowledgeFSProductPermission.QUERY:
|
||||
return "query"
|
||||
if resource_resolver == "job":
|
||||
return "job"
|
||||
if method == "GET":
|
||||
return "read"
|
||||
return "write"
|
||||
|
||||
|
||||
KNOWLEDGE_FS_PRODUCT_OPERATIONS: Final[MappingProxyType[str, KnowledgeFSProductOperation]] = MappingProxyType(
|
||||
{
|
||||
"batchSpaceSummaries": _operation(
|
||||
"POST",
|
||||
"batchKnowledgeSpaceProductSummaries",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/internal/knowledge-spaces/product-summaries/batch",
|
||||
"json",
|
||||
resource_resolver="namespace",
|
||||
billing_cost=2,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getSpace": _operation(
|
||||
"GET",
|
||||
"getKnowledgeSpace",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=1,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=256 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"updateSpace": _operation(
|
||||
"PATCH",
|
||||
"updateKnowledgeSpace",
|
||||
KnowledgeFSProductPermission.EDIT,
|
||||
"/knowledge-spaces/{id}",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=3,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=256 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getSettings": _operation(
|
||||
"GET",
|
||||
"getKnowledgeSpaceProductSettings",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/product-settings",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=1,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=256 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"updateSettings": _operation(
|
||||
"PATCH",
|
||||
"updateKnowledgeSpaceProductSettings",
|
||||
KnowledgeFSProductPermission.EDIT,
|
||||
"/knowledge-spaces/{id}/product-settings",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=5,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=256 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listDocuments": _operation(
|
||||
"GET",
|
||||
"listDocuments",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=2,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"createDocument": _operation(
|
||||
"POST",
|
||||
"uploadDocument",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/documents",
|
||||
"multipart",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=10,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=0,
|
||||
stream_kind="buffered-multipart",
|
||||
),
|
||||
"getDocument": _operation(
|
||||
"GET",
|
||||
"getDocument",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=2,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getDocumentOutline": _operation(
|
||||
"GET",
|
||||
"getDocumentOutline",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}/outline",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=3,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listDocumentRevisions": _operation(
|
||||
"GET",
|
||||
"listDocumentRevisions",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}/revisions",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=2,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"updateDocumentMetadata": _operation(
|
||||
"PATCH",
|
||||
"patchDocumentMetadata",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}/metadata",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=4,
|
||||
max_request_bytes=128 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listDocumentChunks": _operation(
|
||||
"GET",
|
||||
"listDocumentChunks",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getDocumentChunk": _operation(
|
||||
"GET",
|
||||
"getDocumentChunk",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=2,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"deleteDocument": _operation(
|
||||
"DELETE",
|
||||
"requestDocumentDeletion",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/documents/{documentId}",
|
||||
"json",
|
||||
resource_resolver="document",
|
||||
billing_cost=8,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"bulkDeleteDocuments": _operation(
|
||||
"DELETE",
|
||||
"requestBulkDocumentDeletion",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/documents/bulk",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=20,
|
||||
max_request_bytes=1024 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"reindexDocuments": _operation(
|
||||
"POST",
|
||||
"bulkReindexDocuments",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/documents/bulk/reindex",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=20,
|
||||
max_request_bytes=1024 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
rate_limit_bucket="import",
|
||||
),
|
||||
"getCompilationJob": _operation(
|
||||
"GET",
|
||||
"getDocumentCompilationJob",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/jobs/{id}",
|
||||
"json",
|
||||
resource_resolver="job",
|
||||
billing_cost=1,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"cancelCompilationJob": _operation(
|
||||
"DELETE",
|
||||
"cancelDocumentCompilationJob",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/jobs/{id}",
|
||||
"json",
|
||||
resource_resolver="job",
|
||||
billing_cost=5,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"retryCompilationJob": _operation(
|
||||
"POST",
|
||||
"retryDocumentCompilationJob",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/jobs/{id}/retry",
|
||||
"json",
|
||||
resource_resolver="job",
|
||||
billing_cost=8,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getBulkJob": _operation(
|
||||
"GET",
|
||||
"getBulkOperation",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/bulk-jobs/{id}",
|
||||
"json",
|
||||
resource_resolver="job",
|
||||
billing_cost=1,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listSources": _operation(
|
||||
"GET",
|
||||
"listKnowledgeSpaceSources",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/sources",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=2,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"createSource": _operation(
|
||||
"POST",
|
||||
"createKnowledgeSpaceSource",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=8,
|
||||
max_request_bytes=256 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getSource": _operation(
|
||||
"GET",
|
||||
"getKnowledgeSpaceSource",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=1,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"updateSource": _operation(
|
||||
"PATCH",
|
||||
"updateKnowledgeSpaceSource",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=4,
|
||||
max_request_bytes=256 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"deleteSource": _operation(
|
||||
"DELETE",
|
||||
"requestSourceDeletion",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=8,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"testSource": _operation(
|
||||
"POST",
|
||||
"testKnowledgeSpaceSource",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/test",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=3,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=256 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"crawlSource": _operation(
|
||||
"POST",
|
||||
"crawlKnowledgeSpaceSource",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/crawl",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=20,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=8 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
rate_limit_bucket="import",
|
||||
),
|
||||
"listSourcePages": _operation(
|
||||
"GET",
|
||||
"listKnowledgeSpaceSourcePages",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/pages",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"importSourcePages": _operation(
|
||||
"POST",
|
||||
"importKnowledgeSpaceSourcePages",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/import",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=25,
|
||||
max_request_bytes=1024 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
rate_limit_bucket="import",
|
||||
),
|
||||
"listSourceFiles": _operation(
|
||||
"GET",
|
||||
"listKnowledgeSpaceSourceFiles",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/files",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=3,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"importSourceFiles": _operation(
|
||||
"POST",
|
||||
"importKnowledgeSpaceSourceFiles",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/sources/{sourceId}/import-files",
|
||||
"json",
|
||||
resource_resolver="source",
|
||||
billing_cost=25,
|
||||
max_request_bytes=1024 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
rate_limit_bucket="import",
|
||||
),
|
||||
"createQuery": _operation(
|
||||
"POST",
|
||||
"createQuery",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/queries",
|
||||
"direct",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=20,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=0,
|
||||
stream_kind="sse",
|
||||
),
|
||||
"listResearchTasks": _operation(
|
||||
"GET",
|
||||
"listKnowledgeSpaceResearchTasks",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/research-tasks",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=2,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"createResearchTask": _operation(
|
||||
"POST",
|
||||
"createResearchTask",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/research-tasks",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=25,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"planResearchTask": _operation(
|
||||
"POST",
|
||||
"planResearchTask",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/research-tasks/plan",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=8,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getResearchTask": _operation(
|
||||
"GET",
|
||||
"getResearchTask",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/research-tasks/{id}",
|
||||
"json",
|
||||
resource_resolver="research_task",
|
||||
billing_cost=1,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listResearchTaskPartials": _operation(
|
||||
"GET",
|
||||
"listResearchTaskPartials",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/research-tasks/{id}/partials",
|
||||
"json",
|
||||
resource_resolver="research_task",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=8 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"cancelResearchTask": _operation(
|
||||
"DELETE",
|
||||
"cancelResearchTask",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/research-tasks/{id}",
|
||||
"json",
|
||||
resource_resolver="research_task",
|
||||
billing_cost=5,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=512 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listTraces": _operation(
|
||||
"GET",
|
||||
"listKnowledgeSpaceQualityTraces",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/knowledge-spaces/{id}/quality/traces",
|
||||
"json",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"getTrace": _operation(
|
||||
"GET",
|
||||
"getAnswerTrace",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/queries/{traceId}",
|
||||
"json",
|
||||
resource_resolver="query",
|
||||
billing_cost=2,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=2 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listTraceEvidence": _operation(
|
||||
"GET",
|
||||
"listQueryEvidence",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/queries/{traceId}/evidence",
|
||||
"json",
|
||||
resource_resolver="query",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listTraceConflicts": _operation(
|
||||
"GET",
|
||||
"listQueryConflicts",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/queries/{traceId}/conflicts",
|
||||
"json",
|
||||
resource_resolver="query",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"listTraceMissing": _operation(
|
||||
"GET",
|
||||
"listQueryMissing",
|
||||
KnowledgeFSProductPermission.QUERY,
|
||||
"/queries/{traceId}/missing",
|
||||
"json",
|
||||
resource_resolver="query",
|
||||
billing_cost=3,
|
||||
max_request_bytes=16 * 1024,
|
||||
max_response_bytes=4 * 1024 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"createUploadSession": _operation(
|
||||
"POST",
|
||||
"createUploadSession",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/knowledge-spaces/{id}/upload-sessions",
|
||||
"direct",
|
||||
resource_resolver="knowledge_space",
|
||||
billing_cost=5,
|
||||
max_request_bytes=64 * 1024,
|
||||
max_response_bytes=64 * 1024,
|
||||
stream_kind="direct-upload",
|
||||
),
|
||||
"presignUploadSessionPart": _operation(
|
||||
"POST",
|
||||
"presignUploadSessionPart",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/upload-sessions/{id}/parts/{partNumber}/presign",
|
||||
"direct",
|
||||
resource_resolver="upload_session",
|
||||
billing_cost=1,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=64 * 1024,
|
||||
stream_kind="direct-upload",
|
||||
),
|
||||
"uploadSmallFile": _operation(
|
||||
"POST",
|
||||
"uploadSmallFile",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/upload-sessions/{id}/small-file",
|
||||
"binary",
|
||||
resource_resolver="upload_session",
|
||||
billing_cost=8,
|
||||
max_request_bytes=8 * 1024 * 1024,
|
||||
max_response_bytes=128 * 1024,
|
||||
stream_kind="json",
|
||||
),
|
||||
"completeUploadSession": _operation(
|
||||
"POST",
|
||||
"completeUploadSession",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/upload-sessions/{id}/complete",
|
||||
"direct",
|
||||
resource_resolver="upload_session",
|
||||
billing_cost=8,
|
||||
max_request_bytes=128 * 1024,
|
||||
max_response_bytes=128 * 1024,
|
||||
stream_kind="direct-upload",
|
||||
),
|
||||
"abortUploadSession": _operation(
|
||||
"POST",
|
||||
"abortUploadSession",
|
||||
KnowledgeFSProductPermission.DOCUMENT_WRITE,
|
||||
"/upload-sessions/{id}/abort",
|
||||
"direct",
|
||||
resource_resolver="upload_session",
|
||||
billing_cost=1,
|
||||
max_request_bytes=32 * 1024,
|
||||
max_response_bytes=64 * 1024,
|
||||
stream_kind="direct-upload",
|
||||
),
|
||||
"streamResearchTask": _operation(
|
||||
"GET",
|
||||
"streamResearchTaskProgress",
|
||||
KnowledgeFSProductPermission.READ,
|
||||
"/research-tasks/{id}/events",
|
||||
"direct",
|
||||
resource_resolver="research_task",
|
||||
billing_cost=5,
|
||||
max_request_bytes=0,
|
||||
max_response_bytes=0,
|
||||
stream_kind="sse",
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def is_product_operation_registered(operation_id: str) -> bool:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
|
||||
if operation is None or operation.capability_operation_id is None or operation.kfs_path is None:
|
||||
return False
|
||||
capability = KNOWLEDGE_FS_CAPABILITY_OPERATIONS.get(operation.capability_operation_id)
|
||||
return bool(
|
||||
capability
|
||||
and capability.method == operation.method
|
||||
and capability.path == operation.kfs_path
|
||||
and capability.action
|
||||
and capability.resource_type == operation.resource_resolver
|
||||
and operation.permission == operation.rbac_permission
|
||||
)
|
||||
|
||||
|
||||
def is_product_operation_ready(operation_id: str) -> bool:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(operation_id)
|
||||
return bool(
|
||||
operation
|
||||
and operation.transport in {"binary", "direct", "json"}
|
||||
and is_product_operation_registered(operation_id)
|
||||
)
|
||||
|
||||
|
||||
def knowledge_fs_product_operation_gaps() -> tuple[str, ...]:
|
||||
return tuple(
|
||||
operation_id for operation_id in KNOWLEDGE_FS_PRODUCT_OPERATIONS if not is_product_operation_ready(operation_id)
|
||||
)
|
||||
|
||||
|
||||
def product_operation_action(operation_id: str) -> str:
|
||||
if not is_product_operation_ready(operation_id):
|
||||
raise KeyError(operation_id)
|
||||
capability_operation_id = KNOWLEDGE_FS_PRODUCT_OPERATIONS[operation_id].capability_operation_id
|
||||
if capability_operation_id is None:
|
||||
raise KeyError(operation_id)
|
||||
return KNOWLEDGE_FS_CAPABILITY_OPERATIONS[capability_operation_id].action
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KNOWLEDGE_FS_PRODUCT_OPERATIONS",
|
||||
"KnowledgeFSProductOperation",
|
||||
"KnowledgeFSProductPermission",
|
||||
"is_product_operation_ready",
|
||||
"is_product_operation_registered",
|
||||
"knowledge_fs_product_operation_gaps",
|
||||
"product_operation_action",
|
||||
]
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Ports for capability-authenticated, manifest-bounded KnowledgeFS BFF calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal, NamedTuple, Protocol
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from services.knowledge_fs.product_dto import KnowledgeFSTechnicalSummary
|
||||
|
||||
|
||||
class KnowledgeFSProductRemoteError(RuntimeError):
|
||||
"""KnowledgeFS could not provide an authoritative product response."""
|
||||
|
||||
|
||||
class KnowledgeFSOperationUnavailableError(RuntimeError):
|
||||
"""The Dify/KFS/Capability operation manifests are not yet aligned."""
|
||||
|
||||
|
||||
class KnowledgeFSProductRequestRejectedError(RuntimeError):
|
||||
"""A bounded product request was rejected locally or by authoritative KFS validation."""
|
||||
|
||||
def __init__(self, *, status_code: Literal[409, 413, 422]) -> None:
|
||||
super().__init__(f"KnowledgeFS rejected the product request with HTTP {status_code}")
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
class KnowledgeFSRemoteJSONRequest(NamedTuple):
|
||||
operation_id: str
|
||||
method: str
|
||||
path: str
|
||||
namespace_id: str
|
||||
knowledge_space_id: str
|
||||
capability_token: str
|
||||
trace_id: str
|
||||
payload: JsonValue | None
|
||||
query: tuple[tuple[str, str], ...] = ()
|
||||
headers: tuple[tuple[str, str], ...] = ()
|
||||
|
||||
|
||||
class KnowledgeFSRemoteBinaryRequest(NamedTuple):
|
||||
operation_id: str
|
||||
method: str
|
||||
path: str
|
||||
namespace_id: str
|
||||
knowledge_space_id: str
|
||||
capability_token: str
|
||||
trace_id: str
|
||||
body: bytes
|
||||
query: tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
class KnowledgeFSProductRemotePort(Protocol):
|
||||
def batch_space_summaries(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str,
|
||||
knowledge_space_ids: tuple[str, ...],
|
||||
capability_token: str,
|
||||
trace_id: str,
|
||||
) -> dict[str, KnowledgeFSTechnicalSummary]:
|
||||
"""Fetch exactly the explicit authorized Space IDs in one remote call."""
|
||||
|
||||
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
|
||||
"""Execute one manifest-approved JSON request using only its operation capability."""
|
||||
|
||||
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
|
||||
"""Execute one strictly bounded binary request using only its operation capability."""
|
||||
|
||||
|
||||
class UnavailableKnowledgeFSProductRemote:
|
||||
"""Fail-closed default until a manifest-aligned KFS transport is assembled."""
|
||||
|
||||
def batch_space_summaries(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str,
|
||||
knowledge_space_ids: tuple[str, ...],
|
||||
capability_token: str,
|
||||
trace_id: str,
|
||||
) -> dict[str, KnowledgeFSTechnicalSummary]:
|
||||
_ = (namespace_id, knowledge_space_ids, capability_token, trace_id)
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
|
||||
|
||||
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
|
||||
_ = request
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
|
||||
|
||||
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
|
||||
_ = request
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS product remote is not configured")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeFSOperationUnavailableError",
|
||||
"KnowledgeFSProductRemoteError",
|
||||
"KnowledgeFSProductRemotePort",
|
||||
"KnowledgeFSProductRequestRejectedError",
|
||||
"KnowledgeFSRemoteBinaryRequest",
|
||||
"KnowledgeFSRemoteJSONRequest",
|
||||
"UnavailableKnowledgeFSProductRemote",
|
||||
]
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Capability-only HTTP transport for manifest-approved JSON and bounded binary calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from core.helper import ssrf_proxy
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSBatchTechnicalSummaryResponse,
|
||||
KnowledgeFSTechnicalSummary,
|
||||
)
|
||||
from services.knowledge_fs.product_operations import KNOWLEDGE_FS_PRODUCT_OPERATIONS, is_product_operation_ready
|
||||
from services.knowledge_fs.product_remote import (
|
||||
KnowledgeFSOperationUnavailableError,
|
||||
KnowledgeFSProductRemoteError,
|
||||
KnowledgeFSProductRequestRejectedError,
|
||||
KnowledgeFSRemoteBinaryRequest,
|
||||
KnowledgeFSRemoteJSONRequest,
|
||||
)
|
||||
|
||||
_JSON_ADAPTER: TypeAdapter[JsonValue] = TypeAdapter(JsonValue)
|
||||
_MAX_BATCH_SUMMARIES = 100
|
||||
|
||||
|
||||
class HTTPKnowledgeFSProductRemoteClient:
|
||||
"""Build outbound headers from trusted capability input; browser headers never enter this class."""
|
||||
|
||||
def __init__(self, *, base_url: str, timeout_seconds: float, max_response_bytes: int = 4 * 1024 * 1024) -> None:
|
||||
self._base_url = base_url
|
||||
self._timeout_seconds = timeout_seconds
|
||||
self._max_response_bytes = max_response_bytes
|
||||
|
||||
def batch_space_summaries(
|
||||
self,
|
||||
*,
|
||||
namespace_id: str,
|
||||
knowledge_space_ids: tuple[str, ...],
|
||||
capability_token: str,
|
||||
trace_id: str,
|
||||
) -> dict[str, KnowledgeFSTechnicalSummary]:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS["batchSpaceSummaries"]
|
||||
if not is_product_operation_ready("batchSpaceSummaries") or operation.kfs_path is None:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS batch summary operation is unavailable")
|
||||
if (
|
||||
not namespace_id
|
||||
or not capability_token
|
||||
or not trace_id
|
||||
or not knowledge_space_ids
|
||||
or len(knowledge_space_ids) > _MAX_BATCH_SUMMARIES
|
||||
or any(not knowledge_space_id for knowledge_space_id in knowledge_space_ids)
|
||||
or len(set(knowledge_space_ids)) != len(knowledge_space_ids)
|
||||
):
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS batch request binding is invalid")
|
||||
payload = self._request_json(
|
||||
method=operation.method,
|
||||
path=operation.kfs_path,
|
||||
capability_token=capability_token,
|
||||
trace_id=trace_id,
|
||||
payload={"knowledgeSpaceIds": list(knowledge_space_ids)},
|
||||
query=(),
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
)
|
||||
try:
|
||||
response = KnowledgeFSBatchTechnicalSummaryResponse.model_validate(payload)
|
||||
except ValidationError as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an invalid batch summary") from exc
|
||||
requested_ids = frozenset(knowledge_space_ids)
|
||||
summaries: dict[str, KnowledgeFSTechnicalSummary] = {}
|
||||
for summary in response.items:
|
||||
if summary.knowledge_space_id not in requested_ids or summary.knowledge_space_id in summaries:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an out-of-scope batch summary")
|
||||
summaries[summary.knowledge_space_id] = summary
|
||||
return summaries
|
||||
|
||||
def execute_json(self, request: KnowledgeFSRemoteJSONRequest) -> JsonValue:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(request.operation_id)
|
||||
if operation is None or operation.transport != "json" or not is_product_operation_ready(request.operation_id):
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {request.operation_id}")
|
||||
if (
|
||||
request.method != operation.method
|
||||
or operation.kfs_path is None
|
||||
or not _matches_path(operation.kfs_path, request.path)
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request does not match its operation manifest")
|
||||
if (
|
||||
not request.namespace_id
|
||||
or not request.knowledge_space_id
|
||||
or not request.capability_token
|
||||
or not request.trace_id
|
||||
):
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request binding is incomplete")
|
||||
|
||||
return self._request_json(
|
||||
method=request.method,
|
||||
path=request.path,
|
||||
capability_token=request.capability_token,
|
||||
trace_id=request.trace_id,
|
||||
payload=request.payload,
|
||||
query=request.query,
|
||||
extra_headers=request.headers,
|
||||
max_request_bytes=operation.max_request_bytes,
|
||||
max_response_bytes=operation.max_response_bytes,
|
||||
)
|
||||
|
||||
def execute_binary(self, request: KnowledgeFSRemoteBinaryRequest) -> JsonValue:
|
||||
operation = KNOWLEDGE_FS_PRODUCT_OPERATIONS.get(request.operation_id)
|
||||
if operation is None or operation.transport != "binary" or not is_product_operation_ready(request.operation_id):
|
||||
raise KnowledgeFSOperationUnavailableError(f"KnowledgeFS operation is unavailable: {request.operation_id}")
|
||||
if (
|
||||
request.method != operation.method
|
||||
or operation.kfs_path is None
|
||||
or not _matches_path(operation.kfs_path, request.path)
|
||||
):
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS request does not match its operation manifest")
|
||||
if (
|
||||
not request.namespace_id
|
||||
or not request.knowledge_space_id
|
||||
or not request.capability_token
|
||||
or not request.trace_id
|
||||
or request.query != (("knowledgeSpaceId", request.knowledge_space_id),)
|
||||
):
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS binary request binding is incomplete")
|
||||
if not isinstance(request.body, bytes) or not request.body:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=422)
|
||||
if len(request.body) > operation.max_request_bytes:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=413)
|
||||
response_limit = min(self._max_response_bytes, operation.max_response_bytes)
|
||||
if response_limit <= 0:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS operation response limit is unavailable")
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Encoding": "identity",
|
||||
"Authorization": f"Bearer {request.capability_token}",
|
||||
"Content-Type": "application/octet-stream",
|
||||
"X-Trace-Id": request.trace_id,
|
||||
}
|
||||
try:
|
||||
upstream_url = httpx.URL(f"{self._base_url.rstrip('/')}/").join(request.path.lstrip("/"))
|
||||
response = ssrf_proxy.make_request(
|
||||
method=request.method,
|
||||
url=str(upstream_url),
|
||||
headers=headers,
|
||||
params=request.query,
|
||||
content=request.body,
|
||||
timeout=self._timeout_seconds,
|
||||
follow_redirects=False,
|
||||
max_retries=0,
|
||||
stream_response=True,
|
||||
)
|
||||
response = ssrf_proxy.buffer_response(response, max_response_bytes=response_limit)
|
||||
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request failed") from exc
|
||||
try:
|
||||
if response.status_code == 409:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=409)
|
||||
if response.status_code == 413:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=413)
|
||||
if response.status_code == 422:
|
||||
raise KnowledgeFSProductRequestRejectedError(status_code=422)
|
||||
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
|
||||
if content_type != "application/json" and not content_type.endswith("+json"):
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
|
||||
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
|
||||
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
|
||||
try:
|
||||
return _JSON_ADAPTER.validate_python(response.json())
|
||||
except (ValueError, ValidationError) as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned invalid JSON") from exc
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
def _request_json(
|
||||
self,
|
||||
*,
|
||||
method: str,
|
||||
path: str,
|
||||
capability_token: str,
|
||||
trace_id: str,
|
||||
payload: JsonValue | None,
|
||||
query: tuple[tuple[str, str], ...],
|
||||
extra_headers: tuple[tuple[str, str], ...] = (),
|
||||
max_request_bytes: int,
|
||||
max_response_bytes: int,
|
||||
) -> JsonValue:
|
||||
request_size = len(urlencode(query).encode("utf-8"))
|
||||
if payload is not None:
|
||||
try:
|
||||
request_size += len(
|
||||
json.dumps(payload, ensure_ascii=False, allow_nan=False, separators=(",", ":")).encode("utf-8")
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request payload is invalid") from exc
|
||||
if request_size > max_request_bytes:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request exceeds its operation byte limit")
|
||||
response_limit = min(self._max_response_bytes, max_response_bytes)
|
||||
if response_limit <= 0:
|
||||
raise KnowledgeFSOperationUnavailableError("KnowledgeFS operation response limit is unavailable")
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"Accept-Encoding": "identity",
|
||||
"Authorization": f"Bearer {capability_token}",
|
||||
"X-Trace-Id": trace_id,
|
||||
}
|
||||
for name, value in extra_headers:
|
||||
if name.lower() != "idempotency-key" or not 8 <= len(value.strip()) <= 255:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request header binding is invalid")
|
||||
headers["Idempotency-Key"] = value.strip()
|
||||
request_kwargs: dict[str, object] = {
|
||||
"headers": headers,
|
||||
"params": query,
|
||||
"timeout": self._timeout_seconds,
|
||||
"follow_redirects": False,
|
||||
}
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
request_kwargs["json"] = payload
|
||||
try:
|
||||
upstream_url = httpx.URL(f"{self._base_url.rstrip('/')}/").join(path.lstrip("/"))
|
||||
response = ssrf_proxy.make_request(
|
||||
method=method,
|
||||
url=str(upstream_url),
|
||||
max_retries=0,
|
||||
stream_response=True,
|
||||
**request_kwargs,
|
||||
)
|
||||
response = ssrf_proxy.buffer_response(response, max_response_bytes=response_limit)
|
||||
except (ssrf_proxy.ResponseLimitError, httpx.RequestError, ToolSSRFError) as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS request failed") from exc
|
||||
try:
|
||||
content_type = response.headers.get("content-type", "").partition(";")[0].strip().lower()
|
||||
if content_type != "application/json" and not content_type.endswith("+json"):
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned an unsupported media type")
|
||||
if not HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES:
|
||||
raise KnowledgeFSProductRemoteError(f"KnowledgeFS returned HTTP {response.status_code}")
|
||||
try:
|
||||
return _JSON_ADAPTER.validate_python(response.json())
|
||||
except (ValueError, ValidationError) as exc:
|
||||
raise KnowledgeFSProductRemoteError("KnowledgeFS returned invalid JSON") from exc
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def _matches_path(template: str, path: str) -> bool:
|
||||
template_segments = template.strip("/").split("/")
|
||||
path_segments = path.strip("/").split("/")
|
||||
if len(template_segments) != len(path_segments):
|
||||
return False
|
||||
for expected, actual in zip(template_segments, path_segments, strict=True):
|
||||
if expected.startswith("{") and expected.endswith("}"):
|
||||
if not actual or actual in {".", ".."} or any(character in actual for character in ("%", "?", "#", "\\")):
|
||||
return False
|
||||
elif expected != actual:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
__all__ = ["HTTPKnowledgeFSProductRemoteClient"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user