feat(dataset): expose New RAG KnowledgeFS contracts (#39314)
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Stephen Zhou
co-authored by
autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
parent
018fe7a9d3
commit
bd8f0104d6
@@ -8,8 +8,9 @@ can be validated explicitly against the pinned KnowledgeFS contract during devel
|
||||
Console auth and contract-specific dataset RBAC run before forwarding. Request
|
||||
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
|
||||
SSE responses remain streaming with a bounded idle read timeout, and only safe
|
||||
response headers are exposed. Upstream 401 responses become 502 so they cannot
|
||||
trigger Dify browser-session recovery; resource-level 403 responses remain 403.
|
||||
response headers are exposed. Operation-specific upstream error mappings are
|
||||
applied before Console JSON error handling; the default maps 401 to 502 so it
|
||||
cannot trigger browser-session recovery and preserves resource-level 403.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -27,9 +28,11 @@ from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
GatewayTimeout,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
default_exceptions,
|
||||
)
|
||||
|
||||
from configs import dify_config
|
||||
@@ -41,21 +44,28 @@ from controllers.console.wraps import (
|
||||
)
|
||||
from core.helper import ssrf_proxy
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from services.knowledge_fs_operations import KnowledgeFSMethod
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
type _KnowledgeFSRequestForwarder = Callable[
|
||||
[str | None, str | None, bytes | None, bytes | None],
|
||||
KnowledgeFSUpstreamResponse,
|
||||
]
|
||||
|
||||
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
|
||||
_RESPONSE_HEADER_ALLOWLIST = (
|
||||
"Cache-Control",
|
||||
@@ -128,27 +138,25 @@ def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
|
||||
|
||||
|
||||
def _knowledge_fs_operation_access_required(
|
||||
view: Callable[[KnowledgeFSMethod, str], ResponseReturnValue],
|
||||
view: Callable[[KnowledgeFSAuthorization], ResponseReturnValue],
|
||||
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
|
||||
"""Authorize one declared operation before billing and request-body work."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
|
||||
try:
|
||||
operation = get_knowledge_fs_operation(method, upstream_path)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
authorize_knowledge_fs_request(
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=current_user,
|
||||
tenant_id=tenant_id,
|
||||
operation=operation,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
except KnowledgeFSAccessDeniedError as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return view(method, upstream_path)
|
||||
return view(authorization)
|
||||
|
||||
return decorated
|
||||
|
||||
@@ -190,21 +198,26 @@ def _proxy_response(
|
||||
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
BadGateway: KnowledgeFS rejects the configured server credential.
|
||||
Forbidden: KnowledgeFS denies the account access to the requested resource.
|
||||
HTTPException: KnowledgeFS returns a status normalized by the operation contract.
|
||||
"""
|
||||
upstream = upstream_result.response
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
mapped_status = dict(upstream_result.operation.error_status_map).get(upstream.status_code)
|
||||
if mapped_status is not None:
|
||||
upstream.close()
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
raise BadGateway("KnowledgeFS authentication failed")
|
||||
if upstream.status_code == HTTPStatus.FORBIDDEN:
|
||||
upstream.close()
|
||||
raise Forbidden()
|
||||
description = "KnowledgeFS upstream request failed"
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
description = "KnowledgeFS authentication failed"
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
exception_type = default_exceptions.get(mapped_status)
|
||||
if exception_type is None:
|
||||
exception = HTTPException(description)
|
||||
exception.code = mapped_status
|
||||
raise exception
|
||||
raise exception_type(description)
|
||||
|
||||
allowed_header_names = dict.fromkeys(
|
||||
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
|
||||
@@ -237,26 +250,21 @@ def _proxy_response(
|
||||
return Response(content, status=upstream.status_code, headers=headers)
|
||||
|
||||
|
||||
def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
|
||||
"""Forward the current raw request and return its filtered upstream response.
|
||||
|
||||
The call performs one outbound KnowledgeFS request. Integration failures are
|
||||
converted to Console HTTP exceptions for the outer JSON error adapter.
|
||||
"""
|
||||
def _proxy_current_request(
|
||||
*,
|
||||
method: KnowledgeFSMethod,
|
||||
tenant_id: str,
|
||||
forward: _KnowledgeFSRequestForwarder,
|
||||
) -> Response:
|
||||
"""Forward the current raw request through one preconfigured service entry."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
proxy_result = proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=request.headers.get("Accept"),
|
||||
content_type=request.content_type,
|
||||
query=request.query_string or None,
|
||||
body=_request_body() if method != "GET" else None,
|
||||
request_headers=request.headers,
|
||||
proxy_result = forward(
|
||||
request.headers.get("Accept"),
|
||||
request.content_type,
|
||||
request.query_string or None,
|
||||
_request_body() if method != "GET" else None,
|
||||
)
|
||||
except (
|
||||
KnowledgeFSConfigurationError,
|
||||
@@ -274,15 +282,77 @@ def _proxy_request(method: KnowledgeFSMethod, upstream_path: str) -> Response:
|
||||
)
|
||||
|
||||
|
||||
def _proxy_request(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
) -> Response:
|
||||
"""Authorize and forward the current request through the combined service use case."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
def _proxy_authorized_request(authorization: KnowledgeFSAuthorization) -> Response:
|
||||
"""Forward the current request using one previously authorized operation capability.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability produced before billing and body parsing.
|
||||
|
||||
Returns:
|
||||
The filtered response returned by KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: The integration is disabled or forwarding fails.
|
||||
"""
|
||||
operation = authorization.operation
|
||||
tenant_id = authorization.tenant_id
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=operation.method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
@_knowledge_fs_enabled
|
||||
@_knowledge_fs_operation_access_required
|
||||
@cloud_edition_billing_rate_limit_check("knowledge")
|
||||
def _proxy_knowledge_fs_non_get(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
) -> ResponseReturnValue:
|
||||
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
|
||||
return _proxy_request(method, upstream_path)
|
||||
return _proxy_authorized_request(authorization)
|
||||
|
||||
|
||||
@bp.route(
|
||||
|
||||
@@ -12,6 +12,7 @@ import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
@@ -24,6 +25,16 @@ LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
|
||||
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
CONSOLE_PROXY_ERROR_SCHEMA_NAME = "ConsoleProxyError"
|
||||
CONSOLE_PROXY_ERROR_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["code", "message", "status"],
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"status": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
@@ -38,6 +49,7 @@ class ContractDeclaration(TypedDict):
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
error_status_map: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
@@ -106,8 +118,11 @@ def main() -> None:
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
if args.output_openapi:
|
||||
filtered_document = filter_openapi_document(document, declarations)
|
||||
filtered_document["x-dify-source-openapi-sha256"] = openapi_sha256
|
||||
filtered_document["x-dify-console-declarations-sha256"] = contract_declarations_sha256(declarations)
|
||||
args.output_openapi.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_openapi.write_text(json.dumps(filter_openapi_document(document, declarations), indent=2) + "\n")
|
||||
args.output_openapi.write_text(json.dumps(filtered_document, indent=2) + "\n")
|
||||
|
||||
if args.update_lock:
|
||||
LOCK_PATH.write_text(
|
||||
@@ -153,8 +168,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: ContractDeclaration = {
|
||||
"operation_id": operation_id,
|
||||
expected: dict[DeclarationField, object] = {
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(operation),
|
||||
@@ -172,6 +186,7 @@ def validate_declarations(document: dict[str, Any], declarations: tuple[Contract
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
validate_error_status_map(operation_id, declaration["error_status_map"])
|
||||
|
||||
|
||||
def filter_openapi_document(
|
||||
@@ -191,19 +206,65 @@ def filter_openapi_document(
|
||||
source_path_item = source_paths[path]
|
||||
path_metadata = {key: value for key, value in source_path_item.items() if key not in OPENAPI_METHODS}
|
||||
filtered_path_item = filtered_paths.setdefault(path, path_metadata)
|
||||
filtered_path_item[method] = source_path_item[method]
|
||||
filtered_operation = deepcopy(source_path_item[method])
|
||||
_rewrite_proxy_error_responses(filtered_operation, declaration["error_status_map"])
|
||||
filtered_path_item[method] = filtered_operation
|
||||
|
||||
filtered_document["paths"] = filtered_paths
|
||||
|
||||
source_components = document.get("components", {})
|
||||
filtered_components = {key: value for key, value in source_components.items() if key != "schemas"}
|
||||
source_schemas = source_components.get("schemas", {})
|
||||
schema_names = _referenced_schema_names(filtered_paths, source_schemas)
|
||||
filtered_components["schemas"] = {name: schema for name, schema in source_schemas.items() if name in schema_names}
|
||||
available_schemas = {**source_schemas, CONSOLE_PROXY_ERROR_SCHEMA_NAME: CONSOLE_PROXY_ERROR_SCHEMA}
|
||||
schema_names = _referenced_schema_names(filtered_paths, available_schemas)
|
||||
filtered_components["schemas"] = {
|
||||
name: schema for name, schema in available_schemas.items() if name in schema_names
|
||||
}
|
||||
filtered_document["components"] = filtered_components
|
||||
return filtered_document
|
||||
|
||||
|
||||
def validate_error_status_map(operation_id: str, error_status_map: tuple[tuple[int, int], ...]) -> None:
|
||||
"""Validate the status normalization advertised by one Console operation."""
|
||||
upstream_statuses: set[int] = set()
|
||||
for upstream_status, console_status in error_status_map:
|
||||
if upstream_status in upstream_statuses:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has duplicate error status: {upstream_status}")
|
||||
if not 400 <= upstream_status <= 599 or not 400 <= console_status <= 599:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has invalid error status mapping")
|
||||
upstream_statuses.add(upstream_status)
|
||||
|
||||
|
||||
def _rewrite_proxy_error_responses(
|
||||
operation: dict[str, Any],
|
||||
error_status_map: tuple[tuple[int, int], ...],
|
||||
) -> None:
|
||||
responses = operation.setdefault("responses", {})
|
||||
proxy_error_response = {
|
||||
"description": "Error normalized by the Dify Console KnowledgeFS proxy.",
|
||||
"content": {
|
||||
"application/json": {"schema": {"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"}}
|
||||
},
|
||||
}
|
||||
for upstream_status, console_status in error_status_map:
|
||||
existing_target = responses.get(str(console_status)) if upstream_status != console_status else None
|
||||
responses.pop(str(upstream_status), None)
|
||||
normalized_response: dict[str, Any] = deepcopy(proxy_error_response)
|
||||
existing_schema = (
|
||||
existing_target.get("content", {}).get("application/json", {}).get("schema")
|
||||
if isinstance(existing_target, dict)
|
||||
else None
|
||||
)
|
||||
if existing_schema is not None:
|
||||
normalized_response["content"]["application/json"]["schema"] = {
|
||||
"oneOf": [
|
||||
deepcopy(existing_schema),
|
||||
{"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"},
|
||||
]
|
||||
}
|
||||
responses[str(console_status)] = normalized_response
|
||||
|
||||
|
||||
def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
reference_prefix = "#/components/schemas/"
|
||||
selected: set[str] = set()
|
||||
@@ -232,7 +293,7 @@ def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""Return transport declarations from the runtime Console operation registry."""
|
||||
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
{
|
||||
@@ -245,11 +306,18 @@ def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
)
|
||||
|
||||
|
||||
def contract_declarations_sha256(declarations: tuple[ContractDeclaration, ...]) -> str:
|
||||
"""Return a stable digest for the runtime Console operation declarations."""
|
||||
content = json.dumps(declarations, separators=(",", ":"), sort_keys=True).encode()
|
||||
return sha256(content)
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
media_types = response_media_types(operation)
|
||||
if "text/event-stream" in media_types:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"commit": "4310e2d582d25e7de58183f27720afab01e123cf",
|
||||
"openapiSha256": "5827ca930ce38462bfd1b2bef387efbf37eb7ffcaedde4558af2fbaeccbfbc4b",
|
||||
"commit": "a0f50470612cc0b3656f89e4f2435aaf412e6e3b",
|
||||
"openapiSha256": "f18910e9c45a64f0855e0643a7a626fb2889021b4f943458de86c6bd2469facb",
|
||||
"repository": "https://github.com/langgenius/knowledge-fs"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,502 @@
|
||||
"""Product-facing KnowledgeFS operation and authorization declarations.
|
||||
|
||||
This registry is Dify's explicit Console surface. Transport concerns live in
|
||||
knowledge_fs_proxy so contract review does not require reading proxy mechanics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final, Literal, NamedTuple
|
||||
|
||||
from core.rbac import RBACPermission
|
||||
|
||||
type KnowledgeFSMethod = Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
|
||||
type KnowledgeFSResponseKind = Literal["binary", "buffered", "stream"]
|
||||
type KnowledgeFSRequiredScope = Literal["knowledge-spaces:read", "knowledge-spaces:write"]
|
||||
type KnowledgeFSLegacyRole = Literal["reader", "dataset_editor", "admin"]
|
||||
type KnowledgeFSErrorStatusMap = tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
class KnowledgeFSOperation(NamedTuple):
|
||||
operation_id: str
|
||||
method: KnowledgeFSMethod
|
||||
path: str
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
required_scope: KnowledgeFSRequiredScope
|
||||
rbac_permission: RBACPermission
|
||||
legacy_role: KnowledgeFSLegacyRole
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
error_status_map: KnowledgeFSErrorStatusMap
|
||||
|
||||
|
||||
def _console_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
rbac_permission: RBACPermission,
|
||||
legacy_role: KnowledgeFSLegacyRole,
|
||||
max_response_bytes: int = 1_048_576,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
response_kind: KnowledgeFSResponseKind = "buffered",
|
||||
response_media_types: tuple[str, ...] = ("application/json",),
|
||||
error_status_map: KnowledgeFSErrorStatusMap = ((401, 502), (403, 403)),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare one contract-pinned operation with an explicit Dify authorization policy."""
|
||||
is_read = method == "GET"
|
||||
return KnowledgeFSOperation(
|
||||
operation_id=operation_id,
|
||||
method=method,
|
||||
path=path,
|
||||
response_kind=response_kind,
|
||||
required_scope="knowledge-spaces:read" if is_read else "knowledge-spaces:write",
|
||||
rbac_permission=rbac_permission,
|
||||
legacy_role=legacy_role,
|
||||
max_response_bytes=max_response_bytes,
|
||||
request_headers=request_headers,
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=response_media_types,
|
||||
error_status_map=error_status_map,
|
||||
)
|
||||
|
||||
|
||||
def _dataset_read_operation(operation_id: str, path: str) -> KnowledgeFSOperation:
|
||||
"""Declare a dataset-readable buffered JSON operation."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
"GET",
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
)
|
||||
|
||||
|
||||
def _dataset_edit_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare a dataset-editable buffered JSON operation."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
method,
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
def _external_source_operation(
|
||||
operation_id: str,
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
*,
|
||||
request_headers: tuple[str, ...] = ("x-trace-id",),
|
||||
) -> KnowledgeFSOperation:
|
||||
"""Declare a source-connection operation restricted to dataset editors."""
|
||||
return _console_operation(
|
||||
operation_id,
|
||||
method,
|
||||
path,
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS: Final[tuple[KnowledgeFSOperation, ...]] = (
|
||||
_console_operation(
|
||||
operation_id="listKnowledgeSpaces",
|
||||
method="GET",
|
||||
path="knowledge-spaces",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="createKnowledgeSpace",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
rbac_permission=RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesById",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation("patchKnowledgeSpacesById", "PATCH", "knowledge-spaces/{id}"),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesById",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation("getKnowledgeSpacesByIdStats", "knowledge-spaces/{id}/stats"),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdAccessPolicy",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/access-policy",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="patchKnowledgeSpacesByIdAccessPolicy",
|
||||
method="PATCH",
|
||||
path="knowledge-spaces/{id}/access-policy",
|
||||
rbac_permission=RBACPermission.DATASET_ACCESS_CONFIG,
|
||||
legacy_role="admin",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getSourceProviders",
|
||||
method="GET",
|
||||
path="source-providers",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceConnections",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-connections",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceConnections",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-connections",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourceConnectionsOauth",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/source-connections/oauth",
|
||||
),
|
||||
_external_source_operation("postSourceOauthCallback", "POST", "source-oauth/callback"),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-connections/{connectionId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/source-connections/{connectionId}",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-connections/{connectionId}/refresh",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSources",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/sources",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSources",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/sources",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"patchKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"PATCH",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"PUT",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/credentials",
|
||||
),
|
||||
_external_source_operation(
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/credentials",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdSync",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/sync",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/crawl-preview",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdWorkflowImports",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/workflow-imports",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdPages",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/pages",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdFiles",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/files",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawl",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/crawl",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImport",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/import",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdTest",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/test",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImportFiles",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/{sourceId}/import-files",
|
||||
),
|
||||
_external_source_operation(
|
||||
"postKnowledgeSpacesByIdSourcesBulk",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/sources/bulk",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceWorkflows",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-workflows",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceWorkflowsByRunId",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_external_source_operation(
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdBulkItems",
|
||||
"GET",
|
||||
"knowledge-spaces/{id}/source-workflows/{runId}/bulk-items",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/pages",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/cancel",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/retry",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/source-workflows/{runId}/selection",
|
||||
rbac_permission=RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
legacy_role="dataset_editor",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
method="PUT",
|
||||
path="knowledge-spaces/{id}/sources/{sourceId}/sync-policy",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_dataset_read_operation("getKnowledgeSpacesByIdDocuments", "knowledge-spaces/{id}/documents"),
|
||||
_dataset_edit_operation("postKnowledgeSpacesByIdDocuments", "POST", "knowledge-spaces/{id}/documents"),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdDocumentsBulk",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/documents/bulk",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsBulk",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/bulk",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsBulkReindex",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/bulk/reindex",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/documents/{documentId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdLogicalDocuments",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/logical-documents",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"deleteKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"DELETE",
|
||||
"knowledge-spaces/{id}/logical-documents/{documentId}",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdOutline",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/outline",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/logical-documents/{documentId}",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/revisions",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionRollback",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/rollback",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"patchKnowledgeSpacesByIdDocumentsByDocumentIdMetadata",
|
||||
"PATCH",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/metadata",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkIdState",
|
||||
"POST",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/revisions/{revision}/chunks/{chunkId}/state",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdProcessingTasks",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/processing-tasks",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasks",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/processing-tasks",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents",
|
||||
method="GET",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}/events",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
legacy_role="reader",
|
||||
max_response_bytes=67_108_864,
|
||||
request_headers=("last-event-id", "x-trace-id"),
|
||||
response_kind="stream",
|
||||
response_media_types=("text/event-stream",),
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
method="DELETE",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_console_operation(
|
||||
operation_id="postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry",
|
||||
method="POST",
|
||||
path="knowledge-spaces/{id}/documents/{documentId}/processing-tasks/{taskId}/retry",
|
||||
rbac_permission=RBACPermission.DATASET_EDIT,
|
||||
legacy_role="dataset_editor",
|
||||
),
|
||||
_dataset_read_operation(
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/settings",
|
||||
),
|
||||
_dataset_edit_operation(
|
||||
"putKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"PUT",
|
||||
"knowledge-spaces/{id}/documents/{documentId}/settings",
|
||||
),
|
||||
_dataset_read_operation("getJobsById", "jobs/{id}"),
|
||||
_dataset_edit_operation("deleteJobsById", "DELETE", "jobs/{id}"),
|
||||
_dataset_edit_operation("postJobsByIdRetry", "POST", "jobs/{id}/retry"),
|
||||
_dataset_read_operation("getDeletionJobsByJobId", "deletion-jobs/{jobId}"),
|
||||
_dataset_edit_operation(
|
||||
"postDeletionJobsByJobIdRetry",
|
||||
"POST",
|
||||
"deletion-jobs/{jobId}/retry",
|
||||
request_headers=("idempotency-key", "x-trace-id"),
|
||||
),
|
||||
_dataset_read_operation("getBulkJobsById", "bulk-jobs/{id}"),
|
||||
)
|
||||
@@ -1,33 +1,32 @@
|
||||
"""Transport-only forwarding for the explicitly enabled KnowledgeFS Console operations.
|
||||
"""Authorize and forward the explicitly enabled KnowledgeFS Console operations.
|
||||
|
||||
KnowledgeFS owns the request and response contract. This module binds short-lived
|
||||
account and workspace identities, enforces Dify's coarse workspace policy, and
|
||||
normalizes transport failures. Dify deliberately maintains a small product-facing
|
||||
operation registry instead of exposing the full upstream OpenAPI surface. The
|
||||
dedicated request path uses Dify's shared SSRF policy, never follows redirects,
|
||||
bounds buffered responses, and rejects compressed responses.
|
||||
The dedicated request path uses Dify's shared SSRF policy, never follows redirects,
|
||||
bounds buffered responses, and rejects compressed streaming responses.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from typing import Final, Literal, NamedTuple, Protocol
|
||||
from typing import NamedTuple, Protocol
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
|
||||
from configs import dify_config
|
||||
from core.helper import ssrf_proxy
|
||||
from core.rbac import RBACPermission, RBACResourceScope
|
||||
from core.rbac import RBACResourceScope
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from models import Account
|
||||
from services.enterprise.rbac_service import RBACService
|
||||
|
||||
type KnowledgeFSMethod = Literal["DELETE", "GET", "PATCH", "POST", "PUT"]
|
||||
type KnowledgeFSResponseKind = Literal["binary", "buffered", "stream"]
|
||||
type KnowledgeFSRequiredScope = Literal["knowledge-spaces:read", "knowledge-spaces:write"]
|
||||
from services.knowledge_fs_operations import (
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
KnowledgeFSResponseKind,
|
||||
)
|
||||
|
||||
_JWT_AUDIENCE = "knowledge-fs"
|
||||
_JWT_ISSUER = "dify"
|
||||
@@ -35,56 +34,55 @@ _JWT_TTL_SECONDS = 60
|
||||
_MAX_BUFFERED_RESPONSE_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
class KnowledgeFSOperation(NamedTuple):
|
||||
operation_id: str
|
||||
method: KnowledgeFSMethod
|
||||
path: str
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
required_scope: KnowledgeFSRequiredScope
|
||||
rbac_permission: RBACPermission
|
||||
requires_dataset_editor: bool
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
|
||||
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS: Final[tuple[KnowledgeFSOperation, ...]] = (
|
||||
KnowledgeFSOperation(
|
||||
operation_id="listKnowledgeSpaces",
|
||||
method="GET",
|
||||
path="knowledge-spaces",
|
||||
response_kind="buffered",
|
||||
required_scope="knowledge-spaces:read",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
requires_dataset_editor=False,
|
||||
max_response_bytes=1_048_576,
|
||||
request_headers=("x-trace-id",),
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=("application/json",),
|
||||
),
|
||||
KnowledgeFSOperation(
|
||||
operation_id="createKnowledgeSpace",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
response_kind="buffered",
|
||||
required_scope="knowledge-spaces:write",
|
||||
rbac_permission=RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
requires_dataset_editor=True,
|
||||
max_response_bytes=1_048_576,
|
||||
request_headers=("x-trace-id",),
|
||||
response_headers=("x-trace-id",),
|
||||
response_media_types=("application/json",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class KnowledgeFSUpstreamResponse(NamedTuple):
|
||||
response: httpx.Response
|
||||
response_kind: KnowledgeFSResponseKind
|
||||
operation: KnowledgeFSOperation
|
||||
|
||||
|
||||
_AUTHORIZATION_MARKER = object()
|
||||
|
||||
|
||||
@dataclass(eq=False, frozen=True, init=False, slots=True)
|
||||
class KnowledgeFSAuthorization:
|
||||
"""Single-use forwarding capability created after Dify workspace policy checks.
|
||||
|
||||
Callers obtain this value from :func:`authorize_knowledge_fs_request`. Direct
|
||||
construction and repeated forwarding are rejected before outbound I/O.
|
||||
"""
|
||||
|
||||
account_id: str
|
||||
tenant_id: str
|
||||
operation: KnowledgeFSOperation
|
||||
_used: bool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
account_id: str,
|
||||
tenant_id: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
*,
|
||||
_marker: object | None = None,
|
||||
) -> None:
|
||||
if _marker is not _AUTHORIZATION_MARKER:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS authorization must be created by workspace authorization")
|
||||
object.__setattr__(self, "account_id", account_id)
|
||||
object.__setattr__(self, "tenant_id", tenant_id)
|
||||
object.__setattr__(self, "operation", operation)
|
||||
object.__setattr__(self, "_used", False)
|
||||
|
||||
def consume(self) -> tuple[str, str, KnowledgeFSOperation]:
|
||||
"""Return the authorized principals and canonical operation exactly once.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSAccessDeniedError: The capability was already consumed.
|
||||
"""
|
||||
if self._used:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS authorization has already been used")
|
||||
object.__setattr__(self, "_used", True)
|
||||
return self.account_id, self.tenant_id, self.operation
|
||||
|
||||
|
||||
class _RequestHeaders(Protocol):
|
||||
def items(self) -> Iterable[tuple[str, str]]: ...
|
||||
|
||||
@@ -113,20 +111,29 @@ def authorize_knowledge_fs_request(
|
||||
*,
|
||||
account: Account,
|
||||
tenant_id: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
method: KnowledgeFSMethod,
|
||||
path: str,
|
||||
) -> KnowledgeFSAuthorization:
|
||||
"""Enforce Dify's workspace policy before KFS performs resource authorization.
|
||||
|
||||
Args:
|
||||
account: Authenticated Dify account with its current workspace role.
|
||||
tenant_id: Current Dify workspace identifier.
|
||||
operation: Dify-maintained KnowledgeFS operation and policy metadata.
|
||||
method: Requested upstream HTTP method.
|
||||
path: Requested relative KnowledgeFS path.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSRouteNotAllowedError: The method and path do not resolve to a declared operation.
|
||||
KnowledgeFSAccessDeniedError: The account lacks a required legacy or enterprise permission.
|
||||
|
||||
Returns:
|
||||
A request-scoped capability binding the authorized account, workspace, and operation.
|
||||
"""
|
||||
if operation.requires_dataset_editor and not account.is_dataset_editor:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS mutations require dataset edit access")
|
||||
operation = get_knowledge_fs_operation(method, path)
|
||||
if operation.legacy_role == "dataset_editor" and not account.is_dataset_editor:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation requires dataset edit access")
|
||||
if operation.legacy_role == "admin" and not account.is_admin_or_owner:
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation requires workspace administration access")
|
||||
if not RBACService.CheckAccess.check(
|
||||
tenant_id,
|
||||
account.id,
|
||||
@@ -134,6 +141,7 @@ def authorize_knowledge_fs_request(
|
||||
resource_type=RBACResourceScope.DATASET.value,
|
||||
):
|
||||
raise KnowledgeFSAccessDeniedError("KnowledgeFS operation is denied by workspace RBAC")
|
||||
return KnowledgeFSAuthorization(account.id, tenant_id, operation, _marker=_AUTHORIZATION_MARKER)
|
||||
|
||||
|
||||
def proxy_knowledge_fs_request(
|
||||
@@ -149,20 +157,62 @@ def proxy_knowledge_fs_request(
|
||||
request_headers: _RequestHeaders | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
"""Authorize and forward one allowlisted KnowledgeFS request as a single use case."""
|
||||
operation = get_knowledge_fs_operation(method, path)
|
||||
authorize_knowledge_fs_request(
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id=tenant_id,
|
||||
operation=operation,
|
||||
method=method,
|
||||
path=path,
|
||||
)
|
||||
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request_headers,
|
||||
)
|
||||
|
||||
|
||||
def proxy_authorized_knowledge_fs_request(
|
||||
*,
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
accept: str | None = None,
|
||||
content_type: str | None = None,
|
||||
query: bytes | None = None,
|
||||
body: bytes | None = None,
|
||||
request_headers: _RequestHeaders | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
"""Forward one request whose operation and workspace policy were already authorized.
|
||||
|
||||
This performs one outbound KnowledgeFS request and does not repeat Dify RBAC checks.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability returned by :func:`authorize_knowledge_fs_request`.
|
||||
accept: Original Accept header, when present.
|
||||
content_type: Original request Content-Type header, when present.
|
||||
query: Original encoded query string from the Console request.
|
||||
body: Original request body, when present.
|
||||
request_headers: Incoming headers; only names declared by the operation are forwarded.
|
||||
|
||||
Returns:
|
||||
The bounded KnowledgeFS response together with its transport metadata.
|
||||
|
||||
Raises:
|
||||
KnowledgeFSConfigurationError: The connection is incomplete or blocked by outbound policy.
|
||||
KnowledgeFSRouteNotAllowedError: A forwarded request header is outside the operation contract.
|
||||
KnowledgeFSTimeoutError: KnowledgeFS exceeds the configured timeout.
|
||||
KnowledgeFSTransportError: The request fails or its response violates transport bounds.
|
||||
"""
|
||||
account_id, tenant_id, operation = authorization.consume()
|
||||
incoming_request_headers = {name.lower(): value for name, value in (request_headers or {}).items()}
|
||||
contract_request_headers = {
|
||||
name: incoming_request_headers[name] for name in operation.request_headers if name in incoming_request_headers
|
||||
}
|
||||
return _forward_knowledge_fs_request(
|
||||
account_id=account.id,
|
||||
method=method,
|
||||
path=path,
|
||||
account_id=account_id,
|
||||
method=operation.method,
|
||||
path=operation.path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
|
||||
@@ -7,9 +7,11 @@ from unittest.mock import MagicMock
|
||||
import httpx
|
||||
import pytest
|
||||
from flask import Flask, Response
|
||||
from pydantic import SecretStr
|
||||
from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
@@ -26,12 +28,14 @@ from controllers.console.knowledge_fs_proxy import (
|
||||
proxy_knowledge_fs_write,
|
||||
)
|
||||
from controllers.console.wraps import RBACPermission
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
from services.knowledge_fs_operations import (
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
KnowledgeFSResponseKind,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
get_knowledge_fs_operation,
|
||||
@@ -47,6 +51,7 @@ def _upstream(
|
||||
response: httpx.Response,
|
||||
kind: KnowledgeFSResponseKind = "buffered",
|
||||
*,
|
||||
error_status_map: tuple[tuple[int, int], ...] = ((401, 502), (403, 403)),
|
||||
max_response_bytes: int | None = None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
operation = KnowledgeFSOperation(
|
||||
@@ -56,7 +61,7 @@ def _upstream(
|
||||
response_kind=kind,
|
||||
required_scope="knowledge-spaces:read",
|
||||
rbac_permission=RBACPermission.DATASET_READONLY,
|
||||
requires_dataset_editor=False,
|
||||
legacy_role="reader",
|
||||
max_response_bytes=max_response_bytes
|
||||
or (64 * 1024 * 1024 if kind == "stream" else 25 * 1024 * 1024 if kind == "binary" else 1024 * 1024),
|
||||
request_headers=(),
|
||||
@@ -67,6 +72,7 @@ def _upstream(
|
||||
"x-session-id",
|
||||
),
|
||||
response_media_types=(),
|
||||
error_status_map=error_status_map,
|
||||
)
|
||||
return KnowledgeFSUpstreamResponse(response, kind, operation)
|
||||
|
||||
@@ -93,7 +99,7 @@ def _set_current_workspace(
|
||||
def _bypass_policy_wrappers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.knowledge_fs_proxy._proxy_knowledge_fs_non_get",
|
||||
unwrap(_proxy_knowledge_fs_non_get),
|
||||
lambda method, path: _proxy_request(method, path),
|
||||
)
|
||||
|
||||
|
||||
@@ -287,10 +293,8 @@ def test_read_post_applies_knowledge_rate_limit_once(
|
||||
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.wraps.FeatureService.get_knowledge_rate_limit",
|
||||
MagicMock(return_value=MagicMock(enabled=True, limit=10)),
|
||||
@@ -300,14 +304,19 @@ def test_read_post_applies_knowledge_rate_limit_once(
|
||||
monkeypatch.setattr("controllers.console.wraps.redis_client.zremrangebyscore", MagicMock())
|
||||
monkeypatch.setattr("controllers.console.wraps.redis_client.zcard", MagicMock(return_value=1))
|
||||
proxy = MagicMock(return_value=Response(status=200))
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy._proxy_request", proxy)
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy._proxy_authorized_request", proxy)
|
||||
|
||||
with app.test_request_context("/console/api/knowledge-fs/knowledge-spaces", method="POST"):
|
||||
response = _proxy_knowledge_fs_non_get("POST", "knowledge-spaces")
|
||||
|
||||
assert isinstance(response, Response)
|
||||
zadd.assert_called_once()
|
||||
proxy.assert_called_once_with("POST", "knowledge-spaces")
|
||||
proxy.assert_called_once()
|
||||
authorization = proxy.call_args.args[0]
|
||||
assert authorization.account_id == "account-1"
|
||||
assert authorization.tenant_id == "tenant-1"
|
||||
assert authorization.operation.operation_id == "createKnowledgeSpace"
|
||||
check_access.assert_called_once()
|
||||
|
||||
|
||||
def test_denied_write_does_not_consume_the_workspace_rate_limit(
|
||||
@@ -447,6 +456,65 @@ def test_generic_write_forwards_path_raw_body_and_current_tenant(
|
||||
assert response.get_json()["tenantId"] == "tenant-1"
|
||||
|
||||
|
||||
def test_generic_write_forwards_through_the_authorized_production_path(
|
||||
app: Flask,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
|
||||
def current_workspace() -> tuple[MagicMock, str]:
|
||||
return account, "tenant-1"
|
||||
|
||||
monkeypatch.setattr("controllers.console.knowledge_fs_proxy.current_account_with_tenant", current_workspace)
|
||||
monkeypatch.setattr("controllers.console.wraps.current_account_with_tenant", current_workspace)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr(
|
||||
"controllers.console.wraps.FeatureService.get_knowledge_rate_limit",
|
||||
MagicMock(return_value=MagicMock(enabled=False)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.dify_config.KNOWLEDGE_FS_BASE_URL",
|
||||
"http://knowledge-fs.test",
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.dify_config.KNOWLEDGE_FS_JWT_SECRET",
|
||||
SecretStr("production-secret-with-at-least-32-bytes"),
|
||||
raising=False,
|
||||
)
|
||||
upstream_request = MagicMock(
|
||||
return_value=httpx.Response(
|
||||
201,
|
||||
content=b'{"id":"space-1","tenantId":"tenant-1"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", upstream_request)
|
||||
route = unwrap(proxy_knowledge_fs_write)
|
||||
body = b'{"idempotencyKey":"create-product-docs","name":"Product docs"}'
|
||||
|
||||
with app.test_request_context(
|
||||
"/console/api/knowledge-fs/knowledge-spaces",
|
||||
method="POST",
|
||||
query_string={"source": "console"},
|
||||
data=body,
|
||||
content_type="application/json",
|
||||
headers={"X-Trace-Id": "trace-1"},
|
||||
):
|
||||
response = route("knowledge-spaces")
|
||||
|
||||
assert isinstance(response, Response)
|
||||
assert response.status_code == 201
|
||||
assert response.get_json() == {"id": "space-1", "tenantId": "tenant-1"}
|
||||
check_access.assert_called_once()
|
||||
assert upstream_request.call_args.kwargs["method"] == "POST"
|
||||
assert upstream_request.call_args.kwargs["url"] == "http://knowledge-fs.test/knowledge-spaces"
|
||||
assert upstream_request.call_args.kwargs["params"] == b"source=console"
|
||||
assert upstream_request.call_args.kwargs["content"] == body
|
||||
assert upstream_request.call_args.kwargs["headers"]["x-trace-id"] == "trace-1"
|
||||
|
||||
|
||||
def test_generic_write_forwards_contract_declared_request_headers(
|
||||
app: Flask,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -617,6 +685,43 @@ def test_resource_authorization_rejection_is_exposed_as_forbidden(
|
||||
route("knowledge-spaces")
|
||||
|
||||
|
||||
def test_proxy_response_applies_operation_specific_error_status_mapping() -> None:
|
||||
upstream = httpx.Response(
|
||||
429,
|
||||
content=b'{"error":"rate limited"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with pytest.raises(ServiceUnavailable):
|
||||
_proxy_response(
|
||||
_upstream(upstream, error_status_map=((429, 503),)),
|
||||
tenant_id="tenant-1",
|
||||
contract_response_headers=(),
|
||||
max_response_bytes=1024 * 1024,
|
||||
)
|
||||
|
||||
assert upstream.is_closed
|
||||
|
||||
|
||||
def test_proxy_response_preserves_nonstandard_mapped_error_status() -> None:
|
||||
upstream = httpx.Response(
|
||||
429,
|
||||
content=b'{"error":"rate limited"}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
_proxy_response(
|
||||
_upstream(upstream, error_status_map=((429, 499),)),
|
||||
tenant_id="tenant-1",
|
||||
contract_response_headers=(),
|
||||
max_response_bytes=1024 * 1024,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == 499
|
||||
assert upstream.is_closed
|
||||
|
||||
|
||||
def test_contract_response_headers_are_deduplicated_case_insensitively() -> None:
|
||||
upstream = httpx.Response(
|
||||
200,
|
||||
@@ -671,6 +776,7 @@ def test_disallowed_non_get_route_is_hidden_as_not_found(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
method: KnowledgeFSMethod,
|
||||
) -> None:
|
||||
_set_current_workspace(monkeypatch)
|
||||
route = unwrap(proxy_knowledge_fs_write)
|
||||
|
||||
with app.test_request_context("/console/api/knowledge-fs/not-a-route", method=method):
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -15,7 +16,7 @@ from dev.generate_knowledge_fs_contract import (
|
||||
filter_openapi_document,
|
||||
validate_declarations,
|
||||
)
|
||||
from services.knowledge_fs_proxy import KNOWLEDGE_FS_CONSOLE_OPERATIONS, KnowledgeFSOperation
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS, KnowledgeFSOperation
|
||||
|
||||
|
||||
def test_contract_cli_updates_checks_and_detects_openapi_drift(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -111,7 +112,7 @@ def test_contract_script_loads_runtime_registry_outside_api_directory(tmp_path:
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.stdout.strip() == "2"
|
||||
assert result.stdout.strip() == str(len(KNOWLEDGE_FS_CONSOLE_OPERATIONS))
|
||||
|
||||
|
||||
def test_validate_declarations_accepts_matching_contract() -> None:
|
||||
@@ -178,47 +179,115 @@ def test_filter_openapi_document_keeps_only_declared_operations_and_referenced_s
|
||||
|
||||
assert set(filtered["paths"]) == {"/knowledge-spaces"}
|
||||
assert set(filtered["paths"]["/knowledge-spaces"]) == {"get"}
|
||||
assert set(filtered["components"]["schemas"]) == {"KnowledgeSpaceList", "KnowledgeSpace"}
|
||||
assert set(filtered["components"]["schemas"]) == {
|
||||
"ConsoleProxyError",
|
||||
"KnowledgeSpaceList",
|
||||
"KnowledgeSpace",
|
||||
}
|
||||
assert filtered["components"]["securitySchemes"] == document["components"]["securitySchemes"]
|
||||
|
||||
|
||||
def test_console_operation_registry_matches_contract() -> None:
|
||||
list_route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
create_route = operation("knowledge-spaces:write", "createKnowledgeSpace")
|
||||
for route in (list_route, create_route):
|
||||
route["parameters"] = [{"in": "header", "name": "X-Trace-Id"}]
|
||||
route["responses"] = {
|
||||
"200": {
|
||||
"content": {"application/json": {}},
|
||||
"headers": {"X-Trace-Id": {}},
|
||||
}
|
||||
}
|
||||
def test_filter_openapi_document_keeps_sse_for_streaming_orpc_contracts() -> None:
|
||||
json_declaration = declaration()
|
||||
stream_declaration = declaration(
|
||||
operation_id="streamTask",
|
||||
path="tasks/{id}/events",
|
||||
response_kind="stream",
|
||||
response_media_types=("text/event-stream",),
|
||||
)
|
||||
json_operation = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
stream_operation = operation(
|
||||
"knowledge-spaces:read",
|
||||
"streamTask",
|
||||
responses={"200": {"content": {"text/event-stream": {"schema": {"$ref": "#/components/schemas/TaskEvent"}}}}},
|
||||
)
|
||||
|
||||
validate_declarations(
|
||||
filtered = filter_openapi_document(
|
||||
{
|
||||
"paths": {
|
||||
"/knowledge-spaces": {
|
||||
"get": list_route,
|
||||
"post": create_route,
|
||||
}
|
||||
}
|
||||
"/knowledge-spaces": {"get": json_operation},
|
||||
"/tasks/{id}/events": {"get": stream_operation},
|
||||
},
|
||||
"components": {"schemas": {"TaskEvent": {"type": "object"}}},
|
||||
},
|
||||
(json_declaration, stream_declaration),
|
||||
)
|
||||
|
||||
assert set(filtered["paths"]) == {"/knowledge-spaces", "/tasks/{id}/events"}
|
||||
assert filtered["components"]["schemas"]["TaskEvent"] == {"type": "object"}
|
||||
|
||||
|
||||
def test_filter_openapi_document_rewrites_proxy_error_responses() -> None:
|
||||
route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
route["responses"] = {
|
||||
"200": {"content": {"application/json": {}}},
|
||||
"401": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}},
|
||||
"403": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}},
|
||||
}
|
||||
document = {
|
||||
"paths": {"/knowledge-spaces": {"get": route}},
|
||||
"components": {"schemas": {"ErrorResponse": {"type": "object"}}},
|
||||
}
|
||||
|
||||
filtered = filter_openapi_document(
|
||||
document,
|
||||
(declaration(error_status_map=((401, 502), (403, 403))),),
|
||||
)
|
||||
|
||||
responses = filtered["paths"]["/knowledge-spaces"]["get"]["responses"]
|
||||
assert "401" not in responses
|
||||
assert responses["403"]["content"]["application/json"]["schema"] == {
|
||||
"$ref": "#/components/schemas/ConsoleProxyError"
|
||||
}
|
||||
assert responses["502"]["content"]["application/json"]["schema"] == {
|
||||
"$ref": "#/components/schemas/ConsoleProxyError"
|
||||
}
|
||||
assert filtered["components"]["schemas"]["ConsoleProxyError"]["required"] == ["code", "message", "status"]
|
||||
|
||||
|
||||
def test_console_operation_registry_matches_contract() -> None:
|
||||
validate_declarations(
|
||||
console_registry_document(),
|
||||
tuple(_contract_declaration(operation) for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS),
|
||||
)
|
||||
|
||||
|
||||
def test_generated_contract_metadata_matches_current_pin_and_registry() -> None:
|
||||
metadata = (
|
||||
contract_validator.WORKSPACE_ROOT / "packages/contracts/generated/knowledge-fs/metadata.gen.ts"
|
||||
).read_text()
|
||||
lock = json.loads(contract_validator.LOCK_PATH.read_text())
|
||||
|
||||
assert _metadata_string(metadata, "knowledgeFsSourceOpenapiSha256") == lock["openapiSha256"]
|
||||
assert _metadata_string(
|
||||
metadata,
|
||||
"knowledgeFsConsoleDeclarationsSha256",
|
||||
) == contract_validator.contract_declarations_sha256(contract_validator.console_contract_declarations())
|
||||
|
||||
|
||||
def _metadata_string(source: str, export_name: str) -> str:
|
||||
match = re.search(rf"export const {export_name}\s*=\s*'([0-9a-f]{{64}})'", source)
|
||||
assert match is not None, f"missing generated metadata export: {export_name}"
|
||||
return match.group(1)
|
||||
|
||||
|
||||
def console_registry_document() -> dict[str, object]:
|
||||
list_route = operation("knowledge-spaces:read", "listKnowledgeSpaces")
|
||||
create_route = operation("knowledge-spaces:write", "createKnowledgeSpace")
|
||||
for route in (list_route, create_route):
|
||||
route["parameters"] = [{"in": "header", "name": "X-Trace-Id"}]
|
||||
route["responses"] = {
|
||||
"200": {
|
||||
"content": {"application/json": {}},
|
||||
"headers": {"X-Trace-Id": {}},
|
||||
}
|
||||
}
|
||||
return {"paths": {"/knowledge-spaces": {"get": list_route, "post": create_route}}}
|
||||
paths: dict[str, dict[str, object]] = {}
|
||||
for console_operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS:
|
||||
route = operation(
|
||||
console_operation.required_scope,
|
||||
console_operation.operation_id,
|
||||
parameters=[{"in": "header", "name": name} for name in console_operation.request_headers],
|
||||
responses={
|
||||
"200": {
|
||||
"content": {media_type: {} for media_type in console_operation.response_media_types},
|
||||
"headers": {name: {} for name in console_operation.response_headers},
|
||||
}
|
||||
},
|
||||
)
|
||||
route["x-knowledge-fs-max-response-bytes"] = console_operation.max_response_bytes
|
||||
paths.setdefault(f"/{console_operation.path}", {})[console_operation.method.lower()] = route
|
||||
return {"paths": paths}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -377,6 +446,7 @@ def declaration(**overrides: object) -> ContractDeclaration:
|
||||
"request_headers": (),
|
||||
"response_headers": (),
|
||||
"response_media_types": ("application/json",),
|
||||
"error_status_map": ((401, 502), (403, 403)),
|
||||
}
|
||||
value.update(overrides)
|
||||
return cast(ContractDeclaration, value)
|
||||
@@ -393,6 +463,7 @@ def _contract_declaration(operation: KnowledgeFSOperation) -> ContractDeclaratio
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -10,16 +10,21 @@ from pydantic import SecretStr
|
||||
from core.helper import ssrf_proxy
|
||||
from core.rbac import RBACPermission
|
||||
from core.tools.errors import ToolSSRFError
|
||||
from services.knowledge_fs_proxy import (
|
||||
from services.knowledge_fs_operations import (
|
||||
KNOWLEDGE_FS_CONSOLE_OPERATIONS,
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSMethod,
|
||||
KnowledgeFSOperation,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
from services.knowledge_fs_proxy import (
|
||||
@@ -28,16 +33,178 @@ from services.knowledge_fs_proxy import (
|
||||
|
||||
_JWT_SECRET = "production-secret-with-at-least-32-bytes"
|
||||
|
||||
_HAPPY_PATH_OPERATION_IDS = (
|
||||
"listKnowledgeSpaces",
|
||||
"createKnowledgeSpace",
|
||||
"getKnowledgeSpacesById",
|
||||
"getKnowledgeSpacesByIdAccessPolicy",
|
||||
"patchKnowledgeSpacesByIdAccessPolicy",
|
||||
"getSourceProviders",
|
||||
"getKnowledgeSpacesByIdSourceConnections",
|
||||
"postKnowledgeSpacesByIdSourceConnections",
|
||||
"postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh",
|
||||
"getKnowledgeSpacesByIdSources",
|
||||
"postKnowledgeSpacesByIdSources",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunId",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry",
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy",
|
||||
"getKnowledgeSpacesByIdLogicalDocuments",
|
||||
"getKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks",
|
||||
"getKnowledgeSpacesByIdProcessingTasks",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents",
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry",
|
||||
)
|
||||
|
||||
_REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS = {
|
||||
"patchKnowledgeSpacesById",
|
||||
"deleteKnowledgeSpacesById",
|
||||
"getKnowledgeSpacesByIdStats",
|
||||
"postKnowledgeSpacesByIdSourceConnectionsOauth",
|
||||
"postSourceOauthCallback",
|
||||
"getKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"deleteKnowledgeSpacesByIdSourceConnectionsByConnectionId",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"patchKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceId",
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"deleteKnowledgeSpacesByIdSourcesBySourceIdCredentials",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdSync",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdWorkflowImports",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdPages",
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdFiles",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawl",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImport",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdTest",
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdImportFiles",
|
||||
"postKnowledgeSpacesByIdSourcesBulk",
|
||||
"getKnowledgeSpacesByIdSourceWorkflows",
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdBulkItems",
|
||||
"getKnowledgeSpacesByIdDocuments",
|
||||
"postKnowledgeSpacesByIdDocuments",
|
||||
"deleteKnowledgeSpacesByIdDocumentsBulk",
|
||||
"postKnowledgeSpacesByIdDocumentsBulk",
|
||||
"postKnowledgeSpacesByIdDocumentsBulkReindex",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentId",
|
||||
"deleteKnowledgeSpacesByIdLogicalDocumentsByDocumentId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdOutline",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionRollback",
|
||||
"patchKnowledgeSpacesByIdDocumentsByDocumentIdMetadata",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkId",
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunksByChunkIdState",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasks",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId",
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"putKnowledgeSpacesByIdDocumentsByDocumentIdSettings",
|
||||
"getJobsById",
|
||||
"deleteJobsById",
|
||||
"postJobsByIdRetry",
|
||||
"getDeletionJobsByJobId",
|
||||
"postDeletionJobsByJobIdRetry",
|
||||
"getBulkJobsById",
|
||||
}
|
||||
|
||||
_EXPANDED_EXTERNAL_SOURCE_OPERATION_IDS = {
|
||||
operation_id
|
||||
for operation_id in _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
if "Source" in operation_id or operation_id == "postSourceOauthCallback"
|
||||
}
|
||||
|
||||
_OPERATION_AUTHORIZATION_POLICIES = {
|
||||
"listKnowledgeSpaces": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"createKnowledgeSpace": (RBACPermission.DATASET_CREATE_AND_MANAGEMENT, "dataset_editor"),
|
||||
"getKnowledgeSpacesById": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdAccessPolicy": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"patchKnowledgeSpacesByIdAccessPolicy": (RBACPermission.DATASET_ACCESS_CONFIG, "admin"),
|
||||
"getSourceProviders": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"getKnowledgeSpacesByIdSourceConnections": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourceConnections": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourceConnectionsByConnectionIdRefresh": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSources": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"postKnowledgeSpacesByIdSources": (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor"),
|
||||
"postKnowledgeSpacesByIdSourcesBySourceIdCrawlPreview": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunId": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourceWorkflowsByRunIdPages": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdCancel": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdRetry": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdSourceWorkflowsByRunIdSelection": (
|
||||
RBACPermission.DATASET_EXTERNAL_CONNECT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"getKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"putKnowledgeSpacesByIdSourcesBySourceIdSyncPolicy": (RBACPermission.DATASET_EDIT, "dataset_editor"),
|
||||
"getKnowledgeSpacesByIdLogicalDocuments": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdLogicalDocumentsByDocumentId": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisions": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdRevisionsByRevisionChunks": (
|
||||
RBACPermission.DATASET_READONLY,
|
||||
"reader",
|
||||
),
|
||||
"getKnowledgeSpacesByIdProcessingTasks": (RBACPermission.DATASET_READONLY, "reader"),
|
||||
"getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents": (
|
||||
RBACPermission.DATASET_READONLY,
|
||||
"reader",
|
||||
),
|
||||
"deleteKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskId": (
|
||||
RBACPermission.DATASET_EDIT,
|
||||
"dataset_editor",
|
||||
),
|
||||
"postKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdRetry": (
|
||||
RBACPermission.DATASET_EDIT,
|
||||
"dataset_editor",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _materialized_path(operation: KnowledgeFSOperation) -> str:
|
||||
segments = []
|
||||
for segment in operation.path.split("/"):
|
||||
if segment == "{revision}":
|
||||
segments.append("1")
|
||||
elif segment.startswith("{"):
|
||||
segments.append("00000000-0000-4000-8000-000000000001")
|
||||
else:
|
||||
segments.append(segment)
|
||||
return "/".join(segments)
|
||||
|
||||
|
||||
def _set_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
base_url: str | None = "http://knowledge-fs.test",
|
||||
sse_read_timeout_seconds: float = 90.0,
|
||||
timeout_seconds: float = 7.5,
|
||||
jwt_secret: str | None = _JWT_SECRET,
|
||||
) -> None:
|
||||
values = {
|
||||
"KNOWLEDGE_FS_BASE_URL": base_url,
|
||||
"KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS": sse_read_timeout_seconds,
|
||||
"KNOWLEDGE_FS_TIMEOUT_SECONDS": timeout_seconds,
|
||||
"KNOWLEDGE_FS_JWT_SECRET": SecretStr(jwt_secret) if jwt_secret is not None else None,
|
||||
}
|
||||
@@ -45,43 +212,68 @@ def _set_config(
|
||||
monkeypatch.setattr(f"services.knowledge_fs_proxy.dify_config.{name}", value, raising=False)
|
||||
|
||||
|
||||
def test_console_registry_starts_with_list_and_create_operations() -> None:
|
||||
assert tuple(operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS) == (
|
||||
"listKnowledgeSpaces",
|
||||
"createKnowledgeSpace",
|
||||
def _processing_task_events_path() -> str:
|
||||
operation = next(
|
||||
operation
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
if operation.operation_id == "getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents"
|
||||
)
|
||||
return _materialized_path(operation)
|
||||
|
||||
|
||||
def test_console_registry_exposes_only_the_new_rag_happy_path_operations() -> None:
|
||||
operation_ids = {operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS}
|
||||
|
||||
assert operation_ids == set(_HAPPY_PATH_OPERATION_IDS) | _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
|
||||
|
||||
def test_console_registry_exposes_existing_upstream_contracts_needed_by_all_happy_path_pages() -> None:
|
||||
operation_ids = {operation.operation_id for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS}
|
||||
|
||||
assert operation_ids >= _REQUIRED_EXPANDED_HAPPY_PATH_OPERATION_IDS
|
||||
|
||||
|
||||
def test_console_registry_preserves_explicit_scope_and_authorization_policies() -> None:
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS:
|
||||
is_read = operation.method == "GET"
|
||||
assert operation.required_scope == f"knowledge-spaces:{'read' if is_read else 'write'}"
|
||||
expected_policy = _OPERATION_AUTHORIZATION_POLICIES.get(operation.operation_id)
|
||||
if expected_policy is None and operation.operation_id in _EXPANDED_EXTERNAL_SOURCE_OPERATION_IDS:
|
||||
expected_policy = (RBACPermission.DATASET_EXTERNAL_CONNECT, "dataset_editor")
|
||||
if expected_policy is None:
|
||||
expected_policy = (
|
||||
(RBACPermission.DATASET_READONLY, "reader")
|
||||
if is_read
|
||||
else (RBACPermission.DATASET_EDIT, "dataset_editor")
|
||||
)
|
||||
assert (operation.rbac_permission, operation.legacy_role) == expected_policy
|
||||
assert operation.response_headers == ("x-trace-id",)
|
||||
|
||||
|
||||
def test_console_registry_preserves_special_transport_contracts() -> None:
|
||||
crawl_preview = get_knowledge_fs_operation(
|
||||
"POST",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/sources/"
|
||||
"00000000-0000-4000-8000-000000000002/crawl-preview",
|
||||
)
|
||||
selection = get_knowledge_fs_operation(
|
||||
"POST",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/source-workflows/"
|
||||
"00000000-0000-4000-8000-000000000002/selection",
|
||||
)
|
||||
events = get_knowledge_fs_operation(
|
||||
"GET",
|
||||
"knowledge-spaces/00000000-0000-4000-8000-000000000001/documents/"
|
||||
"00000000-0000-4000-8000-000000000002/processing-tasks/"
|
||||
"00000000-0000-4000-8000-000000000003/events",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "operation_id", "scope", "permission", "requires_dataset_editor"),
|
||||
[
|
||||
("GET", "listKnowledgeSpaces", "knowledge-spaces:read", RBACPermission.DATASET_READONLY, False),
|
||||
(
|
||||
"POST",
|
||||
"createKnowledgeSpace",
|
||||
"knowledge-spaces:write",
|
||||
RBACPermission.DATASET_CREATE_AND_MANAGEMENT,
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_console_registry_preserves_contract_and_policy(
|
||||
method: KnowledgeFSMethod,
|
||||
operation_id: str,
|
||||
scope: str,
|
||||
permission: RBACPermission,
|
||||
requires_dataset_editor: bool,
|
||||
) -> None:
|
||||
operation = get_knowledge_fs_operation(method, "knowledge-spaces")
|
||||
|
||||
assert operation.operation_id == operation_id
|
||||
assert operation.required_scope == scope
|
||||
assert operation.rbac_permission == permission
|
||||
assert operation.requires_dataset_editor is requires_dataset_editor
|
||||
assert operation.max_response_bytes == 1_048_576
|
||||
assert operation.request_headers == ("x-trace-id",)
|
||||
assert operation.response_headers == ("x-trace-id",)
|
||||
assert operation.response_media_types == ("application/json",)
|
||||
assert crawl_preview.request_headers == ("idempotency-key", "x-trace-id")
|
||||
assert selection.request_headers == ("idempotency-key", "x-trace-id")
|
||||
assert events.response_kind == "stream"
|
||||
assert events.max_response_bytes == 67_108_864
|
||||
assert events.request_headers == ("last-event-id", "x-trace-id")
|
||||
assert events.response_media_types == ("text/event-stream",)
|
||||
|
||||
|
||||
def test_unconfigured_kfs_is_rejected_before_external_io(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@@ -148,7 +340,112 @@ def test_proxy_forwards_only_registry_declared_headers(monkeypatch: pytest.Monke
|
||||
assert forward.call_args.kwargs["request_headers"] == {"x-trace-id": "trace-1"}
|
||||
|
||||
|
||||
def test_authorization_rejects_workspace_rbac_denial(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_authorized_proxy_does_not_repeat_workspace_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
check_access = MagicMock(return_value=True)
|
||||
forward = MagicMock(return_value=MagicMock())
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy._forward_knowledge_fs_request", forward)
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
|
||||
check_access.assert_called_once()
|
||||
assert forward.call_args.kwargs["account_id"] == "account-1"
|
||||
assert forward.call_args.kwargs["tenant_id"] == "tenant-1"
|
||||
assert forward.call_args.kwargs["method"] == "POST"
|
||||
assert forward.call_args.kwargs["path"] == "knowledge-spaces"
|
||||
|
||||
|
||||
def test_authorization_capability_cannot_be_constructed_directly() -> None:
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="must be created by workspace authorization"):
|
||||
KnowledgeFSAuthorization("account-1", "tenant-1", operation)
|
||||
|
||||
|
||||
def test_authorization_resolves_the_canonical_operation_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="dataset edit access"):
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("attribute", "value"),
|
||||
[
|
||||
("account_id", "account-2"),
|
||||
("tenant_id", "tenant-2"),
|
||||
("operation", get_knowledge_fs_operation("GET", "knowledge-spaces")),
|
||||
],
|
||||
)
|
||||
def test_authorization_capability_binding_cannot_be_mutated(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
attribute: str,
|
||||
value: object,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
with pytest.raises(AttributeError):
|
||||
setattr(authorization, attribute, value)
|
||||
|
||||
assert authorization.account_id == "account-1"
|
||||
assert authorization.tenant_id == "tenant-1"
|
||||
assert authorization.operation == get_knowledge_fs_operation("POST", "knowledge-spaces")
|
||||
|
||||
|
||||
def test_authorization_capability_cannot_be_reused(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
forward = MagicMock(return_value=MagicMock())
|
||||
monkeypatch.setattr(
|
||||
"services.knowledge_fs_proxy.RBACService.CheckAccess.check",
|
||||
MagicMock(return_value=True),
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy._forward_knowledge_fs_request", forward)
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method="POST",
|
||||
path="knowledge-spaces",
|
||||
)
|
||||
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="already been used"):
|
||||
proxy_authorized_knowledge_fs_request(authorization=authorization)
|
||||
forward.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", KNOWLEDGE_FS_CONSOLE_OPERATIONS, ids=lambda operation: operation.operation_id)
|
||||
def test_authorization_rejects_workspace_rbac_denial(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True)
|
||||
check_access = MagicMock(return_value=False)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
@@ -157,19 +454,28 @@ def test_authorization_rejects_workspace_rbac_denial(monkeypatch: pytest.MonkeyP
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
operation=get_knowledge_fs_operation("GET", "knowledge-spaces"),
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_called_once_with(
|
||||
"tenant-1",
|
||||
"account-1",
|
||||
scene="dataset_readonly",
|
||||
scene=operation.rbac_permission.value,
|
||||
resource_type="dataset",
|
||||
)
|
||||
|
||||
|
||||
def test_create_rejects_non_dataset_editor_before_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False)
|
||||
@pytest.mark.parametrize(
|
||||
"operation",
|
||||
tuple(operation for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS if operation.legacy_role == "dataset_editor"),
|
||||
ids=lambda operation: operation.operation_id,
|
||||
)
|
||||
def test_dataset_editor_operations_reject_legacy_viewers_before_rbac(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
|
||||
@@ -177,41 +483,66 @@ def test_create_rejects_non_dataset_editor_before_rbac(monkeypatch: pytest.Monke
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
operation=get_knowledge_fs_operation("POST", "knowledge-spaces"),
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
def test_authorization_uses_the_declared_editor_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False)
|
||||
def test_admin_operation_rejects_legacy_editors_before_rbac(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=True, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
operation = get_knowledge_fs_operation("POST", "knowledge-spaces")._replace(requires_dataset_editor=False)
|
||||
operation = get_knowledge_fs_operation(
|
||||
"PATCH", "knowledge-spaces/00000000-0000-4000-8000-000000000001/access-policy"
|
||||
)
|
||||
|
||||
authorize_knowledge_fs_request(account=account, tenant_id="tenant-1", operation=operation)
|
||||
with pytest.raises(KnowledgeFSAccessDeniedError, match="administration access"):
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_not_called()
|
||||
|
||||
|
||||
def test_authorization_uses_the_declared_reader_policy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
account = MagicMock(id="account-1", is_dataset_editor=False, is_admin_or_owner=False)
|
||||
check_access = MagicMock(return_value=True)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.RBACService.CheckAccess.check", check_access)
|
||||
operation = get_knowledge_fs_operation("GET", "knowledge-spaces")
|
||||
|
||||
authorize_knowledge_fs_request(
|
||||
account=account,
|
||||
tenant_id="tenant-1",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
)
|
||||
|
||||
check_access.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "expected_scope"),
|
||||
[("GET", "knowledge-spaces:read"), ("POST", "knowledge-spaces:write")],
|
||||
)
|
||||
@pytest.mark.parametrize("operation", KNOWLEDGE_FS_CONSOLE_OPERATIONS, ids=lambda operation: operation.operation_id)
|
||||
def test_auth_signs_current_principals_and_declared_scope(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
method: KnowledgeFSMethod,
|
||||
expected_scope: str,
|
||||
operation: KnowledgeFSOperation,
|
||||
) -> None:
|
||||
_set_config(monkeypatch)
|
||||
response = httpx.Response(200, content=b'{"items":[]}', headers={"Content-Type": "application/json"})
|
||||
response = httpx.Response(
|
||||
200,
|
||||
content=b"data" if operation.response_kind == "stream" else b'{"items":[]}',
|
||||
headers={"Content-Type": operation.response_media_types[0]},
|
||||
)
|
||||
request = MagicMock(return_value=response)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", request)
|
||||
|
||||
forward_knowledge_fs_request(
|
||||
account_id="account-1",
|
||||
method=method,
|
||||
path="knowledge-spaces",
|
||||
method=operation.method,
|
||||
path=_materialized_path(operation),
|
||||
tenant_id="tenant-1",
|
||||
)
|
||||
|
||||
@@ -226,7 +557,7 @@ def test_auth_signs_current_principals_and_declared_scope(
|
||||
assert claims["dify_account_id"] == "dify-account:account-1"
|
||||
assert claims["sub"] == "dify-workspace:tenant-1"
|
||||
assert claims["tenant_id"] == "tenant-1"
|
||||
assert claims["scopes"] == [expected_scope]
|
||||
assert claims["scopes"] == [operation.required_scope]
|
||||
assert claims["caller_kind"] == "interactive"
|
||||
assert claims["exp"] - claims["iat"] == 60
|
||||
|
||||
@@ -244,6 +575,69 @@ def test_buffered_response_rejects_non_empty_body_without_content_type(monkeypat
|
||||
assert response.is_closed
|
||||
|
||||
|
||||
def test_sse_response_remains_streaming_and_uses_the_dedicated_read_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_set_config(monkeypatch, sse_read_timeout_seconds=120.0)
|
||||
request = httpx.Request(
|
||||
"GET",
|
||||
"http://knowledge-fs.test/events",
|
||||
extensions={"timeout": {"connect": 7.5, "read": 7.5}},
|
||||
)
|
||||
response = httpx.Response(
|
||||
200,
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
request=request,
|
||||
stream=httpx.ByteStream(b"event: progress\n\n"),
|
||||
)
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", MagicMock(return_value=response))
|
||||
buffer_response = MagicMock()
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.buffer_response", buffer_response)
|
||||
|
||||
result = forward_knowledge_fs_request(
|
||||
account_id="account-dev",
|
||||
method="GET",
|
||||
path=_processing_task_events_path(),
|
||||
tenant_id="tenant-dev",
|
||||
)
|
||||
|
||||
assert result.response is response
|
||||
assert result.response_kind == "stream"
|
||||
assert request.extensions["timeout"]["read"] == 120.0
|
||||
assert not response.is_closed
|
||||
buffer_response.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("headers", "message"),
|
||||
[
|
||||
({"Content-Type": "application/json"}, "unsupported media type"),
|
||||
(
|
||||
{"Content-Type": "text/event-stream", "Content-Encoding": "gzip"},
|
||||
"unsupported encoding",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sse_response_rejects_invalid_stream_headers_and_closes_upstream(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
headers: dict[str, str],
|
||||
message: str,
|
||||
) -> None:
|
||||
_set_config(monkeypatch)
|
||||
response = httpx.Response(200, headers=headers, stream=httpx.ByteStream(b"data"))
|
||||
monkeypatch.setattr("services.knowledge_fs_proxy.ssrf_proxy.make_request", MagicMock(return_value=response))
|
||||
|
||||
with pytest.raises(KnowledgeFSTransportError, match=message):
|
||||
forward_knowledge_fs_request(
|
||||
account_id="account-dev",
|
||||
method="GET",
|
||||
path=_processing_task_events_path(),
|
||||
tenant_id="tenant-dev",
|
||||
)
|
||||
|
||||
assert response.is_closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "expected_exception"),
|
||||
[
|
||||
@@ -277,10 +671,10 @@ def test_transport_failures_are_normalized(
|
||||
("method", "path"),
|
||||
[
|
||||
("GET", "openapi.json"),
|
||||
("GET", "knowledge-spaces/space-1"),
|
||||
("GET", "knowledge-spaces/space-1/manifest"),
|
||||
("PATCH", "knowledge-spaces"),
|
||||
("POST", "queries"),
|
||||
("POST", "knowledge-spaces/space-1/documents"),
|
||||
("POST", "knowledge-spaces/space-1/uploads"),
|
||||
],
|
||||
)
|
||||
def test_unregistered_route_is_rejected_before_external_io(
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.
|
||||
// Do not edit it manually.
|
||||
|
||||
export const knowledgeFsSourceOpenapiSha256 =
|
||||
'f18910e9c45a64f0855e0643a7a626fb2889021b4f943458de86c6bd2469facb'
|
||||
export const knowledgeFsConsoleDeclarationsSha256 =
|
||||
'8bd1924747fdd0d478ca085817cbe000eb7e8630b2c6a03f4f13a6a0fac07946'
|
||||
export const knowledgeFsGeneratedArtifactSha256 = {
|
||||
'orpc.gen.ts': 'e0d9954f817e97a4e95dd38c4522fb403c8659d741ce485fa671b1b4ce90a540',
|
||||
'types.gen.ts': 'a558ab80f32a8555bb5b44b7a596ef4a4a7a8cb7904390993aabcc587915f530',
|
||||
'zod.gen.ts': '6aa3d3e768fe0008ca405ecbe423ee325bd8f65f745dd2d1c86f2bdb887a97fd',
|
||||
} as const
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,47 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
knowledgeFsGeneratedArtifactSha256,
|
||||
knowledgeFsSourceOpenapiSha256,
|
||||
} from './generated/knowledge-fs/metadata.gen'
|
||||
import { getStreamingOperationIds } from './scripts/knowledge-fs-contract-utils.mjs'
|
||||
|
||||
const packageRoot = dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
describe('KnowledgeFS contract generation', () => {
|
||||
it.each(['200', '2XX'])('detects an SSE response declared with %s', (status) => {
|
||||
expect(
|
||||
getStreamingOperationIds({
|
||||
paths: {
|
||||
'/tasks/{id}/events': {
|
||||
get: {
|
||||
operationId: 'streamTaskEvents',
|
||||
responses: {
|
||||
[status]: {
|
||||
content: {
|
||||
'text/event-stream': {},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toEqual(['streamTaskEvents'])
|
||||
})
|
||||
|
||||
it('matches the pinned source contract and committed generated artifacts', async () => {
|
||||
const lock = JSON.parse(
|
||||
await readFile(join(packageRoot, '../../api/knowledge-fs-contract.lock.json'), 'utf8'),
|
||||
)
|
||||
|
||||
expect(knowledgeFsSourceOpenapiSha256).toBe(lock.openapiSha256)
|
||||
for (const [fileName, expectedSha256] of Object.entries(knowledgeFsGeneratedArtifactSha256)) {
|
||||
const content = await readFile(join(packageRoot, 'generated/knowledge-fs', fileName))
|
||||
expect(createHash('sha256').update(content).digest('hex'), fileName).toBe(expectedSha256)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,8 +1,10 @@
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { getStreamingOperationIds } from './knowledge-fs-contract-utils.mjs'
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const workspaceRoot = resolve(packageRoot, '../..')
|
||||
@@ -31,11 +33,83 @@ try {
|
||||
run('pnpm', ['exec', 'openapi-ts', '-f', 'openapi-ts.knowledge-fs.config.ts'], packageRoot, {
|
||||
KNOWLEDGE_FS_OPENAPI: openapiPath,
|
||||
})
|
||||
await patchStreamingContracts(openapiPath)
|
||||
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs'], packageRoot)
|
||||
await writeContractMetadata(openapiPath, await generatedArtifactSha256())
|
||||
run('pnpm', ['exec', 'vp', 'fmt', 'generated/knowledge-fs/metadata.gen.ts'], packageRoot)
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { force: true, recursive: true })
|
||||
}
|
||||
|
||||
async function patchStreamingContracts(openapiPath) {
|
||||
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
|
||||
const streamingOperationIds = getStreamingOperationIds(document)
|
||||
if (streamingOperationIds.length === 0) return
|
||||
|
||||
const outputPath = join(packageRoot, 'generated/knowledge-fs/orpc.gen.ts')
|
||||
let source = await readFile(outputPath, 'utf8')
|
||||
source = replaceOnce(
|
||||
source,
|
||||
"import { oc } from '@orpc/contract'",
|
||||
"import { eventIterator, oc } from '@orpc/contract'",
|
||||
)
|
||||
|
||||
for (const operationId of streamingOperationIds) {
|
||||
const responseSchema = `z${capitalize(operationId)}Response`
|
||||
source = replaceOnce(
|
||||
source,
|
||||
`.output(${responseSchema})`,
|
||||
`.output(eventIterator(${responseSchema}))`,
|
||||
)
|
||||
}
|
||||
|
||||
await writeFile(outputPath, source)
|
||||
}
|
||||
|
||||
function capitalize(value) {
|
||||
return value.charAt(0).toUpperCase() + value.slice(1)
|
||||
}
|
||||
|
||||
function replaceOnce(source, target, replacement) {
|
||||
const firstIndex = source.indexOf(target)
|
||||
if (firstIndex === -1 || source.indexOf(target, firstIndex + target.length) !== -1)
|
||||
throw new Error(`Expected exactly one generated occurrence of ${target}`)
|
||||
|
||||
return source.slice(0, firstIndex) + replacement + source.slice(firstIndex + target.length)
|
||||
}
|
||||
|
||||
async function generatedArtifactSha256() {
|
||||
const generatedDirectory = join(packageRoot, 'generated/knowledge-fs')
|
||||
const fileNames = (await readdir(generatedDirectory))
|
||||
.filter((fileName) => fileName.endsWith('.gen.ts') && fileName !== 'metadata.gen.ts')
|
||||
.sort()
|
||||
|
||||
return Object.fromEntries(
|
||||
await Promise.all(
|
||||
fileNames.map(async (fileName) => [
|
||||
fileName,
|
||||
createHash('sha256')
|
||||
.update(await readFile(join(generatedDirectory, fileName)))
|
||||
.digest('hex'),
|
||||
]),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
async function writeContractMetadata(openapiPath, artifactSha256) {
|
||||
const document = JSON.parse(await readFile(openapiPath, 'utf8'))
|
||||
const source = [
|
||||
'// This file is auto-generated by scripts/generate-knowledge-fs-contract.mjs.',
|
||||
'// Do not edit it manually.',
|
||||
'',
|
||||
`export const knowledgeFsSourceOpenapiSha256 = ${JSON.stringify(document['x-dify-source-openapi-sha256'])}`,
|
||||
`export const knowledgeFsConsoleDeclarationsSha256 = ${JSON.stringify(document['x-dify-console-declarations-sha256'])}`,
|
||||
`export const knowledgeFsGeneratedArtifactSha256 = ${JSON.stringify(artifactSha256, null, 2)} as const`,
|
||||
'',
|
||||
].join('\n')
|
||||
await writeFile(join(packageRoot, 'generated/knowledge-fs/metadata.gen.ts'), source)
|
||||
}
|
||||
|
||||
function run(command, args, cwd, extraEnv = {}) {
|
||||
execFileSync(command, args, {
|
||||
cwd,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
export function getStreamingOperationIds(document) {
|
||||
return Object.values(document.paths ?? {})
|
||||
.flatMap((pathItem) =>
|
||||
Object.values(pathItem).flatMap((operation) => {
|
||||
if (typeof operation !== 'object' || operation === null) return []
|
||||
const isEventStream = Object.entries(operation.responses ?? {}).some(
|
||||
([status, response]) =>
|
||||
(status === '2XX' || /^2\d\d$/.test(status)) &&
|
||||
typeof response === 'object' &&
|
||||
response !== null &&
|
||||
'text/event-stream' in (response.content ?? {}),
|
||||
)
|
||||
return isEventStream && typeof operation.operationId === 'string'
|
||||
? [operation.operationId]
|
||||
: []
|
||||
}),
|
||||
)
|
||||
.sort()
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { ApiBasedExtensionResponse } from '@dify/contracts/api/console/api-based-extension/types.gen'
|
||||
import type { TagResponse as Tag } from '@dify/contracts/api/console/tags/types.gen'
|
||||
import type { DocumentProcessingTaskEvent } from '@dify/contracts/knowledge-fs/types.gen'
|
||||
import type { MutationFunctionContext, QueryFunctionContext } from '@tanstack/react-query'
|
||||
import type { consoleQuery as ConsoleQuery } from './client'
|
||||
import { QueryClient } from '@tanstack/react-query'
|
||||
@@ -358,6 +359,74 @@ describe('consoleQuery transport context', () => {
|
||||
)
|
||||
expect(request.mock.calls[0]![0]).not.toContain('ids%5B0%5D')
|
||||
})
|
||||
|
||||
it('should consume KnowledgeFS processing events through the generated stream contract', async () => {
|
||||
const request = vi.fn().mockResolvedValue(
|
||||
new Response(
|
||||
[
|
||||
'id: task-1:1',
|
||||
'event: message',
|
||||
'data: {"event":"progress","data":{"progressPercent":25,"stage":"parsed","state":"running","updatedAt":"2026-07-22T10:00:00.000Z"}}',
|
||||
'',
|
||||
'id: task-1:terminal',
|
||||
'event: message',
|
||||
'data: {"event":"terminal","data":{"state":"succeeded"}}',
|
||||
'',
|
||||
'',
|
||||
].join('\n'),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'text/event-stream',
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
const consoleQuery = await loadConsoleQueryWithRequest(request)
|
||||
const queryOptions =
|
||||
consoleQuery.knowledgeFs.getKnowledgeSpacesByIdDocumentsByDocumentIdProcessingTasksByTaskIdEvents.experimental_streamedOptions(
|
||||
{
|
||||
input: {
|
||||
headers: {
|
||||
'last-event-id': 'task-1:0',
|
||||
},
|
||||
params: {
|
||||
documentId: 'document-1',
|
||||
id: 'space-1',
|
||||
taskId: 'task-1',
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
const events = await queryOptions.queryFn({
|
||||
client: new QueryClient(),
|
||||
signal: new AbortController().signal,
|
||||
} as QueryFunctionContext)
|
||||
|
||||
expectTypeOf(events[0]!).toMatchTypeOf<DocumentProcessingTaskEvent>()
|
||||
expect(events).toEqual([
|
||||
{
|
||||
data: {
|
||||
progressPercent: 25,
|
||||
stage: 'parsed',
|
||||
state: 'running',
|
||||
updatedAt: '2026-07-22T10:00:00.000Z',
|
||||
},
|
||||
event: 'progress',
|
||||
},
|
||||
{ data: { state: 'succeeded' }, event: 'terminal' },
|
||||
])
|
||||
expect(request).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
'/knowledge-fs/knowledge-spaces/space-1/documents/document-1/processing-tasks/task-1/events',
|
||||
),
|
||||
expect.any(Object),
|
||||
expect.objectContaining({
|
||||
fetchCompat: true,
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
// Scenario: console OpenAPI query arrays follow backend parser expectations.
|
||||
|
||||
Reference in New Issue
Block a user