Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
49b638a099 | ||
|
|
99d9a6f6a2 | ||
|
|
83f6e7daf9 | ||
|
|
e8de10a3b5 | ||
|
|
f5ab5e7eb3 | ||
|
|
0c40e1c2a0 | ||
|
|
c29d76757e | ||
|
|
91c1d3ad81 | ||
|
|
57b02e341c | ||
|
|
b94ff65e9f | ||
|
|
678260e34e | ||
|
|
739e34d08a | ||
|
|
825fb9cb89 | ||
|
|
0e1f19a380 | ||
|
|
332d1ea533 | ||
|
|
9cdeffd0b1 | ||
|
|
09ef785a20 | ||
|
|
d2788d7aba | ||
|
|
cee90a4e82 | ||
|
|
b2710b875b | ||
|
|
6464255d33 | ||
|
|
50face5760 |
@@ -166,6 +166,7 @@
|
||||
|
||||
# Frontend - App - API Documentation
|
||||
/web/app/components/develop/ @JzoNgKVO @iamjoel
|
||||
/web/app/components/develop/template/*.mdx @JzoNgKVO @iamjoel @RiskeyL
|
||||
|
||||
# Frontend - App - Logs and Annotations
|
||||
/web/app/components/app/workflow-log/ @JzoNgKVO @iamjoel
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ COPY api/providers ./providers
|
||||
COPY dify-agent/pyproject.toml dify-agent/README.md /app/dify-agent/
|
||||
COPY dify-agent/src /app/dify-agent/src
|
||||
# Trust the checked-in lock during image builds; local path sources are copied from the repository context.
|
||||
RUN uv sync --frozen --no-dev
|
||||
RUN uv sync --frozen --no-dev --no-editable
|
||||
|
||||
# production stage
|
||||
FROM base AS production
|
||||
|
||||
@@ -38,6 +38,8 @@ from clients.agent_backend.request_builder import (
|
||||
AgentBackendOutputConfig,
|
||||
AgentBackendRunRequestBuilder,
|
||||
AgentBackendWorkflowNodeRunInput,
|
||||
CleanupLayerSpec,
|
||||
extract_cleanup_layer_specs,
|
||||
redact_for_agent_backend_log,
|
||||
)
|
||||
|
||||
@@ -68,9 +70,11 @@ __all__ = [
|
||||
"AgentBackendTransportError",
|
||||
"AgentBackendValidationError",
|
||||
"AgentBackendWorkflowNodeRunInput",
|
||||
"CleanupLayerSpec",
|
||||
"DifyAgentBackendRunClient",
|
||||
"FakeAgentBackendRunClient",
|
||||
"FakeAgentBackendScenario",
|
||||
"create_agent_backend_run_client",
|
||||
"extract_cleanup_layer_specs",
|
||||
"redact_for_agent_backend_log",
|
||||
]
|
||||
|
||||
@@ -20,6 +20,8 @@ from dify_agent.protocol import (
|
||||
RunEvent,
|
||||
RunFailedEvent,
|
||||
RunFailedEventData,
|
||||
RunPausedEvent,
|
||||
RunPausedEventData,
|
||||
RunStartedEvent,
|
||||
RunStatusResponse,
|
||||
RunSucceededEvent,
|
||||
@@ -34,6 +36,7 @@ class FakeAgentBackendScenario(StrEnum):
|
||||
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
PAUSED = "paused"
|
||||
|
||||
|
||||
class FakeAgentBackendRunClient:
|
||||
@@ -89,6 +92,13 @@ class FakeAgentBackendRunClient:
|
||||
updated_at=_FIXED_TIME,
|
||||
error="fake failure",
|
||||
)
|
||||
case FakeAgentBackendScenario.PAUSED:
|
||||
return RunStatusResponse(
|
||||
run_id=run_id,
|
||||
status="paused",
|
||||
created_at=_FIXED_TIME,
|
||||
updated_at=_FIXED_TIME,
|
||||
)
|
||||
|
||||
def _events(self, run_id: str) -> tuple[RunEvent, ...]:
|
||||
match self.scenario:
|
||||
@@ -115,3 +125,17 @@ class FakeAgentBackendRunClient:
|
||||
data=RunFailedEventData(error="fake failure", reason="unit_test"),
|
||||
),
|
||||
)
|
||||
case FakeAgentBackendScenario.PAUSED:
|
||||
return (
|
||||
RunStartedEvent(id="1-0", run_id=run_id, created_at=_FIXED_TIME),
|
||||
RunPausedEvent(
|
||||
id="2-0",
|
||||
run_id=run_id,
|
||||
created_at=_FIXED_TIME,
|
||||
data=RunPausedEventData(
|
||||
reason="human_input_required",
|
||||
message="Agent requested human input.",
|
||||
session_snapshot=CompositorSessionSnapshot(layers=[]),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -11,11 +11,13 @@ composition-driven.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import ClassVar
|
||||
from typing import ClassVar, cast
|
||||
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
from agenton.compositor.schemas import LayerSessionSnapshot
|
||||
from agenton.layers import ExitIntent
|
||||
from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID, PromptLayerConfig
|
||||
from agenton_collections.layers.pydantic_ai import PYDANTIC_AI_HISTORY_LAYER_TYPE_ID
|
||||
from dify_agent.layers.dify_plugin import (
|
||||
DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
|
||||
DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
|
||||
@@ -29,6 +31,7 @@ from dify_agent.layers.execution_context import (
|
||||
)
|
||||
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID, DifyOutputLayerConfig
|
||||
from dify_agent.protocol import (
|
||||
DIFY_AGENT_HISTORY_LAYER_ID,
|
||||
DIFY_AGENT_MODEL_LAYER_ID,
|
||||
DIFY_AGENT_OUTPUT_LAYER_ID,
|
||||
CreateRunRequest,
|
||||
@@ -45,6 +48,84 @@ WORKFLOW_USER_PROMPT_LAYER_ID = "workflow_user_prompt"
|
||||
DIFY_EXECUTION_CONTEXT_LAYER_ID = "execution_context"
|
||||
DIFY_PLUGIN_TOOLS_LAYER_ID = "tools"
|
||||
|
||||
# Layer types that hold credentials in their per-run config. These are excluded
|
||||
# from the cleanup-replay composition (and from the snapshot that is sent with
|
||||
# the cleanup request) because we deliberately do not persist plaintext
|
||||
# credentials between runs.
|
||||
_CLEANUP_EXCLUDED_LAYER_TYPES: tuple[str, ...] = (
|
||||
DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
|
||||
DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
|
||||
)
|
||||
|
||||
|
||||
class CleanupLayerSpec(BaseModel):
|
||||
"""One layer node replayed by an Agent backend cleanup-only run.
|
||||
|
||||
Cleanup composition cannot include credential-bearing plugin layers, so we
|
||||
persist only the non-plugin layer specs together with the original config.
|
||||
Storing the config (rather than just ``name``/``type``) means cleanup does
|
||||
not depend on the original build-time inputs being re-derivable.
|
||||
"""
|
||||
|
||||
name: str
|
||||
type: str
|
||||
deps: dict[str, str] = Field(default_factory=dict)
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
config: JsonValue = None
|
||||
|
||||
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
def extract_cleanup_layer_specs(composition: RunComposition) -> list[CleanupLayerSpec]:
|
||||
"""Project the in-flight composition into the persistable cleanup spec list.
|
||||
|
||||
Plugin layers are intentionally dropped (their configs hold credentials and
|
||||
the lifecycle contract says "do not include an LLM layer" during cleanup).
|
||||
The filtered names must later drive snapshot filtering so the agenton
|
||||
compositor's name-order check still passes for the cleanup run.
|
||||
"""
|
||||
excluded = set(_CLEANUP_EXCLUDED_LAYER_TYPES)
|
||||
specs: list[CleanupLayerSpec] = []
|
||||
for layer in composition.layers:
|
||||
if layer.type in excluded:
|
||||
continue
|
||||
config_value: JsonValue = None
|
||||
if isinstance(layer.config, BaseModel):
|
||||
config_value = layer.config.model_dump(mode="json", warnings=False)
|
||||
else:
|
||||
# ``RunLayerSpec.config`` is typed as ``LayerConfigInput`` which
|
||||
# includes ``Mapping[str, object] | bytes``. In the cleanup-replay
|
||||
# pipeline our builder only emits BaseModel-derived configs or
|
||||
# ``None``, so the wider input alias narrows safely here.
|
||||
config_value = cast(JsonValue, layer.config)
|
||||
specs.append(
|
||||
CleanupLayerSpec(
|
||||
name=layer.name,
|
||||
type=layer.type,
|
||||
deps=dict(layer.deps),
|
||||
metadata=dict(layer.metadata),
|
||||
config=config_value,
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
|
||||
def _filter_snapshot_to_specs(
|
||||
snapshot: CompositorSessionSnapshot,
|
||||
specs: list[CleanupLayerSpec],
|
||||
) -> CompositorSessionSnapshot:
|
||||
"""Keep only snapshot layers whose names appear in the cleanup spec list.
|
||||
|
||||
The agenton compositor rejects a snapshot whose layer-name sequence does
|
||||
not match the active composition exactly. Cleanup-replay drops plugin
|
||||
layers, so we must drop the matching snapshot entries here.
|
||||
"""
|
||||
kept_names = {spec.name for spec in specs}
|
||||
filtered_layers: list[LayerSessionSnapshot] = [layer for layer in snapshot.layers if layer.name in kept_names]
|
||||
if len(filtered_layers) == len(snapshot.layers):
|
||||
return snapshot
|
||||
return CompositorSessionSnapshot(schema_version=snapshot.schema_version, layers=filtered_layers)
|
||||
|
||||
|
||||
class AgentBackendModelConfig(BaseModel):
|
||||
"""API-side model/plugin selection before it is converted to Dify Agent layers."""
|
||||
@@ -86,7 +167,8 @@ class AgentBackendWorkflowNodeRunInput(BaseModel):
|
||||
output: AgentBackendOutputConfig | None = None
|
||||
tools: DifyPluginToolsLayerConfig | None = None
|
||||
session_snapshot: CompositorSessionSnapshot | None = None
|
||||
suspend_on_exit: bool = False
|
||||
include_history: bool = True
|
||||
suspend_on_exit: bool = True
|
||||
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
||||
|
||||
model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
||||
@@ -102,6 +184,50 @@ class AgentBackendWorkflowNodeRunInput(BaseModel):
|
||||
class AgentBackendRunRequestBuilder:
|
||||
"""Converts API product state into the public ``dify-agent`` run protocol."""
|
||||
|
||||
def build_cleanup_request(
|
||||
self,
|
||||
*,
|
||||
session_snapshot: CompositorSessionSnapshot,
|
||||
composition_layer_specs: list[CleanupLayerSpec],
|
||||
idempotency_key: str | None = None,
|
||||
metadata: dict[str, JsonValue] | None = None,
|
||||
) -> CreateRunRequest:
|
||||
"""Build a lifecycle-only cleanup request that replays the prior layers.
|
||||
|
||||
The agenton compositor enforces that the session snapshot's layer names
|
||||
match the active composition in order, so cleanup must replay the same
|
||||
non-plugin layer graph that produced the snapshot. Plugin layers
|
||||
(``dify.plugin.llm``, ``dify.plugin.tools``) are excluded from both the
|
||||
composition and the snapshot before submission because their configs
|
||||
require credentials that are not persisted between runs.
|
||||
"""
|
||||
if not composition_layer_specs:
|
||||
raise ValueError(
|
||||
"build_cleanup_request requires composition_layer_specs; an empty "
|
||||
"composition would fail the agent backend's snapshot validation."
|
||||
)
|
||||
request_metadata = dict(metadata or {})
|
||||
request_metadata["agent_backend_lifecycle"] = "session_cleanup"
|
||||
layers = [
|
||||
RunLayerSpec(
|
||||
name=spec.name,
|
||||
type=spec.type,
|
||||
deps=dict(spec.deps),
|
||||
metadata=dict(spec.metadata),
|
||||
config=spec.config,
|
||||
)
|
||||
for spec in composition_layer_specs
|
||||
]
|
||||
filtered_snapshot = _filter_snapshot_to_specs(session_snapshot, composition_layer_specs)
|
||||
return CreateRunRequest(
|
||||
composition=RunComposition(layers=layers),
|
||||
purpose="workflow_node",
|
||||
idempotency_key=idempotency_key,
|
||||
metadata=request_metadata,
|
||||
session_snapshot=filtered_snapshot,
|
||||
on_exit=LayerExitSignals(default=ExitIntent.DELETE),
|
||||
)
|
||||
|
||||
def build_for_workflow_node(self, run_input: AgentBackendWorkflowNodeRunInput) -> CreateRunRequest:
|
||||
"""Build a workflow Agent Node run request without defining another wire schema."""
|
||||
layers: list[RunLayerSpec] = []
|
||||
@@ -135,6 +261,20 @@ class AgentBackendRunRequestBuilder:
|
||||
metadata=run_input.metadata,
|
||||
config=run_input.execution_context,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
if run_input.include_history:
|
||||
layers.append(
|
||||
RunLayerSpec(
|
||||
name=DIFY_AGENT_HISTORY_LAYER_ID,
|
||||
type=PYDANTIC_AI_HISTORY_LAYER_TYPE_ID,
|
||||
metadata={**run_input.metadata, "origin": "agent_session_history"},
|
||||
)
|
||||
)
|
||||
|
||||
layers.extend(
|
||||
[
|
||||
RunLayerSpec(
|
||||
name=DIFY_AGENT_MODEL_LAYER_ID,
|
||||
type=DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
|
||||
|
||||
@@ -4,6 +4,12 @@ CLI command modules extracted from `commands.py`.
|
||||
|
||||
from .account import create_tenant, reset_email, reset_password
|
||||
from .data_migrate import data_migrate, legacy_model_types
|
||||
from .data_migration import (
|
||||
export_migration_data,
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
)
|
||||
from .plugin import (
|
||||
extract_plugins,
|
||||
extract_unique_plugins,
|
||||
@@ -26,7 +32,12 @@ from .retention import (
|
||||
restore_workflow_runs,
|
||||
)
|
||||
from .storage import clear_orphaned_file_records, file_usage, migrate_oss, remove_orphaned_files_on_storage
|
||||
from .system import convert_to_agent_apps, fix_app_site_missing, reset_encrypt_key_pair, upgrade_db
|
||||
from .system import (
|
||||
convert_to_agent_apps,
|
||||
fix_app_site_missing,
|
||||
reset_encrypt_key_pair,
|
||||
upgrade_db,
|
||||
)
|
||||
from .vector import (
|
||||
add_qdrant_index,
|
||||
migrate_annotation_vector_database,
|
||||
@@ -48,10 +59,13 @@ __all__ = [
|
||||
"data_migrate",
|
||||
"delete_archived_workflow_runs",
|
||||
"export_app_messages",
|
||||
"export_migration_data",
|
||||
"export_migration_data_template",
|
||||
"extract_plugins",
|
||||
"extract_unique_plugins",
|
||||
"file_usage",
|
||||
"fix_app_site_missing",
|
||||
"import_migration_data",
|
||||
"install_plugins",
|
||||
"install_rag_pipeline_plugins",
|
||||
"legacy_model_types",
|
||||
@@ -59,6 +73,7 @@ __all__ = [
|
||||
"migrate_data_for_plugin",
|
||||
"migrate_knowledge_vector_database",
|
||||
"migrate_oss",
|
||||
"migration_data_wizard",
|
||||
"old_metadata_migration",
|
||||
"remove_orphaned_files_on_storage",
|
||||
"reset_email",
|
||||
|
||||
@@ -0,0 +1,754 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
import click
|
||||
import sqlalchemy as sa
|
||||
import yaml
|
||||
|
||||
from extensions.ext_database import db
|
||||
from models import Tenant
|
||||
from models.model import App
|
||||
from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider
|
||||
from services.app_dsl_service import AppDslService
|
||||
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService
|
||||
from services.data_migration.entities import (
|
||||
DependencyKind,
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
ReportContext,
|
||||
ResourceReportItem,
|
||||
)
|
||||
from services.data_migration.export_service import ExportConfigParser, MigrationExportService
|
||||
from services.data_migration.import_service import ImportRequest, MigrationImportService
|
||||
from services.data_migration.package_service import MigrationPackageService
|
||||
from services.data_migration.report_service import MigrationReportService
|
||||
|
||||
ID_STRATEGY_CHOICES = ["preserve-id", "generate-new-id"]
|
||||
CONFLICT_STRATEGY_CHOICES = ["fail", "skip", "update"]
|
||||
SUPPORTED_WIZARD_APP_MODES = ["workflow", "advanced-chat"]
|
||||
WizardToolMap = dict[str, dict[str, str | None]]
|
||||
WizardToolSelection = dict[str, list[str]]
|
||||
|
||||
|
||||
def _scripted_export_template() -> dict[str, Any]:
|
||||
return {
|
||||
"source_tenant": {
|
||||
"mode": "single",
|
||||
"id": "",
|
||||
"name": "admin's Workspace",
|
||||
},
|
||||
"apps": {
|
||||
"modes": ["workflow", "advanced-chat"],
|
||||
"ids": [],
|
||||
"all": True,
|
||||
},
|
||||
"include_referenced_tools": True,
|
||||
"additional_tools": {
|
||||
"api_tools": [],
|
||||
"workflow_tools": [],
|
||||
"mcp_tools": [],
|
||||
},
|
||||
"include_secrets": False,
|
||||
"import_options": {
|
||||
"create_app_api_token_on_import": False,
|
||||
"id_strategy": "preserve-id",
|
||||
"conflict_strategy": "fail",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@click.command("app-migration-template", help="Print or write a scripted export config JSON template.")
|
||||
@click.option(
|
||||
"--output",
|
||||
"output_file",
|
||||
required=False,
|
||||
type=click.Path(dir_okay=False),
|
||||
help="Path to write the export config JSON template. Prints to stdout when omitted.",
|
||||
)
|
||||
@click.option("--overwrite", is_flag=True, default=False, help="Overwrite output if it already exists.")
|
||||
def export_migration_data_template(output_file: str | None, overwrite: bool) -> None:
|
||||
template_json = json.dumps(_scripted_export_template(), indent=2, ensure_ascii=False) + "\n"
|
||||
if output_file is None:
|
||||
click.echo(template_json, nl=False)
|
||||
return
|
||||
path = Path(output_file)
|
||||
if path.exists() and not overwrite:
|
||||
raise click.ClickException(f"Output file already exists: {output_file}")
|
||||
path.write_text(template_json)
|
||||
click.echo(click.style(f"Output written to {output_file}", fg="green"))
|
||||
|
||||
|
||||
@click.command("export-app-migration", help="Export workflow migration data to a versioned JSON package.")
|
||||
@click.option(
|
||||
"--input",
|
||||
"input_file",
|
||||
required=False,
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
help="Path to export config JSON.",
|
||||
)
|
||||
@click.option(
|
||||
"--output",
|
||||
"output_file",
|
||||
required=False,
|
||||
type=click.Path(dir_okay=False),
|
||||
help="Path to migration package JSON.",
|
||||
)
|
||||
@click.option("--overwrite", is_flag=True, default=False, help="Overwrite output if it already exists.")
|
||||
def export_migration_data(input_file: str | None, output_file: str | None, overwrite: bool) -> None:
|
||||
try:
|
||||
_require_options(("--input", input_file), ("--output", output_file))
|
||||
assert input_file is not None
|
||||
assert output_file is not None
|
||||
raw_config = _load_json_object(input_file, "Export config")
|
||||
selection = ExportConfigParser().parse(raw_config)
|
||||
result = MigrationExportService().export(selection)
|
||||
MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite)
|
||||
click.echo(click.style(f"Output written to {output_file}", fg="green"))
|
||||
_render_report(result.report_items, context=_with_output_path(result.report_context, output_file))
|
||||
except MigrationDataError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
@click.command("import-app-migration", help="Import a versioned migration data package.")
|
||||
@click.option(
|
||||
"--input",
|
||||
"input_file",
|
||||
required=False,
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
help="Path to migration package JSON.",
|
||||
)
|
||||
@click.option("--target-tenant", default=None, help="Target tenant/workspace name. Overrides package metadata.")
|
||||
@click.option("--operator-email", default=None, help="Operator account email in the target tenant.")
|
||||
@click.option(
|
||||
"--id-strategy",
|
||||
default=None,
|
||||
type=click.Choice(ID_STRATEGY_CHOICES),
|
||||
help="Override package ID strategy.",
|
||||
)
|
||||
@click.option(
|
||||
"--conflict-strategy",
|
||||
default=None,
|
||||
type=click.Choice(CONFLICT_STRATEGY_CHOICES),
|
||||
help="Override package conflict strategy.",
|
||||
)
|
||||
@click.option(
|
||||
"--create-app-api-token-on-import/--no-create-app-api-token-on-import",
|
||||
default=None,
|
||||
help="Override package app API token creation behavior.",
|
||||
)
|
||||
def import_migration_data(
|
||||
input_file: str | None,
|
||||
target_tenant: str | None,
|
||||
operator_email: str | None,
|
||||
id_strategy: str | None,
|
||||
conflict_strategy: str | None,
|
||||
create_app_api_token_on_import: bool | None,
|
||||
) -> None:
|
||||
try:
|
||||
_require_options(("--input", input_file))
|
||||
assert input_file is not None
|
||||
package = MigrationPackageService().load_package(input_file)
|
||||
result = MigrationImportService().import_package(
|
||||
ImportRequest(
|
||||
package=package,
|
||||
cli_target_tenant=target_tenant,
|
||||
operator_email=operator_email,
|
||||
options_override=_build_options_override(
|
||||
package.metadata.import_options,
|
||||
id_strategy=id_strategy,
|
||||
conflict_strategy=conflict_strategy,
|
||||
create_app_api_token_on_import=create_app_api_token_on_import,
|
||||
),
|
||||
)
|
||||
)
|
||||
_render_report(result.report_items, context=result.report_context)
|
||||
except MigrationDataError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def parse_index_selection(raw: str, values: list[str]) -> list[str]:
|
||||
normalized = raw.strip().lower()
|
||||
if normalized == "all":
|
||||
return values
|
||||
|
||||
selected: list[str] = []
|
||||
for part in raw.split(","):
|
||||
stripped = part.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
try:
|
||||
index = int(stripped)
|
||||
except ValueError as exc:
|
||||
raise click.ClickException(f"Selection must be 'all' or comma-separated numbers: {raw}") from exc
|
||||
if index < 1 or index > len(values):
|
||||
raise click.ClickException(f"Selection index out of range: {index}")
|
||||
selected.append(values[index - 1])
|
||||
return list(dict.fromkeys(selected))
|
||||
|
||||
|
||||
def _print_wizard_step(title: str) -> None:
|
||||
click.echo("")
|
||||
click.echo(f"==== {title} ====")
|
||||
|
||||
|
||||
def _print_wizard_substep(title: str) -> None:
|
||||
click.echo("")
|
||||
click.echo(f"-- {title} --")
|
||||
|
||||
|
||||
@click.command("app-migration-wizard", help="Interactively export workflow migration data.")
|
||||
def migration_data_wizard() -> None:
|
||||
try:
|
||||
tenant = _prompt_source_tenant()
|
||||
apps = _eligible_apps_for_tenant(tenant.id)
|
||||
app_ids = _prompt_app_ids(apps)
|
||||
_print_wizard_step("Referenced Tools")
|
||||
include_referenced_tools = click.confirm(
|
||||
"Automatically export tools referenced by selected apps? [y/n, default: y]",
|
||||
default=True,
|
||||
show_default=False,
|
||||
)
|
||||
auto_tools = _discover_auto_tools([app for app in apps if app.id in set(app_ids)], include_referenced_tools)
|
||||
auto_tools = _resolve_auto_tool_names(tenant.id, auto_tools)
|
||||
_print_auto_tools(auto_tools)
|
||||
additional_tools = _prompt_additional_tools(tenant.id, auto_tools)
|
||||
include_secrets, create_tokens, id_strategy, conflict_strategy = _prompt_import_options()
|
||||
_print_wizard_step("Output")
|
||||
output_file, overwrite = _prompt_output_file()
|
||||
|
||||
selection = ExportConfigParser().parse(
|
||||
{
|
||||
"source_tenant": {"mode": "single", "id": tenant.id, "name": tenant.name},
|
||||
"apps": {"ids": app_ids, "all": False},
|
||||
"include_referenced_tools": include_referenced_tools,
|
||||
"additional_tools": additional_tools,
|
||||
"include_secrets": include_secrets,
|
||||
"import_options": {
|
||||
"create_app_api_token_on_import": create_tokens,
|
||||
"id_strategy": id_strategy,
|
||||
"conflict_strategy": conflict_strategy,
|
||||
},
|
||||
}
|
||||
)
|
||||
_confirm_wizard_summary(
|
||||
tenant_name=tenant.name,
|
||||
app_names=[app.name for app in apps if app.id in set(app_ids)],
|
||||
auto_tools=auto_tools,
|
||||
additional_tools=additional_tools,
|
||||
manual_labels=_selected_tool_labels_for_tenant(tenant.id, additional_tools),
|
||||
include_referenced_tools=include_referenced_tools,
|
||||
include_secrets=include_secrets,
|
||||
create_tokens=create_tokens,
|
||||
id_strategy=id_strategy,
|
||||
conflict_strategy=conflict_strategy,
|
||||
output_file=output_file,
|
||||
)
|
||||
result = MigrationExportService().export(selection)
|
||||
MigrationPackageService().save_package(result.package, output_file, overwrite=overwrite)
|
||||
click.echo(click.style(f"Output written to {output_file}", fg="green"))
|
||||
_print_wizard_step("Report")
|
||||
_render_report(result.report_items, context=_with_output_path(result.report_context, output_file))
|
||||
except MigrationDataError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
def _load_json_object(path: str, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
with Path(path).open(encoding="utf-8") as file:
|
||||
raw = json.load(file)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise MigrationDataError(f"{label} JSON is invalid: {exc.msg}") from exc
|
||||
if not isinstance(raw, dict):
|
||||
raise MigrationDataError(f"{label} JSON must be an object.")
|
||||
return raw
|
||||
|
||||
|
||||
def _require_options(*options: tuple[str, object | None]) -> None:
|
||||
missing_options = [name for name, value in options if value is None]
|
||||
if missing_options:
|
||||
raise click.UsageError(f"Missing option(s): {', '.join(missing_options)}.")
|
||||
|
||||
|
||||
def _build_options_override(
|
||||
package_options: ImportOptions,
|
||||
*,
|
||||
id_strategy: str | None,
|
||||
conflict_strategy: str | None,
|
||||
create_app_api_token_on_import: bool | None,
|
||||
) -> ImportOptions | None:
|
||||
if id_strategy is None and conflict_strategy is None and create_app_api_token_on_import is None:
|
||||
return None
|
||||
return ImportOptions.from_mapping(
|
||||
{
|
||||
"id_strategy": id_strategy or package_options.id_strategy,
|
||||
"conflict_strategy": conflict_strategy or package_options.conflict_strategy,
|
||||
"create_app_api_token_on_import": (
|
||||
create_app_api_token_on_import
|
||||
if create_app_api_token_on_import is not None
|
||||
else package_options.create_app_api_token_on_import
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _prompt_source_tenant() -> Tenant:
|
||||
tenants = list(db.session.scalars(sa.select(Tenant).order_by(Tenant.name.asc())).all())
|
||||
if not tenants:
|
||||
raise MigrationDataError("No tenants found.")
|
||||
|
||||
_print_wizard_step("Source Tenant")
|
||||
click.echo("Source tenants:")
|
||||
for index, tenant in enumerate(tenants, 1):
|
||||
click.echo(f"{index}. {tenant.name} ({tenant.id})")
|
||||
|
||||
tenant_index = click.prompt("Select one source tenant by number", type=int, default=1, show_default=True)
|
||||
if tenant_index < 1 or tenant_index > len(tenants):
|
||||
raise click.ClickException(f"Selection index out of range: {tenant_index}")
|
||||
return tenants[tenant_index - 1]
|
||||
|
||||
|
||||
def _eligible_apps_for_tenant(tenant_id: str) -> list[App]:
|
||||
return list(
|
||||
db.session.scalars(
|
||||
sa.select(App)
|
||||
.where(App.tenant_id == tenant_id, App.mode.in_(SUPPORTED_WIZARD_APP_MODES))
|
||||
.order_by(App.name.asc())
|
||||
).all()
|
||||
)
|
||||
|
||||
|
||||
def _prompt_app_ids(apps: list[App]) -> list[str]:
|
||||
if not apps:
|
||||
raise MigrationDataError("No workflow or advanced-chat apps found for the selected tenant.")
|
||||
|
||||
_print_wizard_step("App Selection")
|
||||
click.echo("Currently supported app types: workflow and chatflow.")
|
||||
click.echo("Workflow/chatflow apps:")
|
||||
for index, app in enumerate(apps, 1):
|
||||
mode = app.mode.value if hasattr(app.mode, "value") else app.mode
|
||||
click.echo(f"{index}. {app.name} [{mode}] ({app.id})")
|
||||
app_ids = parse_index_selection(
|
||||
click.prompt("Select apps by number, comma-separated numbers, or all", default="all"),
|
||||
[app.id for app in apps],
|
||||
)
|
||||
selected_apps = [app for app in apps if app.id in set(app_ids)]
|
||||
click.echo("Selected apps:")
|
||||
for app in selected_apps:
|
||||
click.echo(f"- {app.name} ({app.id})")
|
||||
return app_ids
|
||||
|
||||
|
||||
def _prompt_import_options() -> tuple[bool, bool, str, str]:
|
||||
_print_wizard_step("Import Options")
|
||||
_print_wizard_substep("Secrets")
|
||||
click.echo("Secrets include workflow/app DSL secret values, custom API tool credentials,")
|
||||
click.echo("and full MCP provider connection data such as server URL, headers, authentication, and tool list.")
|
||||
click.echo("If you choose no, credentials are omitted or masked,")
|
||||
click.echo("and MCP providers are exported as dependency metadata only.")
|
||||
click.echo("Treat the output JSON as sensitive if you choose yes.")
|
||||
include_secrets = click.confirm(
|
||||
"Include secrets in output JSON? [y/n, default: n]",
|
||||
default=False,
|
||||
show_default=False,
|
||||
)
|
||||
_print_wizard_substep("App API Tokens")
|
||||
click.echo("When enabled, import will create an app API token if the imported app has none,")
|
||||
click.echo("or reuse an existing app API token if one already exists.")
|
||||
create_tokens = click.confirm(
|
||||
"Create or reuse app API tokens during import? [y/n, default: n]",
|
||||
default=False,
|
||||
show_default=False,
|
||||
)
|
||||
_print_wizard_substep("ID Strategy")
|
||||
click.echo("ID strategy controls whether imported app and tool IDs preserve source IDs")
|
||||
click.echo("or use target-generated IDs.")
|
||||
click.echo("preserve-id: keep source IDs where the target service supports it.")
|
||||
click.echo("generate-new-id: let the target environment generate new IDs and rewrite references via mapping.")
|
||||
id_strategy = click.prompt(
|
||||
"Import ID strategy. Enter one of: preserve-id, generate-new-id",
|
||||
type=click.Choice(ID_STRATEGY_CHOICES),
|
||||
default="preserve-id",
|
||||
show_default=True,
|
||||
)
|
||||
_print_wizard_substep("Conflict Strategy")
|
||||
click.echo("Conflict strategy controls what import does when a target resource already exists.")
|
||||
click.echo("fail: stop at the first conflict; previously committed resources are not rolled back.")
|
||||
click.echo("skip: keep the existing target resource and skip importing that resource.")
|
||||
click.echo("update: update the existing target resource in place.")
|
||||
conflict_strategy = click.prompt(
|
||||
"Import conflict strategy. Enter one of: fail, skip, update",
|
||||
type=click.Choice(CONFLICT_STRATEGY_CHOICES),
|
||||
default="update",
|
||||
show_default=True,
|
||||
)
|
||||
return include_secrets, create_tokens, id_strategy, conflict_strategy
|
||||
|
||||
|
||||
def _discover_auto_tools(apps: list[App], include_referenced_tools: bool) -> WizardToolMap:
|
||||
auto_tools: WizardToolMap = {"api_tools": {}, "workflow_tools": {}, "mcp_tools": {}}
|
||||
if not include_referenced_tools:
|
||||
return auto_tools
|
||||
discovery_service = DependencyDiscoveryService()
|
||||
for app in apps:
|
||||
dsl_content = AppDslService.export_dsl(app_model=app, include_secret=False)
|
||||
raw_dsl = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
dsl = raw_dsl if isinstance(raw_dsl, dict) else {}
|
||||
for dependency in discovery_service.discover_from_dsl(dsl):
|
||||
if dependency.kind == DependencyKind.API_TOOL:
|
||||
auto_tools["api_tools"][dependency.provider_name or dependency.provider_id] = dependency.provider_id
|
||||
elif dependency.kind == DependencyKind.WORKFLOW_TOOL:
|
||||
auto_tools["workflow_tools"][dependency.provider_name or dependency.provider_id] = (
|
||||
dependency.provider_id
|
||||
)
|
||||
elif dependency.kind == DependencyKind.MCP_TOOL:
|
||||
auto_tools["mcp_tools"][dependency.provider_name or dependency.provider_id] = dependency.provider_id
|
||||
return auto_tools
|
||||
|
||||
|
||||
def _resolve_auto_tool_names(tenant_id: str, auto_tools: WizardToolMap) -> WizardToolMap:
|
||||
return {
|
||||
"api_tools": _resolve_api_tool_names(tenant_id, auto_tools["api_tools"]),
|
||||
"workflow_tools": _resolve_workflow_tool_names(tenant_id, auto_tools["workflow_tools"]),
|
||||
"mcp_tools": _resolve_mcp_tool_names(tenant_id, auto_tools["mcp_tools"]),
|
||||
}
|
||||
|
||||
|
||||
def _resolve_api_tool_names(tenant_id: str, tools: dict[str, str | None]) -> dict[str, str | None]:
|
||||
resolved: dict[str, str | None] = {}
|
||||
for name, identifier in tools.items():
|
||||
predicates = [ApiToolProvider.name == name]
|
||||
if _is_uuid_string(identifier):
|
||||
predicates.append(ApiToolProvider.id == identifier)
|
||||
provider = db.session.scalar(
|
||||
sa.select(ApiToolProvider).where(
|
||||
ApiToolProvider.tenant_id == tenant_id,
|
||||
sa.or_(*predicates),
|
||||
)
|
||||
)
|
||||
resolved[provider.name if provider else name] = provider.id if provider else identifier
|
||||
return resolved
|
||||
|
||||
|
||||
def _resolve_workflow_tool_names(tenant_id: str, tools: dict[str, str | None]) -> dict[str, str | None]:
|
||||
resolved: dict[str, str | None] = {}
|
||||
for name, identifier in tools.items():
|
||||
predicates = [WorkflowToolProvider.name == name]
|
||||
if _is_uuid_string(identifier):
|
||||
predicates.append(WorkflowToolProvider.id == identifier)
|
||||
provider = db.session.scalar(
|
||||
sa.select(WorkflowToolProvider).where(
|
||||
WorkflowToolProvider.tenant_id == tenant_id,
|
||||
sa.or_(*predicates),
|
||||
)
|
||||
)
|
||||
resolved[provider.name if provider else name] = provider.id if provider else identifier
|
||||
return resolved
|
||||
|
||||
|
||||
def _resolve_mcp_tool_names(tenant_id: str, tools: dict[str, str | None]) -> dict[str, str | None]:
|
||||
resolved: dict[str, str | None] = {}
|
||||
for name, identifier in tools.items():
|
||||
predicates = [MCPToolProvider.name == name]
|
||||
if identifier:
|
||||
predicates.append(MCPToolProvider.server_identifier == identifier)
|
||||
if _is_uuid_string(identifier):
|
||||
predicates.append(MCPToolProvider.id == identifier)
|
||||
provider = db.session.scalar(
|
||||
sa.select(MCPToolProvider).where(
|
||||
MCPToolProvider.tenant_id == tenant_id,
|
||||
sa.or_(*predicates),
|
||||
)
|
||||
)
|
||||
resolved[provider.name if provider else name] = provider.id if provider else identifier
|
||||
return resolved
|
||||
|
||||
|
||||
def _is_uuid_string(value: str | None) -> bool:
|
||||
if not value:
|
||||
return False
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _print_auto_tools(auto_tools: WizardToolMap) -> None:
|
||||
_print_wizard_step("Automatically Discovered Tools")
|
||||
click.echo("Automatically discovered tools:")
|
||||
_print_auto_tool_category("Custom API tools", auto_tools["api_tools"])
|
||||
_print_auto_tool_category("Workflow tools", auto_tools["workflow_tools"])
|
||||
_print_auto_tool_category("MCP tools", auto_tools["mcp_tools"])
|
||||
|
||||
|
||||
def _print_auto_tool_category(label: str, values: dict[str, str | None]) -> None:
|
||||
click.echo(label)
|
||||
if not values:
|
||||
click.echo("- none")
|
||||
return
|
||||
for name, identifier in sorted(values.items()):
|
||||
click.echo(f"- {_format_tool_name_id(name, identifier)}")
|
||||
|
||||
|
||||
def _prompt_additional_tools(tenant_id: str, auto_tools: WizardToolMap) -> WizardToolSelection:
|
||||
selections: WizardToolSelection = {"api_tools": [], "workflow_tools": [], "mcp_tools": []}
|
||||
_print_wizard_step("Additional Tools")
|
||||
if not click.confirm(
|
||||
"Export additional tools manually? [y/n, default: n]",
|
||||
default=False,
|
||||
show_default=False,
|
||||
):
|
||||
_print_final_tool_selection(auto_tools, selections, {})
|
||||
return selections
|
||||
manual_labels: dict[str, str] = {}
|
||||
api_tool_options = [
|
||||
(tool.name, tool.name, tool.id)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(ApiToolProvider).where(ApiToolProvider.tenant_id == tenant_id).order_by(ApiToolProvider.name)
|
||||
).all()
|
||||
]
|
||||
selections["api_tools"] = _prompt_tool_category(
|
||||
"Custom API tools",
|
||||
api_tool_options,
|
||||
auto_tools=auto_tools["api_tools"],
|
||||
)
|
||||
manual_labels.update(_selected_tool_labels(api_tool_options, selections["api_tools"]))
|
||||
workflow_tool_options = [
|
||||
(tool.id, tool.name, tool.id)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(WorkflowToolProvider)
|
||||
.where(WorkflowToolProvider.tenant_id == tenant_id)
|
||||
.order_by(WorkflowToolProvider.name)
|
||||
).all()
|
||||
]
|
||||
selections["workflow_tools"] = _prompt_tool_category(
|
||||
"Workflow tools",
|
||||
workflow_tool_options,
|
||||
auto_tools=auto_tools["workflow_tools"],
|
||||
)
|
||||
manual_labels.update(_selected_tool_labels(workflow_tool_options, selections["workflow_tools"]))
|
||||
mcp_tool_options = [
|
||||
(tool.id, tool.name, tool.server_identifier)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id).order_by(MCPToolProvider.name)
|
||||
).all()
|
||||
]
|
||||
selections["mcp_tools"] = _prompt_tool_category(
|
||||
"MCP tools",
|
||||
mcp_tool_options,
|
||||
auto_tools=auto_tools["mcp_tools"],
|
||||
)
|
||||
manual_labels.update(_selected_tool_labels(mcp_tool_options, selections["mcp_tools"]))
|
||||
_print_final_tool_selection(auto_tools, selections, manual_labels)
|
||||
return selections
|
||||
|
||||
|
||||
def _selected_tool_labels_for_tenant(tenant_id: str, selected_tools: WizardToolSelection) -> dict[str, str]:
|
||||
labels: dict[str, str] = {}
|
||||
if selected_tools["api_tools"]:
|
||||
labels.update(
|
||||
_selected_tool_labels(
|
||||
[
|
||||
(tool.name, tool.name, tool.id)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(ApiToolProvider)
|
||||
.where(ApiToolProvider.tenant_id == tenant_id)
|
||||
.order_by(ApiToolProvider.name)
|
||||
).all()
|
||||
],
|
||||
selected_tools["api_tools"],
|
||||
)
|
||||
)
|
||||
if selected_tools["workflow_tools"]:
|
||||
labels.update(
|
||||
_selected_tool_labels(
|
||||
[
|
||||
(tool.id, tool.name, tool.id)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(WorkflowToolProvider)
|
||||
.where(WorkflowToolProvider.tenant_id == tenant_id)
|
||||
.order_by(WorkflowToolProvider.name)
|
||||
).all()
|
||||
],
|
||||
selected_tools["workflow_tools"],
|
||||
)
|
||||
)
|
||||
if selected_tools["mcp_tools"]:
|
||||
labels.update(
|
||||
_selected_tool_labels(
|
||||
[
|
||||
(tool.id, tool.name, tool.server_identifier)
|
||||
for tool in db.session.scalars(
|
||||
sa.select(MCPToolProvider)
|
||||
.where(MCPToolProvider.tenant_id == tenant_id)
|
||||
.order_by(MCPToolProvider.name)
|
||||
).all()
|
||||
],
|
||||
selected_tools["mcp_tools"],
|
||||
)
|
||||
)
|
||||
return labels
|
||||
|
||||
|
||||
def _selected_tool_labels(options: list[tuple[str, str, str]], selected_values: list[str]) -> dict[str, str]:
|
||||
selected = set(selected_values)
|
||||
return {value: _format_tool_name_id(name, detail) for value, name, detail in options if value in selected}
|
||||
|
||||
|
||||
def _prompt_tool_category(
|
||||
label: str,
|
||||
options: list[tuple[str, str, str]],
|
||||
*,
|
||||
auto_tools: dict[str, str | None],
|
||||
) -> list[str]:
|
||||
if not options:
|
||||
click.echo(f"{label}: none")
|
||||
return []
|
||||
_print_wizard_step(label)
|
||||
for index, (value, name, detail) in enumerate(options, 1):
|
||||
marker = "[auto]" if _is_auto_tool(value, name, detail, auto_tools) else "[ ]"
|
||||
click.echo(f"{index}. {marker} {name} ({detail})")
|
||||
raw = click.prompt(
|
||||
f"Select {label.lower()} by number, comma-separated numbers, all, or empty",
|
||||
default="",
|
||||
show_default=cast(Any, "empty"),
|
||||
)
|
||||
if not raw.strip():
|
||||
return []
|
||||
return parse_index_selection(raw, [value for value, _, _ in options])
|
||||
|
||||
|
||||
def _is_auto_tool(value: str, name: str, detail: str, auto_tools: dict[str, str | None]) -> bool:
|
||||
return name in auto_tools or value in auto_tools or value in auto_tools.values() or detail in auto_tools.values()
|
||||
|
||||
|
||||
def _print_final_tool_selection(
|
||||
auto_tools: WizardToolMap,
|
||||
additional_tools: WizardToolSelection,
|
||||
manual_labels: dict[str, str],
|
||||
) -> None:
|
||||
_print_wizard_step("Final Tool Selection")
|
||||
_print_tool_selection_body(auto_tools, additional_tools, manual_labels)
|
||||
|
||||
|
||||
def _print_tool_selection_body(
|
||||
auto_tools: WizardToolMap,
|
||||
additional_tools: WizardToolSelection,
|
||||
manual_labels: dict[str, str],
|
||||
) -> None:
|
||||
click.echo("Final tools to export:")
|
||||
_print_final_tool_category(
|
||||
"Custom API tools",
|
||||
auto_tools["api_tools"],
|
||||
additional_tools["api_tools"],
|
||||
manual_labels,
|
||||
)
|
||||
_print_final_tool_category(
|
||||
"Workflow tools",
|
||||
auto_tools["workflow_tools"],
|
||||
additional_tools["workflow_tools"],
|
||||
manual_labels,
|
||||
)
|
||||
_print_final_tool_category("MCP tools", auto_tools["mcp_tools"], additional_tools["mcp_tools"], manual_labels)
|
||||
|
||||
|
||||
def _print_final_tool_category(
|
||||
label: str,
|
||||
auto_tools: dict[str, str | None],
|
||||
manual_values: list[str],
|
||||
manual_labels: dict[str, str],
|
||||
) -> None:
|
||||
click.echo(label)
|
||||
lines = [f"- [auto] {_format_tool_name_id(name, identifier)}" for name, identifier in sorted(auto_tools.items())]
|
||||
auto_identifiers = {identifier for identifier in auto_tools.values() if identifier}
|
||||
lines.extend(
|
||||
f"- [manual] {manual_labels.get(value, value)}"
|
||||
for value in manual_values
|
||||
if value not in auto_tools and value not in auto_identifiers
|
||||
)
|
||||
if not lines:
|
||||
click.echo("- none")
|
||||
return
|
||||
for line in lines:
|
||||
click.echo(line)
|
||||
|
||||
|
||||
def _format_tool_name_id(name: str, identifier: str | None) -> str:
|
||||
if identifier and identifier != name:
|
||||
return f"{name}: {identifier}"
|
||||
return name
|
||||
|
||||
|
||||
def _confirm_wizard_summary(
|
||||
*,
|
||||
tenant_name: str,
|
||||
app_names: list[str],
|
||||
auto_tools: WizardToolMap,
|
||||
additional_tools: WizardToolSelection,
|
||||
manual_labels: dict[str, str],
|
||||
include_referenced_tools: bool,
|
||||
include_secrets: bool,
|
||||
create_tokens: bool,
|
||||
id_strategy: str,
|
||||
conflict_strategy: str,
|
||||
output_file: str,
|
||||
) -> None:
|
||||
_print_wizard_step("Summary")
|
||||
click.echo("Migration export summary:")
|
||||
click.echo(f"source tenant: {tenant_name}")
|
||||
click.echo(f"selected apps: {len(app_names)}")
|
||||
for app_name in app_names:
|
||||
click.echo(f"- {app_name}")
|
||||
click.echo(f"auto referenced tools: {str(include_referenced_tools).lower()}")
|
||||
_print_tool_selection_body(auto_tools, additional_tools, manual_labels)
|
||||
click.echo(f"include secrets: {str(include_secrets).lower()}")
|
||||
click.echo(f"create app api token on import: {str(create_tokens).lower()}")
|
||||
click.echo(f"id strategy: {id_strategy}")
|
||||
click.echo(f"conflict strategy: {conflict_strategy}")
|
||||
click.echo(f"output path: {output_file}")
|
||||
if not click.confirm("Write migration package? [y/n, default: y]", default=True, show_default=False):
|
||||
raise click.Abort()
|
||||
|
||||
|
||||
def _prompt_output_file() -> tuple[str, bool]:
|
||||
default_output = f"migration-data-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
|
||||
output_file = click.prompt("Output path", default=default_output, show_default=True)
|
||||
if output_file.lower() in {"y", "yes", "n", "no"}:
|
||||
raise click.ClickException("Output path must be a file path. Press Enter to use the default path.")
|
||||
overwrite = False
|
||||
if Path(output_file).exists():
|
||||
overwrite = click.confirm(
|
||||
"Output file exists. Overwrite? [y/n, default: n]",
|
||||
default=False,
|
||||
show_default=False,
|
||||
)
|
||||
if not overwrite:
|
||||
raise click.ClickException(f"Output file already exists: {output_file}")
|
||||
return output_file, overwrite
|
||||
|
||||
|
||||
def _with_output_path(context: ReportContext | None, output_path: str) -> ReportContext:
|
||||
if context is None:
|
||||
return ReportContext(output_path=output_path)
|
||||
return ReportContext(
|
||||
output_path=output_path,
|
||||
source_scope=context.source_scope,
|
||||
selected_app_count=context.selected_app_count,
|
||||
include_secrets=context.include_secrets,
|
||||
target_tenant=context.target_tenant,
|
||||
operator_email=context.operator_email,
|
||||
app_api_tokens_created=context.app_api_tokens_created,
|
||||
app_api_tokens_reused=context.app_api_tokens_reused,
|
||||
id_mapping_count=context.id_mapping_count,
|
||||
id_mappings=context.id_mappings,
|
||||
)
|
||||
|
||||
|
||||
def _render_report(report_items: list[ResourceReportItem], *, context: ReportContext | None = None) -> None:
|
||||
for line in MigrationReportService().render(report_items, context=context):
|
||||
click.echo(line)
|
||||
@@ -30,7 +30,7 @@ def vdb_migrate(scope: str):
|
||||
|
||||
def migrate_annotation_vector_database():
|
||||
"""
|
||||
Migrate annotation datas to target vector database .
|
||||
Migrate annotation data to target vector database.
|
||||
"""
|
||||
click.echo(click.style("Starting annotation data migration.", fg="green"))
|
||||
create_count = 0
|
||||
@@ -140,7 +140,7 @@ def migrate_annotation_vector_database():
|
||||
|
||||
def migrate_knowledge_vector_database():
|
||||
"""
|
||||
Migrate vector database datas to target vector database .
|
||||
Migrate vector database data to target vector database.
|
||||
"""
|
||||
click.echo(click.style("Starting vector database migration.", fg="green"))
|
||||
create_count = 0
|
||||
|
||||
@@ -11,7 +11,7 @@ from controllers.console.app.error import (
|
||||
ProviderNotInitializeError,
|
||||
ProviderQuotaExceededError,
|
||||
)
|
||||
from controllers.console.wraps import account_initialization_required, setup_required
|
||||
from controllers.console.wraps import account_initialization_required, setup_required, with_current_tenant_id
|
||||
from core.app.app_config.entities import ModelConfig
|
||||
from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
|
||||
from core.helper.code_executor.code_node_provider import CodeNodeProvider
|
||||
@@ -22,7 +22,7 @@ from core.llm_generator.llm_generator import LLMGenerator
|
||||
from extensions.ext_database import db
|
||||
from graphon.model_runtime.entities.llm_entities import LLMMode
|
||||
from graphon.model_runtime.errors.invoke import InvokeError
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from libs.login import login_required
|
||||
from models import App
|
||||
from services.workflow_service import WorkflowService
|
||||
|
||||
@@ -64,9 +64,9 @@ class RuleGenerateApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def post(self):
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str):
|
||||
args = RuleGeneratePayload.model_validate(console_ns.payload)
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
|
||||
try:
|
||||
rules = LLMGenerator.generate_rule_config(tenant_id=current_tenant_id, args=args)
|
||||
@@ -93,9 +93,9 @@ class RuleCodeGenerateApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def post(self):
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str):
|
||||
args = RuleCodeGeneratePayload.model_validate(console_ns.payload)
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
|
||||
try:
|
||||
code_result = LLMGenerator.generate_code(
|
||||
@@ -125,9 +125,9 @@ class RuleStructuredOutputGenerateApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def post(self):
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str):
|
||||
args = RuleStructuredOutputPayload.model_validate(console_ns.payload)
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
|
||||
try:
|
||||
structured_output = LLMGenerator.generate_structured_output(
|
||||
@@ -157,9 +157,9 @@ class InstructionGenerateApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def post(self):
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str):
|
||||
args = InstructionGeneratePayload.model_validate(console_ns.payload)
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
providers: list[type[CodeNodeProvider]] = [Python3CodeProvider, JavascriptCodeProvider]
|
||||
code_provider: type[CodeNodeProvider] | None = next(
|
||||
(p for p in providers if p.is_accept_language(args.language)), None
|
||||
|
||||
@@ -11,11 +11,16 @@ from werkzeug.exceptions import NotFound
|
||||
from controllers.common.schema import register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.wraps import get_app_model
|
||||
from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
edit_permission_required,
|
||||
setup_required,
|
||||
with_current_tenant_id,
|
||||
)
|
||||
from extensions.ext_database import db
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import to_timestamp
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from libs.login import login_required
|
||||
from models.enums import AppMCPServerStatus
|
||||
from models.model import App, AppMCPServer
|
||||
|
||||
@@ -92,8 +97,8 @@ class AppMCPServerController(Resource):
|
||||
@login_required
|
||||
@setup_required
|
||||
@edit_permission_required
|
||||
def post(self, app_model: App):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, app_model: App):
|
||||
payload = MCPServerCreatePayload.model_validate(console_ns.payload or {})
|
||||
|
||||
description = payload.description
|
||||
@@ -163,8 +168,8 @@ class AppMCPServerRefreshController(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
def get(self, server_id: UUID):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, server_id: UUID):
|
||||
server = db.session.scalar(
|
||||
select(AppMCPServer)
|
||||
.where(AppMCPServer.id == server_id, AppMCPServer.tenant_id == current_tenant_id)
|
||||
|
||||
@@ -83,13 +83,14 @@ def _serialize_var_value(variable: WorkflowDraftVariable):
|
||||
# create a copy of the value to avoid affecting the model cache.
|
||||
value = value.model_copy(deep=True)
|
||||
# Refresh the url signature before returning it to client.
|
||||
if isinstance(value, FileSegment):
|
||||
file = value.value
|
||||
file.remote_url = file.generate_url()
|
||||
elif isinstance(value, ArrayFileSegment):
|
||||
files = value.value
|
||||
for file in files:
|
||||
match value:
|
||||
case FileSegment():
|
||||
file = value.value
|
||||
file.remote_url = file.generate_url()
|
||||
case ArrayFileSegment():
|
||||
files = value.value
|
||||
for file in files:
|
||||
file.remote_url = file.generate_url()
|
||||
return _convert_values_to_json_serializable_object(value)
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from uuid import UUID
|
||||
from flask import request
|
||||
from flask_restx import Resource, marshal
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import String, cast, func, or_, select
|
||||
from sqlalchemy import String, case, cast, func, literal, or_, select
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
@@ -169,9 +169,17 @@ class DatasetDocumentSegmentListApi(Resource):
|
||||
# Use database-specific methods for JSON array search
|
||||
if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql":
|
||||
# PostgreSQL: Use jsonb_array_elements_text to properly handle Unicode/Chinese text
|
||||
# Feed the set-returning function a JSON array in every row. Filtering in
|
||||
# the subquery is not enough because PostgreSQL can still evaluate the
|
||||
# SRF on scalar JSON before applying the predicate.
|
||||
keywords_jsonb = cast(DocumentSegment.keywords, JSONB)
|
||||
keywords_array = case(
|
||||
(func.jsonb_typeof(keywords_jsonb) == "array", keywords_jsonb),
|
||||
else_=cast(literal("[]"), JSONB),
|
||||
)
|
||||
keywords_condition = func.array_to_string(
|
||||
func.array(
|
||||
select(func.jsonb_array_elements_text(cast(DocumentSegment.keywords, JSONB)))
|
||||
select(func.jsonb_array_elements_text(keywords_array))
|
||||
.correlate(DocumentSegment)
|
||||
.scalar_subquery()
|
||||
),
|
||||
|
||||
@@ -8,12 +8,17 @@ from pydantic import BaseModel, Field, field_validator
|
||||
from controllers.common.fields import SimpleResultResponse
|
||||
from controllers.common.schema import register_enum_models, register_response_schema_models, register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
is_admin_or_owner_required,
|
||||
setup_required,
|
||||
with_current_tenant_id,
|
||||
)
|
||||
from graphon.model_runtime.entities.model_entities import ModelType
|
||||
from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
|
||||
from graphon.model_runtime.utils.encoders import jsonable_encoder
|
||||
from libs.helper import uuid_value
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from libs.login import login_required
|
||||
from services.model_load_balancing_service import ModelLoadBalancingService
|
||||
from services.model_provider_service import ModelProviderService
|
||||
|
||||
@@ -138,9 +143,8 @@ class DefaultModelApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def get(self):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def get(self, tenant_id: str):
|
||||
args = ParserGetDefault.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -156,9 +160,8 @@ class DefaultModelApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def post(self):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def post(self, tenant_id: str):
|
||||
args = ParserPostDefault.model_validate(console_ns.payload)
|
||||
model_provider_service = ModelProviderService()
|
||||
model_settings = args.model_settings
|
||||
@@ -189,9 +192,8 @@ class ModelProviderModelApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def get(self, provider):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def get(self, tenant_id: str, provider):
|
||||
model_provider_service = ModelProviderService()
|
||||
models = model_provider_service.get_models_by_provider(tenant_id=tenant_id, provider=provider)
|
||||
|
||||
@@ -202,9 +204,9 @@ class ModelProviderModelApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def post(self, provider: str):
|
||||
@with_current_tenant_id
|
||||
def post(self, tenant_id: str, provider: str):
|
||||
# To save the model's load balance configs
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
args = ParserPostModels.model_validate(console_ns.payload)
|
||||
|
||||
if args.config_from == "custom-model":
|
||||
@@ -249,9 +251,8 @@ class ModelProviderModelApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def delete(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def delete(self, tenant_id: str, provider: str):
|
||||
args = ParserDeleteModels.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -268,9 +269,8 @@ class ModelProviderModelCredentialApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def get(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def get(self, tenant_id: str, provider: str):
|
||||
args = ParserGetCredentials.model_validate(request.args.to_dict(flat=True))
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -323,9 +323,8 @@ class ModelProviderModelCredentialApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def post(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def post(self, tenant_id: str, provider: str):
|
||||
args = ParserCreateCredential.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -355,8 +354,8 @@ class ModelProviderModelCredentialApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def put(self, provider: str):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def put(self, current_tenant_id: str, provider: str):
|
||||
args = ParserUpdateCredential.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -382,8 +381,8 @@ class ModelProviderModelCredentialApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def delete(self, provider: str):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def delete(self, current_tenant_id: str, provider: str):
|
||||
args = ParserDeleteCredential.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -406,8 +405,8 @@ class ModelProviderModelCredentialSwitchApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
def post(self, provider: str):
|
||||
_, current_tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def post(self, current_tenant_id: str, provider: str):
|
||||
args = ParserSwitch.model_validate(console_ns.payload)
|
||||
|
||||
service = ModelProviderService()
|
||||
@@ -430,9 +429,8 @@ class ModelProviderModelEnableApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def patch(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def patch(self, tenant_id: str, provider: str):
|
||||
args = ParserDeleteModels.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -452,9 +450,8 @@ class ModelProviderModelDisableApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def patch(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
@with_current_tenant_id
|
||||
def patch(self, tenant_id: str, provider: str):
|
||||
args = ParserDeleteModels.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -480,8 +477,8 @@ class ModelProviderModelValidateApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def post(self, provider: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def post(self, tenant_id: str, provider: str):
|
||||
args = ParserValidate.model_validate(console_ns.payload)
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
@@ -515,9 +512,9 @@ class ModelProviderModelParameterRuleApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def get(self, provider: str):
|
||||
@with_current_tenant_id
|
||||
def get(self, tenant_id: str, provider: str):
|
||||
args = ParserParameter.model_validate(request.args.to_dict(flat=True))
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
|
||||
model_provider_service = ModelProviderService()
|
||||
parameter_rules = model_provider_service.get_model_parameter_rules(
|
||||
@@ -532,8 +529,8 @@ class ModelProviderAvailableModelApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def get(self, model_type: str):
|
||||
_, tenant_id = current_account_with_tenant()
|
||||
@with_current_tenant_id
|
||||
def get(self, tenant_id: str, model_type: str):
|
||||
model_provider_service = ModelProviderService()
|
||||
models = model_provider_service.get_models_by_model_type(tenant_id=tenant_id, model_type=model_type)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from datetime import UTC, datetime
|
||||
|
||||
from flask import request
|
||||
from flask_restx import Resource
|
||||
from werkzeug.exceptions import BadRequest, NotFound
|
||||
from werkzeug.exceptions import NotFound
|
||||
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.openapi._models import (
|
||||
@@ -17,18 +17,17 @@ from controllers.openapi._models import (
|
||||
SessionRow,
|
||||
WorkspacePayload,
|
||||
)
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from libs.oauth_bearer import (
|
||||
ACCEPT_USER_ANY,
|
||||
AuthContext,
|
||||
SubjectType,
|
||||
Scope,
|
||||
TokenType,
|
||||
get_auth_ctx,
|
||||
validate_bearer,
|
||||
)
|
||||
from libs.rate_limit import (
|
||||
LIMIT_ME_PER_ACCOUNT,
|
||||
LIMIT_ME_PER_EMAIL,
|
||||
enforce,
|
||||
)
|
||||
from services.account_service import AccountService, TenantService
|
||||
@@ -42,32 +41,18 @@ from services.oauth_device_flow import (
|
||||
@openapi_ns.route("/account")
|
||||
class AccountApi(Resource):
|
||||
@openapi_ns.response(200, "Account info", openapi_ns.models[AccountResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
def get(self):
|
||||
ctx = get_auth_ctx()
|
||||
@auth_router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, *, auth_data: AuthData):
|
||||
enforce(LIMIT_ME_PER_ACCOUNT, key=f"account:{auth_data.account_id}")
|
||||
|
||||
if ctx.subject_type == SubjectType.EXTERNAL_SSO:
|
||||
enforce(LIMIT_ME_PER_EMAIL, key=f"subject:{ctx.subject_email}")
|
||||
else:
|
||||
enforce(LIMIT_ME_PER_ACCOUNT, key=f"account:{ctx.account_id}")
|
||||
|
||||
if ctx.subject_type == SubjectType.EXTERNAL_SSO:
|
||||
return AccountResponse(
|
||||
subject_type=ctx.subject_type,
|
||||
subject_email=ctx.subject_email,
|
||||
subject_issuer=ctx.subject_issuer,
|
||||
account=None,
|
||||
workspaces=[],
|
||||
default_workspace_id=None,
|
||||
).model_dump(mode="json")
|
||||
|
||||
account = AccountService.get_account_by_id(db.session, str(ctx.account_id)) if ctx.account_id else None
|
||||
memberships = TenantService.get_account_memberships(db.session, str(ctx.account_id)) if ctx.account_id else []
|
||||
account_id_str = str(auth_data.account_id) if auth_data.account_id else None
|
||||
account = AccountService.get_account_by_id(db.session, account_id_str) if account_id_str else None
|
||||
memberships = TenantService.get_account_memberships(db.session, account_id_str) if account_id_str else []
|
||||
default_ws_id = _pick_default_workspace(memberships)
|
||||
|
||||
return AccountResponse(
|
||||
subject_type=ctx.subject_type,
|
||||
subject_email=ctx.subject_email or (account.email if account else None),
|
||||
subject_type="account",
|
||||
subject_email=account.email if account else None,
|
||||
account=_account_payload(account) if account else None,
|
||||
workspaces=[_workspace_payload(m) for m in memberships],
|
||||
default_workspace_id=default_ws_id,
|
||||
@@ -77,19 +62,17 @@ class AccountApi(Resource):
|
||||
@openapi_ns.route("/account/sessions/self")
|
||||
class AccountSessionsSelfApi(Resource):
|
||||
@openapi_ns.response(200, "Session revoked", openapi_ns.models[RevokeResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
def delete(self):
|
||||
ctx = get_auth_ctx()
|
||||
_require_oauth_subject(ctx)
|
||||
revoke_oauth_token(db.session, redis_client, str(ctx.token_id))
|
||||
@auth_router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def delete(self, *, auth_data: AuthData):
|
||||
revoke_oauth_token(db.session, redis_client, str(auth_data.token_id))
|
||||
return RevokeResponse(status="revoked").model_dump(mode="json"), 200
|
||||
|
||||
|
||||
@openapi_ns.route("/account/sessions")
|
||||
class AccountSessionsApi(Resource):
|
||||
@openapi_ns.response(200, "Session list", openapi_ns.models[SessionListResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
def get(self):
|
||||
@auth_router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, *, auth_data: AuthData):
|
||||
ctx = get_auth_ctx()
|
||||
now = datetime.now(UTC)
|
||||
page = int(request.args.get("page", "1"))
|
||||
@@ -122,10 +105,9 @@ class AccountSessionsApi(Resource):
|
||||
@openapi_ns.route("/account/sessions/<string:session_id>")
|
||||
class AccountSessionByIdApi(Resource):
|
||||
@openapi_ns.response(200, "Session revoked", openapi_ns.models[RevokeResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
def delete(self, session_id: str):
|
||||
@auth_router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def delete(self, session_id: str, *, auth_data: AuthData):
|
||||
ctx = get_auth_ctx()
|
||||
_require_oauth_subject(ctx)
|
||||
|
||||
# 404 (not 403) on cross-subject so the endpoint doesn't leak
|
||||
# token IDs that belong to other subjects.
|
||||
@@ -136,13 +118,6 @@ class AccountSessionByIdApi(Resource):
|
||||
return RevokeResponse(status="revoked").model_dump(mode="json"), 200
|
||||
|
||||
|
||||
def _require_oauth_subject(ctx: AuthContext) -> None:
|
||||
if not ctx.source.startswith("oauth"):
|
||||
raise BadRequest(
|
||||
"this endpoint revokes OAuth bearer tokens; use /openapi/v1/personal-access-tokens/self for PATs"
|
||||
)
|
||||
|
||||
|
||||
def _iso(dt: datetime | None) -> str | None:
|
||||
if dt is None:
|
||||
return None
|
||||
|
||||
@@ -16,7 +16,8 @@ import services
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.openapi._audit import emit_app_run
|
||||
from controllers.openapi._models import AppRunRequest
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from controllers.service_api.app.error import (
|
||||
AppUnavailableError,
|
||||
CompletionRequestError,
|
||||
@@ -124,8 +125,9 @@ _DISPATCH: dict[AppMode, Callable[[App, Any, AppRunRequest], Any]] = {
|
||||
class AppRunApi(Resource):
|
||||
@openapi_ns.expect(openapi_ns.models[AppRunRequest.__name__])
|
||||
@openapi_ns.response(200, "Run result (SSE stream)")
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, app_model: App, caller, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, *, auth_data: AuthData):
|
||||
app_model, caller, caller_kind = auth_data.require_app_context()
|
||||
body = request.get_json(silent=True) or {}
|
||||
try:
|
||||
payload = AppRunRequest.model_validate(body)
|
||||
@@ -158,8 +160,9 @@ class AppRunApi(Resource):
|
||||
@openapi_ns.route("/apps/<string:app_id>/tasks/<string:task_id>/stop")
|
||||
class AppRunTaskStopApi(Resource):
|
||||
@openapi_ns.response(200, "Task stopped")
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, task_id: str, app_model: App, caller, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, task_id: str, *, auth_data: AuthData):
|
||||
app_model, caller, caller_kind = auth_data.require_app_context()
|
||||
AppQueueManager.set_stop_flag_no_user_check(task_id)
|
||||
GraphEngineManager(redis_client).send_stop_command(task_id)
|
||||
return {"result": "success"}
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
"""GET /openapi/v1/apps and per-app reads.
|
||||
|
||||
Decorator order: `method_decorators` is innermost-first. `validate_bearer`
|
||||
is last → outermost → publishes the auth ContextVar before `require_scope`
|
||||
reads it.
|
||||
"""
|
||||
"""GET /openapi/v1/apps and per-app reads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -28,31 +23,17 @@ from controllers.openapi._models import (
|
||||
AppListRow,
|
||||
TagItem,
|
||||
)
|
||||
from controllers.openapi.auth.surface_gate import accept_subjects
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from controllers.service_api.app.error import AppUnavailableError
|
||||
from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import (
|
||||
ACCEPT_USER_ANY,
|
||||
AuthContext,
|
||||
Scope,
|
||||
SubjectType,
|
||||
get_auth_ctx,
|
||||
require_scope,
|
||||
require_workspace_member,
|
||||
validate_bearer,
|
||||
)
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models import App
|
||||
from services.account_service import TenantService
|
||||
from services.app_service import AppListParams, AppService
|
||||
from services.tag_service import TagService
|
||||
|
||||
_APPS_READ_DECORATORS = [
|
||||
require_scope(Scope.APPS_READ),
|
||||
accept_subjects(SubjectType.ACCOUNT),
|
||||
validate_bearer(accept=ACCEPT_USER_ANY),
|
||||
]
|
||||
|
||||
_ALLOWED_DESCRIBE_FIELDS: frozenset[str] = frozenset({"info", "parameters", "input_schema"})
|
||||
|
||||
|
||||
@@ -66,13 +47,9 @@ _EMPTY_PARAMETERS: dict[str, Any] = {
|
||||
|
||||
|
||||
class AppReadResource(Resource):
|
||||
"""Base for per-app read endpoints; subclasses call `_load()` for SSO/membership/exists checks."""
|
||||
|
||||
method_decorators = _APPS_READ_DECORATORS
|
||||
|
||||
def _load(self, app_id: str, workspace_id: str | None = None) -> tuple[App, AuthContext]:
|
||||
ctx: AuthContext = get_auth_ctx()
|
||||
"""Base for per-app read endpoints; subclasses call `_load()` for membership/exists checks."""
|
||||
|
||||
def _load(self, app_id: str, workspace_id: str | None = None) -> App:
|
||||
try:
|
||||
parsed_uuid = _uuid.UUID(app_id)
|
||||
is_uuid = True
|
||||
@@ -99,8 +76,7 @@ class AppReadResource(Resource):
|
||||
raise Conflict("".join(lines))
|
||||
app = matches[0]
|
||||
|
||||
require_workspace_member(ctx, str(app.tenant_id))
|
||||
return app, ctx
|
||||
return app
|
||||
|
||||
|
||||
def parameters_payload(app: App) -> dict:
|
||||
@@ -114,13 +90,14 @@ def parameters_payload(app: App) -> dict:
|
||||
class AppDescribeApi(AppReadResource):
|
||||
@openapi_ns.doc(params=query_params_from_model(AppDescribeQuery))
|
||||
@openapi_ns.response(200, "App description", openapi_ns.models[AppDescribeResponse.__name__])
|
||||
def get(self, app_id: str):
|
||||
@auth_router.guard(scope=Scope.APPS_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, app_id: str, *, auth_data: AuthData):
|
||||
try:
|
||||
query = AppDescribeQuery.model_validate(request.args.to_dict(flat=True))
|
||||
except ValidationError as exc:
|
||||
raise UnprocessableEntity(exc.json())
|
||||
|
||||
app, _ = self._load(app_id, workspace_id=query.workspace_id)
|
||||
app = self._load(app_id, workspace_id=query.workspace_id)
|
||||
|
||||
requested = query.fields
|
||||
want_info = requested is None or "info" in requested
|
||||
@@ -168,20 +145,16 @@ class AppDescribeApi(AppReadResource):
|
||||
|
||||
@openapi_ns.route("/apps")
|
||||
class AppListApi(Resource):
|
||||
method_decorators = _APPS_READ_DECORATORS
|
||||
|
||||
@openapi_ns.doc(params=query_params_from_model(AppListQuery))
|
||||
@openapi_ns.response(200, "App list", openapi_ns.models[AppListResponse.__name__])
|
||||
def get(self):
|
||||
ctx: AuthContext = get_auth_ctx()
|
||||
|
||||
@auth_router.guard(scope=Scope.APPS_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, *, auth_data: AuthData):
|
||||
try:
|
||||
query: AppListQuery = AppListQuery.model_validate(request.args.to_dict(flat=True))
|
||||
except ValidationError as exc:
|
||||
raise UnprocessableEntity(exc.json())
|
||||
|
||||
workspace_id = query.workspace_id
|
||||
require_workspace_member(ctx, workspace_id)
|
||||
|
||||
empty = (
|
||||
AppListResponse(page=query.page, limit=query.limit, total=0, has_more=False, data=[]).model_dump(
|
||||
@@ -237,7 +210,7 @@ class AppListApi(Resource):
|
||||
openapi_visible=True,
|
||||
)
|
||||
|
||||
pagination = AppService().get_paginate_apps(str(ctx.account_id), workspace_id, params)
|
||||
pagination = AppService().get_paginate_apps(str(auth_data.account_id), workspace_id, params)
|
||||
if pagination is None:
|
||||
return empty
|
||||
|
||||
|
||||
@@ -18,37 +18,27 @@ from controllers.openapi._models import (
|
||||
PermittedExternalAppsListQuery,
|
||||
PermittedExternalAppsListResponse,
|
||||
)
|
||||
from controllers.openapi.auth.surface_gate import accept_subjects
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData, Edition
|
||||
from extensions.ext_database import db
|
||||
from libs.device_flow_security import enterprise_only
|
||||
from libs.oauth_bearer import (
|
||||
ACCEPT_USER_ANY,
|
||||
Scope,
|
||||
SubjectType,
|
||||
require_scope,
|
||||
validate_bearer,
|
||||
)
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models import App
|
||||
from services.account_service import TenantService
|
||||
from services.app_service import AppService
|
||||
from services.enterprise.app_permitted_service import list_permitted_apps
|
||||
from services.openapi.license_gate import license_required
|
||||
|
||||
|
||||
@openapi_ns.route("/permitted-external-apps")
|
||||
class PermittedExternalAppsListApi(Resource):
|
||||
method_decorators = [
|
||||
require_scope(Scope.APPS_READ_PERMITTED_EXTERNAL),
|
||||
license_required,
|
||||
accept_subjects(SubjectType.EXTERNAL_SSO),
|
||||
validate_bearer(accept=ACCEPT_USER_ANY),
|
||||
enterprise_only,
|
||||
]
|
||||
|
||||
@openapi_ns.response(
|
||||
200, "Permitted external apps list", openapi_ns.models[PermittedExternalAppsListResponse.__name__]
|
||||
)
|
||||
def get(self):
|
||||
@auth_router.guard(
|
||||
scope=Scope.APPS_READ_PERMITTED_EXTERNAL,
|
||||
allowed_token_types=frozenset({TokenType.OAUTH_EXTERNAL_SSO}),
|
||||
edition=frozenset({Edition.EE}),
|
||||
)
|
||||
def get(self, *, auth_data: AuthData):
|
||||
try:
|
||||
query = PermittedExternalAppsListQuery.model_validate(request.args.to_dict(flat=True))
|
||||
except ValidationError as exc:
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
|
||||
__all__ = ["OAUTH_BEARER_PIPELINE"]
|
||||
__all__ = ["auth_router"]
|
||||
|
||||
@@ -1,46 +1,64 @@
|
||||
"""`OAUTH_BEARER_PIPELINE` — the auth scheme for openapi `/run` endpoints.
|
||||
|
||||
Endpoints attach via `@OAUTH_BEARER_PIPELINE.guard(scope=…)`. No alternative
|
||||
paths. Read endpoints (`/apps`, `/info`, `/parameters`, `/describe`) skip
|
||||
the pipeline and use `validate_bearer + require_scope + require_workspace_member`
|
||||
inline — they don't need `AppAuthzCheck`/`CallerMount`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from controllers.openapi.auth.pipeline import Pipeline
|
||||
from controllers.openapi.auth.steps import (
|
||||
AppAuthzCheck,
|
||||
AppResolver,
|
||||
BearerCheck,
|
||||
CallerMount,
|
||||
ScopeCheck,
|
||||
SurfaceCheck,
|
||||
WorkspaceMembershipCheck,
|
||||
from controllers.openapi.auth.conditions import (
|
||||
EDITION_CE,
|
||||
EDITION_EE,
|
||||
LOADED_APP_IS_PRIVATE,
|
||||
PATH_HAS_APP_ID,
|
||||
WEBAPP_AUTH_ENABLED,
|
||||
)
|
||||
from controllers.openapi.auth.strategies import (
|
||||
AccountMounter,
|
||||
AclStrategy,
|
||||
AppAuthzStrategy,
|
||||
EndUserMounter,
|
||||
MembershipStrategy,
|
||||
from controllers.openapi.auth.data import Edition
|
||||
from controllers.openapi.auth.flow import When
|
||||
from controllers.openapi.auth.pipeline import AuthPipeline, PipelineRoute, PipelineRouter
|
||||
from controllers.openapi.auth.prepare import (
|
||||
load_account,
|
||||
load_app,
|
||||
load_app_access_mode,
|
||||
load_tenant,
|
||||
resolve_external_user,
|
||||
)
|
||||
from libs.oauth_bearer import SubjectType
|
||||
from services.feature_service import FeatureService
|
||||
from controllers.openapi.auth.verify import (
|
||||
check_acl,
|
||||
check_app_access,
|
||||
check_membership,
|
||||
check_private_app_permission,
|
||||
check_scope,
|
||||
)
|
||||
from libs.oauth_bearer import TokenType
|
||||
|
||||
account_pipeline = AuthPipeline(
|
||||
prepare=[
|
||||
When(PATH_HAS_APP_ID, then=load_app),
|
||||
When(PATH_HAS_APP_ID, then=load_tenant),
|
||||
load_account, # all tokens here are account tokens
|
||||
When(PATH_HAS_APP_ID & EDITION_EE, then=load_app_access_mode),
|
||||
],
|
||||
auth=[
|
||||
check_scope,
|
||||
When(EDITION_CE & PATH_HAS_APP_ID, then=check_membership),
|
||||
When(EDITION_EE & PATH_HAS_APP_ID & ~WEBAPP_AUTH_ENABLED, then=check_app_access),
|
||||
When(PATH_HAS_APP_ID & EDITION_EE & WEBAPP_AUTH_ENABLED, then=check_acl),
|
||||
When(EDITION_EE & LOADED_APP_IS_PRIVATE, then=check_private_app_permission),
|
||||
],
|
||||
)
|
||||
|
||||
def _resolve_app_authz_strategy() -> AppAuthzStrategy:
|
||||
if FeatureService.get_system_features().webapp_auth.enabled:
|
||||
return AclStrategy()
|
||||
return MembershipStrategy()
|
||||
external_sso_pipeline = AuthPipeline(
|
||||
prepare=[
|
||||
When(PATH_HAS_APP_ID, then=load_app),
|
||||
When(PATH_HAS_APP_ID, then=load_tenant),
|
||||
When(PATH_HAS_APP_ID, then=resolve_external_user),
|
||||
When(PATH_HAS_APP_ID, then=load_app_access_mode),
|
||||
],
|
||||
auth=[
|
||||
check_scope,
|
||||
When(PATH_HAS_APP_ID & WEBAPP_AUTH_ENABLED, then=check_acl),
|
||||
When(LOADED_APP_IS_PRIVATE, then=check_private_app_permission),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
OAUTH_BEARER_PIPELINE = Pipeline(
|
||||
BearerCheck(),
|
||||
SurfaceCheck(accepted=frozenset({SubjectType.ACCOUNT})),
|
||||
ScopeCheck(),
|
||||
AppResolver(),
|
||||
WorkspaceMembershipCheck(),
|
||||
AppAuthzCheck(_resolve_app_authz_strategy),
|
||||
CallerMount(AccountMounter(), EndUserMounter()),
|
||||
auth_router = PipelineRouter(
|
||||
{
|
||||
TokenType.OAUTH_ACCOUNT: PipelineRoute(account_pipeline),
|
||||
TokenType.OAUTH_EXTERNAL_SSO: PipelineRoute(external_sso_pipeline, required_edition=frozenset({Edition.EE})),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from controllers.openapi.auth.data import AuthData, Edition, RequestContext, current_edition
|
||||
from libs.oauth_bearer import TokenType
|
||||
from services.enterprise.enterprise_service import WebAppAccessMode
|
||||
from services.feature_service import FeatureService
|
||||
|
||||
CondFn = Callable[[RequestContext, AuthData | None], bool]
|
||||
|
||||
|
||||
class Cond:
|
||||
def __init__(self, fn: CondFn) -> None:
|
||||
self._fn = fn
|
||||
|
||||
def __call__(self, ctx: RequestContext, data: AuthData | None = None) -> bool:
|
||||
return self._fn(ctx, data)
|
||||
|
||||
def __and__(self, other: Cond) -> Cond:
|
||||
return Cond(lambda ctx, data: self(ctx, data) and other(ctx, data))
|
||||
|
||||
def __or__(self, other: Cond) -> Cond:
|
||||
return Cond(lambda ctx, data: self(ctx, data) or other(ctx, data))
|
||||
|
||||
def __invert__(self) -> Cond:
|
||||
return Cond(lambda ctx, data: not self(ctx, data))
|
||||
|
||||
|
||||
def request_cond(fn: Callable[[RequestContext], bool]) -> Cond:
|
||||
return Cond(lambda ctx, _: fn(ctx))
|
||||
|
||||
|
||||
def data_cond(fn: Callable[[AuthData], bool]) -> Cond:
|
||||
return Cond(lambda _, data: data is not None and fn(data))
|
||||
|
||||
|
||||
def config_cond(fn: Callable[[], bool]) -> Cond:
|
||||
return Cond(lambda _, __: fn())
|
||||
|
||||
|
||||
TOKEN_IS_OAUTH_ACCOUNT = request_cond(lambda ctx: ctx.token_type == TokenType.OAUTH_ACCOUNT)
|
||||
TOKEN_IS_OAUTH_EXTERNAL_SSO = request_cond(lambda ctx: ctx.token_type == TokenType.OAUTH_EXTERNAL_SSO)
|
||||
|
||||
PATH_HAS_APP_ID = request_cond(lambda ctx: "app_id" in ctx.path_params)
|
||||
|
||||
EDITION_CE = config_cond(lambda: current_edition() == Edition.CE)
|
||||
EDITION_EE = config_cond(lambda: current_edition() == Edition.EE)
|
||||
EDITION_SAAS = config_cond(lambda: current_edition() == Edition.SAAS)
|
||||
|
||||
WEBAPP_AUTH_ENABLED = config_cond(lambda: FeatureService.get_system_features().webapp_auth.enabled)
|
||||
|
||||
LOADED_APP_IS_PRIVATE = data_cond(lambda data: data.app_access_mode == WebAppAccessMode.PRIVATE)
|
||||
@@ -1,68 +0,0 @@
|
||||
"""Mutable per-request context for the openapi auth pipeline.
|
||||
|
||||
Every field starts None / empty and is filled in by a step. The pipeline
|
||||
is the only thing that should construct or mutate Context — handlers
|
||||
read populated values via the decorator's kwargs unpacking.
|
||||
|
||||
Context is intentionally decoupled from Flask's ``Request``: the pipeline
|
||||
guard extracts whatever transport-level inputs the steps need (bearer
|
||||
token, path params) at the boundary and writes them into Context fields,
|
||||
so steps stay testable without a request object and won't leak coupling
|
||||
to a specific framework.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from contextvars import Token
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Literal, Protocol
|
||||
|
||||
from werkzeug.exceptions import Unauthorized
|
||||
|
||||
from libs.oauth_bearer import AuthContext, Scope, SubjectType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from models import App, Tenant
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
required_scope: Scope
|
||||
bearer_token: str | None = None
|
||||
path_params: Mapping[str, str] = field(default_factory=dict)
|
||||
subject_type: SubjectType | None = None
|
||||
subject_email: str | None = None
|
||||
subject_issuer: str | None = None
|
||||
account_id: uuid.UUID | None = None
|
||||
scopes: frozenset[Scope] = field(default_factory=frozenset)
|
||||
token_id: uuid.UUID | None = None
|
||||
token_hash: str | None = None
|
||||
cached_verified_tenants: dict[str, bool] | None = None
|
||||
source: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
app: App | None = None
|
||||
tenant: Tenant | None = None
|
||||
caller: object | None = None
|
||||
caller_kind: Literal["account", "end_user"] | None = None
|
||||
auth_ctx_reset_token: Token[AuthContext] | None = None
|
||||
|
||||
@property
|
||||
def must_tenant(self) -> Tenant:
|
||||
if not self.tenant:
|
||||
raise Unauthorized("tenant is not associated")
|
||||
return self.tenant
|
||||
|
||||
@property
|
||||
def must_subject_type(self) -> SubjectType:
|
||||
if not self.subject_type:
|
||||
raise Unauthorized("subject_type unset — BearerCheck did not run")
|
||||
return self.subject_type
|
||||
|
||||
|
||||
class Step(Protocol):
|
||||
"""One responsibility. Mutate ctx or raise to short-circuit."""
|
||||
|
||||
def __call__(self, ctx: Context) -> None: ...
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from enum import StrEnum
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from werkzeug.exceptions import InternalServerError
|
||||
|
||||
from configs import dify_config
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models.account import Account, Tenant
|
||||
from models.model import App, EndUser
|
||||
from services.enterprise.enterprise_service import WebAppAccessMode
|
||||
|
||||
|
||||
class Edition(StrEnum):
|
||||
CE = "ce"
|
||||
EE = "ee"
|
||||
SAAS = "saas"
|
||||
|
||||
|
||||
def current_edition() -> Edition:
|
||||
if dify_config.EDITION == "CLOUD":
|
||||
return Edition.SAAS
|
||||
if dify_config.ENTERPRISE_ENABLED:
|
||||
return Edition.EE
|
||||
return Edition.CE
|
||||
|
||||
|
||||
class ExternalIdentity(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
email: str
|
||||
issuer: str | None = None
|
||||
|
||||
|
||||
class RequestContext(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
token_type: TokenType
|
||||
scope: Scope | None = None
|
||||
path_params: dict[str, str]
|
||||
|
||||
|
||||
class AuthData(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
required_scope: Scope | None = None
|
||||
token_type: TokenType
|
||||
account_id: uuid.UUID | None = None
|
||||
token_hash: str
|
||||
token_id: uuid.UUID | None = None
|
||||
scopes: frozenset[Scope]
|
||||
tenants: dict[str, bool] = Field(default_factory=dict)
|
||||
external_identity: ExternalIdentity | None = None
|
||||
path_params: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
app: App | None = None
|
||||
tenant: Tenant | None = None
|
||||
app_access_mode: WebAppAccessMode | None = None
|
||||
|
||||
caller: Account | EndUser | None = None
|
||||
caller_kind: Literal["account", "end_user"] | None = None
|
||||
|
||||
def require_app_context(self) -> tuple[App, Account | EndUser, Literal["account", "end_user"]]:
|
||||
if self.app is None or self.caller is None or self.caller_kind is None:
|
||||
raise InternalServerError("pipeline_invariant_violated: app context missing")
|
||||
return self.app, self.caller, self.caller_kind
|
||||
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from controllers.openapi.auth.conditions import Cond
|
||||
from controllers.openapi.auth.data import AuthData, RequestContext
|
||||
|
||||
|
||||
class When:
|
||||
def __init__(self, condition: Cond, *, then: Callable[[Any], None]) -> None:
|
||||
self.condition = condition
|
||||
self._step = then
|
||||
|
||||
def applies(self, ctx: RequestContext, data: AuthData | None = None) -> bool:
|
||||
return self.condition(ctx, data)
|
||||
|
||||
def __call__(self, arg: Any) -> None:
|
||||
self._step(arg)
|
||||
@@ -1,51 +1,209 @@
|
||||
"""Pipeline IS the auth scheme.
|
||||
"""Auth pipeline — entry point for all openapi auth.
|
||||
|
||||
`Pipeline.guard(scope=…)` is the only attachment point for endpoints —
|
||||
that is the design lock-in: forgetting an auth layer is structurally
|
||||
impossible because there is no "sometimes wrap, sometimes don't" choice.
|
||||
`PipelineRouter.guard()` is the only attachment point for endpoints.
|
||||
`AuthPipeline` is a pure step-runner with no routing concerns.
|
||||
`PipelineRoute` binds a pipeline to optional edition requirements.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
|
||||
from flask import request
|
||||
from flask import current_app, request
|
||||
from flask_login import user_logged_in
|
||||
from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.context import Context, Step
|
||||
from libs.oauth_bearer import Scope, extract_bearer, reset_auth_ctx
|
||||
from controllers.openapi._audit import emit_wrong_surface
|
||||
from controllers.openapi.auth.data import (
|
||||
AuthData,
|
||||
Edition,
|
||||
ExternalIdentity,
|
||||
RequestContext,
|
||||
current_edition,
|
||||
)
|
||||
from controllers.openapi.auth.flow import When
|
||||
from libs.oauth_bearer import (
|
||||
AuthContext,
|
||||
Scope,
|
||||
TokenType,
|
||||
extract_bearer,
|
||||
get_authenticator,
|
||||
reset_auth_ctx,
|
||||
set_auth_ctx,
|
||||
)
|
||||
from services.feature_service import FeatureService, LicenseStatus
|
||||
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, *steps: Step) -> None:
|
||||
self._steps = steps
|
||||
class AuthPipeline:
|
||||
"""Pure step-runner — no routing, no guard.
|
||||
|
||||
def run(self, ctx: Context) -> None:
|
||||
for step in self._steps:
|
||||
step(ctx)
|
||||
Both `prepare` and `auth` steps receive the same `AuthData` instance.
|
||||
`prepare` steps populate it; `auth` steps validate it.
|
||||
"""
|
||||
|
||||
def guard(self, *, scope: Scope):
|
||||
def decorator(view):
|
||||
def __init__(self, prepare: list, auth: list) -> None:
|
||||
self._prepare = prepare
|
||||
self._auth = auth
|
||||
|
||||
def _run(
|
||||
self,
|
||||
identity: AuthContext,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
view: Callable,
|
||||
*,
|
||||
scope: Scope | None,
|
||||
) -> Any:
|
||||
req_ctx = RequestContext(
|
||||
token_type=identity.token_type,
|
||||
scope=scope,
|
||||
path_params=dict(request.view_args or {}),
|
||||
)
|
||||
|
||||
data = AuthData(
|
||||
token_type=identity.token_type,
|
||||
account_id=identity.account_id,
|
||||
token_hash=identity.token_hash,
|
||||
token_id=identity.token_id,
|
||||
scopes=frozenset(identity.scopes),
|
||||
tenants=dict(identity.verified_tenants),
|
||||
required_scope=scope,
|
||||
path_params=dict(req_ctx.path_params),
|
||||
external_identity=(
|
||||
ExternalIdentity(email=identity.subject_email, issuer=identity.subject_issuer)
|
||||
if identity.subject_email
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
for step in self._prepare:
|
||||
if _should_run(step, req_ctx, data=None):
|
||||
step(data)
|
||||
|
||||
for step in self._auth:
|
||||
if _should_run(step, req_ctx, data=data):
|
||||
step(data)
|
||||
|
||||
reset_token = set_auth_ctx(identity)
|
||||
if data.caller:
|
||||
_mount_flask_login(data.caller)
|
||||
|
||||
try:
|
||||
kwargs["auth_data"] = data
|
||||
return view(*args, **kwargs)
|
||||
finally:
|
||||
reset_auth_ctx(reset_token)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PipelineRoute:
|
||||
pipeline: AuthPipeline
|
||||
required_edition: frozenset[Edition] | None = None
|
||||
|
||||
|
||||
class PipelineRouter:
|
||||
"""Entry point for openapi auth.
|
||||
|
||||
`guard()` is the decorator that endpoints attach to. It applies
|
||||
global gates (edition, token type) then dispatches to the matching
|
||||
`PipelineRoute` for the token type.
|
||||
"""
|
||||
|
||||
def __init__(self, routes: dict[TokenType, PipelineRoute]) -> None:
|
||||
self._routes = routes
|
||||
|
||||
def guard(
|
||||
self,
|
||||
*,
|
||||
scope: Scope | None = None,
|
||||
allowed_token_types: frozenset[TokenType] | None = None,
|
||||
edition: frozenset[Edition] | None = None,
|
||||
) -> Callable:
|
||||
def decorator(view: Callable) -> Callable:
|
||||
@wraps(view)
|
||||
def decorated(*args, **kwargs):
|
||||
# Extract transport-level inputs at the boundary so steps
|
||||
# stay decoupled from Flask's request object.
|
||||
ctx = Context(
|
||||
required_scope=scope,
|
||||
bearer_token=extract_bearer(request),
|
||||
path_params=dict(request.view_args or {}),
|
||||
def decorated(*args: Any, **kwargs: Any) -> Any:
|
||||
return self._execute(
|
||||
args,
|
||||
kwargs,
|
||||
view,
|
||||
scope=scope,
|
||||
allowed_token_types=allowed_token_types,
|
||||
edition=edition,
|
||||
)
|
||||
try:
|
||||
self.run(ctx)
|
||||
kwargs.update(
|
||||
app_model=ctx.app,
|
||||
caller=ctx.caller,
|
||||
caller_kind=ctx.caller_kind,
|
||||
)
|
||||
return view(*args, **kwargs)
|
||||
finally:
|
||||
if ctx.auth_ctx_reset_token is not None:
|
||||
reset_auth_ctx(ctx.auth_ctx_reset_token)
|
||||
|
||||
return decorated
|
||||
|
||||
return decorator
|
||||
|
||||
def _execute(
|
||||
self,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
view: Callable,
|
||||
*,
|
||||
scope: Scope | None,
|
||||
allowed_token_types: frozenset[TokenType] | None,
|
||||
edition: frozenset[Edition] | None,
|
||||
) -> Any:
|
||||
# 404 not 403 — this edition doesn't expose the feature at all
|
||||
if edition is not None and current_edition() not in edition:
|
||||
raise NotFound()
|
||||
|
||||
license_checked = False
|
||||
if edition is not None and Edition.EE in edition:
|
||||
_check_license()
|
||||
license_checked = True
|
||||
|
||||
token = extract_bearer(request)
|
||||
if not token:
|
||||
raise Unauthorized("bearer required")
|
||||
|
||||
identity = get_authenticator().authenticate(token)
|
||||
|
||||
if allowed_token_types is not None and identity.token_type not in allowed_token_types:
|
||||
emit_wrong_surface(
|
||||
subject_type=_subject_type_str(identity),
|
||||
attempted_path=request.path,
|
||||
client_id=getattr(identity, "client_id", None),
|
||||
token_id=str(identity.token_id) if identity.token_id else None,
|
||||
)
|
||||
raise Forbidden("unsupported_token_type")
|
||||
|
||||
route = self._routes.get(identity.token_type)
|
||||
if route is None:
|
||||
raise Forbidden("unsupported_token_type")
|
||||
|
||||
if route.required_edition is not None:
|
||||
if current_edition() not in route.required_edition:
|
||||
raise Forbidden("external_sso_requires_ee")
|
||||
if not license_checked and Edition.EE in route.required_edition:
|
||||
_check_license()
|
||||
|
||||
return route.pipeline._run(identity, args, kwargs, view, scope=scope)
|
||||
|
||||
|
||||
def _should_run(step: Any, req_ctx: RequestContext, data: AuthData | None) -> bool:
|
||||
if isinstance(step, When):
|
||||
return step.applies(req_ctx, data)
|
||||
return True
|
||||
|
||||
|
||||
def _subject_type_str(identity: Any) -> str | None:
|
||||
subject = getattr(identity, "subject_type", None)
|
||||
if subject is None:
|
||||
return None
|
||||
return subject.value if hasattr(subject, "value") else str(subject)
|
||||
|
||||
|
||||
def _check_license() -> None:
|
||||
settings = FeatureService.get_system_features()
|
||||
if settings.license.status in {LicenseStatus.INACTIVE, LicenseStatus.EXPIRED, LicenseStatus.LOST}:
|
||||
raise Forbidden("license_invalid")
|
||||
|
||||
|
||||
def _mount_flask_login(user: Any) -> None:
|
||||
current_app.login_manager._update_request_context_with_user(user) # type: ignore[attr-defined]
|
||||
user_logged_in.send(current_app._get_current_object(), user=user) # type: ignore[attr-defined]
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from werkzeug.exceptions import Forbidden, InternalServerError, NotFound, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from extensions.ext_database import db
|
||||
from models.account import TenantStatus
|
||||
from services.account_service import AccountService, TenantService
|
||||
from services.app_service import AppService
|
||||
from services.end_user_service import EndUserService
|
||||
from services.enterprise.enterprise_service import EnterpriseService, WebAppAccessMode
|
||||
|
||||
|
||||
def load_app(data: AuthData) -> None:
|
||||
app_id = data.path_params["app_id"]
|
||||
app = AppService.get_app_by_id(db.session, app_id)
|
||||
if not app or app.status != "normal":
|
||||
raise NotFound("app not found")
|
||||
if not app.enable_api:
|
||||
raise Forbidden("service_api_disabled")
|
||||
data.app = app
|
||||
|
||||
|
||||
def load_tenant(data: AuthData) -> None:
|
||||
if data.app is None:
|
||||
raise InternalServerError("pipeline_invariant_violated: app not loaded before load_tenant")
|
||||
tenant = TenantService.get_tenant_by_id(db.session, str(data.app.tenant_id))
|
||||
if tenant is None or tenant.status == TenantStatus.ARCHIVE:
|
||||
raise Forbidden("workspace unavailable")
|
||||
data.tenant = tenant
|
||||
|
||||
|
||||
def load_account(data: AuthData) -> None:
|
||||
account = AccountService.get_account_by_id(db.session, str(data.account_id))
|
||||
if account is None:
|
||||
raise Unauthorized("account not found")
|
||||
if data.tenant:
|
||||
account.current_tenant = data.tenant
|
||||
data.caller = account
|
||||
data.caller_kind = "account"
|
||||
|
||||
|
||||
def resolve_external_user(data: AuthData) -> None:
|
||||
if data.tenant is None or data.app is None or data.external_identity is None:
|
||||
raise Unauthorized("missing context for external user resolution")
|
||||
end_user = EndUserService.get_or_create_end_user_by_type(
|
||||
InvokeFrom.OPENAPI,
|
||||
tenant_id=str(data.tenant.id),
|
||||
app_id=str(data.app.id),
|
||||
user_id=data.external_identity.email,
|
||||
)
|
||||
data.caller = end_user
|
||||
data.caller_kind = "end_user"
|
||||
|
||||
|
||||
def load_app_access_mode(data: AuthData) -> None:
|
||||
if data.app is None:
|
||||
return
|
||||
try:
|
||||
settings = EnterpriseService.WebAppAuth.get_app_access_mode_by_id(app_id=str(data.app.id))
|
||||
if settings is None:
|
||||
data.app_access_mode = None
|
||||
return
|
||||
data.app_access_mode = WebAppAccessMode(settings.access_mode)
|
||||
except ValueError:
|
||||
data.app_access_mode = None
|
||||
@@ -1,170 +0,0 @@
|
||||
"""Pipeline steps. Each is one responsibility.
|
||||
|
||||
`BearerCheck` is the only step that touches the token registry; downstream
|
||||
steps see only the populated `Context`. `BearerCheck` also publishes the
|
||||
resolved identity to the openapi auth ``ContextVar`` (the same one the
|
||||
decorator-level :func:`libs.oauth_bearer.validate_bearer` writes to) so the
|
||||
surface gate and any handler reading the request-scoped context has a single
|
||||
source of truth across both auth-attach paths. The reset token is stashed
|
||||
on `ctx.auth_ctx_reset_token`; `Pipeline.guard` resets the ContextVar in
|
||||
its `finally` so worker-thread reuse can't leak identity across requests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from werkzeug.exceptions import BadRequest, Forbidden, NotFound, Unauthorized
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.strategies import AppAuthzStrategy, CallerMounter
|
||||
from controllers.openapi.auth.surface_gate import check_surface
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import (
|
||||
AuthContext,
|
||||
InvalidBearerError,
|
||||
Scope,
|
||||
SubjectType,
|
||||
check_workspace_membership,
|
||||
get_authenticator,
|
||||
set_auth_ctx,
|
||||
)
|
||||
from models import TenantStatus
|
||||
from services.account_service import TenantService
|
||||
from services.app_service import AppService
|
||||
|
||||
|
||||
class BearerCheck:
|
||||
"""Resolve bearer → populate identity fields. Rate-limit is enforced
|
||||
inside `BearerAuthenticator.authenticate`, so no separate step here.
|
||||
Also publishes the resolved `AuthContext` via
|
||||
:func:`libs.oauth_bearer.set_auth_ctx` — same shape the decorator-level
|
||||
``validate_bearer`` writes — so the surface gate + downstream readers
|
||||
don't see two different identity sources. The reset token is parked on
|
||||
``ctx.auth_ctx_reset_token`` for `Pipeline.guard` to consume."""
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
if not ctx.bearer_token:
|
||||
raise Unauthorized("bearer required")
|
||||
|
||||
try:
|
||||
authn = get_authenticator().authenticate(ctx.bearer_token)
|
||||
except InvalidBearerError as e:
|
||||
raise Unauthorized(str(e))
|
||||
|
||||
ctx.subject_type = authn.subject_type
|
||||
ctx.subject_email = authn.subject_email
|
||||
ctx.subject_issuer = authn.subject_issuer
|
||||
ctx.account_id = authn.account_id
|
||||
ctx.scopes = frozenset(authn.scopes)
|
||||
ctx.source = authn.source
|
||||
ctx.token_id = authn.token_id
|
||||
ctx.expires_at = authn.expires_at
|
||||
ctx.token_hash = authn.token_hash
|
||||
ctx.cached_verified_tenants = dict(authn.verified_tenants)
|
||||
ctx.auth_ctx_reset_token = set_auth_ctx(authn)
|
||||
|
||||
|
||||
class ScopeCheck:
|
||||
"""Verify ctx.scopes (already populated by BearerCheck) covers required."""
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
if Scope.FULL in ctx.scopes or ctx.required_scope in ctx.scopes:
|
||||
return
|
||||
raise Forbidden("insufficient_scope")
|
||||
|
||||
|
||||
class SurfaceCheck:
|
||||
"""Reject the request if the resolved subject is not in `accepted`."""
|
||||
|
||||
def __init__(self, *, accepted: frozenset[SubjectType]) -> None:
|
||||
self._accepted = accepted
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
check_surface(self._accepted)
|
||||
|
||||
|
||||
class AppResolver:
|
||||
"""Read ``app_id`` from ``ctx.path_params``; populate ctx.app + ctx.tenant.
|
||||
|
||||
Every endpoint using the OAuth bearer pipeline must declare
|
||||
``<string:app_id>`` in its route — that is the design lock-in (no body /
|
||||
header coupling). ``Pipeline.guard`` lifts ``request.view_args`` into
|
||||
``ctx.path_params`` at the boundary so this step doesn't need to know
|
||||
about the request object.
|
||||
"""
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
app_id = ctx.path_params.get("app_id")
|
||||
if not app_id:
|
||||
raise BadRequest("app_id is required in path")
|
||||
app = AppService.get_app_by_id(db.session, app_id)
|
||||
if not app or app.status != "normal":
|
||||
raise NotFound("app not found")
|
||||
if not app.enable_api:
|
||||
raise Forbidden("service_api_disabled")
|
||||
tenant = TenantService.get_tenant_by_id(db.session, str(app.tenant_id))
|
||||
if tenant is None or tenant.status == TenantStatus.ARCHIVE:
|
||||
raise Forbidden("workspace unavailable")
|
||||
ctx.app, ctx.tenant = app, tenant
|
||||
|
||||
|
||||
class WorkspaceMembershipCheck:
|
||||
"""Layer 0 — workspace membership gate.
|
||||
|
||||
CE-only (skipped when ENTERPRISE_ENABLED). Account-subject bearers
|
||||
(dfoa_) only — SSO subjects skip.
|
||||
"""
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
if dify_config.ENTERPRISE_ENABLED:
|
||||
return
|
||||
if ctx.subject_type != SubjectType.ACCOUNT:
|
||||
return
|
||||
if ctx.account_id is None or ctx.tenant is None:
|
||||
raise Unauthorized("account_id or tenant unset — BearerCheck or AppResolver did not run")
|
||||
if ctx.token_hash is None:
|
||||
raise Unauthorized("token_hash unset — BearerCheck did not run")
|
||||
|
||||
check_workspace_membership(
|
||||
account_id=ctx.account_id,
|
||||
tenant_id=ctx.must_tenant.id,
|
||||
token_hash=ctx.token_hash,
|
||||
cached_verdicts=ctx.cached_verified_tenants or {},
|
||||
)
|
||||
|
||||
|
||||
class AppAuthzCheck:
|
||||
def __init__(self, resolve_strategy: Callable[[], AppAuthzStrategy]) -> None:
|
||||
self._resolve = resolve_strategy
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
if not self._resolve().authorize(ctx):
|
||||
raise Forbidden("subject_no_app_access")
|
||||
|
||||
|
||||
class CallerMount:
|
||||
def __init__(self, *mounters: CallerMounter) -> None:
|
||||
self._mounters = mounters
|
||||
|
||||
def __call__(self, ctx: Context) -> None:
|
||||
if ctx.subject_type is None:
|
||||
raise Unauthorized("subject_type unset — BearerCheck did not run")
|
||||
for m in self._mounters:
|
||||
if m.applies_to(ctx.must_subject_type):
|
||||
m.mount(ctx)
|
||||
return
|
||||
raise Unauthorized("no caller mounter for subject type")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppAuthzCheck",
|
||||
"AppResolver",
|
||||
"AuthContext",
|
||||
"BearerCheck",
|
||||
"CallerMount",
|
||||
"ScopeCheck",
|
||||
"SurfaceCheck",
|
||||
"WorkspaceMembershipCheck",
|
||||
]
|
||||
@@ -1,168 +0,0 @@
|
||||
"""Strategy classes for the openapi auth pipeline.
|
||||
|
||||
App authorization (Acl/Membership) and caller mounting (Account/EndUser)
|
||||
vary along independent axes; each strategy is one class so the pipeline
|
||||
composition stays a flat list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from flask import current_app
|
||||
from flask_login import user_logged_in
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import SubjectType
|
||||
from services.account_service import AccountService, TenantService
|
||||
from services.end_user_service import EndUserService
|
||||
from services.enterprise.enterprise_service import (
|
||||
EnterpriseService,
|
||||
WebAppAccessMode,
|
||||
)
|
||||
|
||||
|
||||
class AppAuthzStrategy(Protocol):
|
||||
def authorize(self, ctx: Context) -> bool: ...
|
||||
|
||||
|
||||
class AclStrategy:
|
||||
"""Per-app ACL, evaluated in two stages.
|
||||
|
||||
The EE gateway has already enforced tenancy and workspace membership
|
||||
by the time this strategy runs, so AclStrategy only owns per-app ACL:
|
||||
|
||||
1. Subject vs access-mode compatibility (pure rule table). External-SSO
|
||||
bearers belong to public-facing apps only; account bearers cover the
|
||||
full set. A mismatch is an immediate deny — no IO.
|
||||
2. For modes that pair with the subject, decide whether the inner
|
||||
permission API must run. Only `PRIVATE` (per-app selected-user list)
|
||||
requires it; the remaining modes are pass-through.
|
||||
"""
|
||||
|
||||
_ALLOWED_MODES_BY_SUBJECT: dict[SubjectType, frozenset[WebAppAccessMode]] = {
|
||||
SubjectType.ACCOUNT: frozenset(
|
||||
{
|
||||
WebAppAccessMode.PUBLIC,
|
||||
WebAppAccessMode.SSO_VERIFIED,
|
||||
WebAppAccessMode.PRIVATE_ALL,
|
||||
WebAppAccessMode.PRIVATE,
|
||||
}
|
||||
),
|
||||
SubjectType.EXTERNAL_SSO: frozenset(
|
||||
{
|
||||
WebAppAccessMode.PUBLIC,
|
||||
WebAppAccessMode.SSO_VERIFIED,
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
_MODES_REQUIRING_INNER_CHECK: frozenset[WebAppAccessMode] = frozenset({WebAppAccessMode.PRIVATE})
|
||||
|
||||
def authorize(self, ctx: Context) -> bool:
|
||||
if ctx.app is None:
|
||||
return False
|
||||
access_mode = self._fetch_access_mode(ctx.app.id)
|
||||
if access_mode is None:
|
||||
return False
|
||||
if not self._subject_allowed_for_mode(ctx.must_subject_type, access_mode):
|
||||
return False
|
||||
if access_mode not in self._MODES_REQUIRING_INNER_CHECK:
|
||||
return True
|
||||
return self._inner_permission_check(ctx)
|
||||
|
||||
@staticmethod
|
||||
def _fetch_access_mode(app_id: str) -> WebAppAccessMode | None:
|
||||
settings = EnterpriseService.WebAppAuth.get_app_access_mode_by_id(app_id=app_id)
|
||||
if settings is None:
|
||||
return None
|
||||
try:
|
||||
return WebAppAccessMode(settings.access_mode)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _subject_allowed_for_mode(cls, subject_type: SubjectType, access_mode: WebAppAccessMode) -> bool:
|
||||
return access_mode in cls._ALLOWED_MODES_BY_SUBJECT.get(subject_type, frozenset())
|
||||
|
||||
def _inner_permission_check(self, ctx: Context) -> bool:
|
||||
if ctx.app is None:
|
||||
return False
|
||||
user_id = self._resolve_user_id(ctx)
|
||||
if user_id is None:
|
||||
return False
|
||||
return EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp(
|
||||
user_id=user_id,
|
||||
app_id=ctx.app.id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_user_id(ctx: Context) -> str | None:
|
||||
if ctx.subject_type == SubjectType.ACCOUNT:
|
||||
return str(ctx.account_id) if ctx.account_id is not None else None
|
||||
if ctx.subject_email is None:
|
||||
return None
|
||||
account = AccountService.get_account_by_email(db.session, ctx.subject_email)
|
||||
return str(account.id) if account is not None else None
|
||||
|
||||
|
||||
class MembershipStrategy:
|
||||
"""Tenant-membership fallback.
|
||||
|
||||
Used when webapp-auth is disabled (CE deployment). Account-bearing
|
||||
subjects pass if they have a TenantAccountJoin row; EXTERNAL_SSO is
|
||||
denied (it requires the webapp-auth surface).
|
||||
"""
|
||||
|
||||
def authorize(self, ctx: Context) -> bool:
|
||||
if ctx.subject_type == SubjectType.EXTERNAL_SSO:
|
||||
return False
|
||||
if ctx.tenant is None:
|
||||
return False
|
||||
return TenantService.account_belongs_to_tenant(db.session, ctx.account_id, ctx.tenant.id)
|
||||
|
||||
|
||||
def _login_as(user) -> None:
|
||||
"""Set Flask-Login request user so downstream services see the caller."""
|
||||
current_app.login_manager._update_request_context_with_user(user) # type:ignore
|
||||
user_logged_in.send(current_app._get_current_object(), user=user) # type:ignore
|
||||
|
||||
|
||||
class CallerMounter(Protocol):
|
||||
def applies_to(self, subject_type: SubjectType) -> bool: ...
|
||||
|
||||
def mount(self, ctx: Context) -> None: ...
|
||||
|
||||
|
||||
class AccountMounter:
|
||||
def applies_to(self, subject_type: SubjectType) -> bool:
|
||||
return subject_type == SubjectType.ACCOUNT
|
||||
|
||||
def mount(self, ctx: Context) -> None:
|
||||
if ctx.account_id is None:
|
||||
raise RuntimeError("AccountMounter: account_id unset — BearerCheck did not run")
|
||||
account = AccountService.get_account_by_id(db.session, str(ctx.account_id))
|
||||
if account is None:
|
||||
raise RuntimeError("AccountMounter: account row missing for resolved bearer")
|
||||
account.current_tenant = ctx.must_tenant
|
||||
_login_as(account)
|
||||
ctx.caller, ctx.caller_kind = account, "account"
|
||||
|
||||
|
||||
class EndUserMounter:
|
||||
def applies_to(self, subject_type: SubjectType) -> bool:
|
||||
return subject_type == SubjectType.EXTERNAL_SSO
|
||||
|
||||
def mount(self, ctx: Context) -> None:
|
||||
if ctx.tenant is None or ctx.app is None or ctx.subject_email is None:
|
||||
raise RuntimeError("EndUserMounter: tenant/app/subject_email unset — earlier steps did not run")
|
||||
end_user = EndUserService.get_or_create_end_user_by_type(
|
||||
InvokeFrom.OPENAPI,
|
||||
tenant_id=ctx.tenant.id,
|
||||
app_id=ctx.app.id,
|
||||
user_id=ctx.subject_email,
|
||||
)
|
||||
_login_as(end_user)
|
||||
ctx.caller, ctx.caller_kind = end_user, "end_user"
|
||||
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from werkzeug.exceptions import Forbidden, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import Scope, TokenType, check_workspace_membership
|
||||
from services.account_service import AccountService, TenantService
|
||||
from services.enterprise.enterprise_service import EnterpriseService, WebAppAccessMode
|
||||
|
||||
|
||||
def check_scope(data: AuthData) -> None:
|
||||
if data.required_scope is None:
|
||||
return
|
||||
if Scope.FULL in data.scopes or data.required_scope in data.scopes:
|
||||
return
|
||||
raise Forbidden("insufficient_scope")
|
||||
|
||||
|
||||
def check_membership(data: AuthData) -> None:
|
||||
if data.tenant is None:
|
||||
raise Unauthorized("tenant unset")
|
||||
if data.account_id is None:
|
||||
raise Unauthorized("account_id unset")
|
||||
check_workspace_membership(
|
||||
account_id=data.account_id,
|
||||
tenant_id=data.tenant.id,
|
||||
token_hash=data.token_hash,
|
||||
membership_cache=data.tenants,
|
||||
)
|
||||
|
||||
|
||||
def check_app_access(data: AuthData) -> None:
|
||||
if data.tenant is None:
|
||||
return
|
||||
if not TenantService.account_belongs_to_tenant(db.session, data.account_id, data.tenant.id):
|
||||
raise Forbidden("subject_no_app_access")
|
||||
|
||||
|
||||
_ALLOWED_MODES_BY_TOKEN_TYPE: dict[TokenType, frozenset[WebAppAccessMode]] = {
|
||||
TokenType.OAUTH_ACCOUNT: frozenset(
|
||||
{
|
||||
WebAppAccessMode.PUBLIC,
|
||||
WebAppAccessMode.SSO_VERIFIED,
|
||||
WebAppAccessMode.PRIVATE_ALL,
|
||||
WebAppAccessMode.PRIVATE,
|
||||
}
|
||||
),
|
||||
TokenType.OAUTH_EXTERNAL_SSO: frozenset(
|
||||
{
|
||||
WebAppAccessMode.PUBLIC,
|
||||
WebAppAccessMode.SSO_VERIFIED,
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def check_acl(data: AuthData) -> None:
|
||||
if data.app is None or data.app_access_mode is None:
|
||||
raise Forbidden("app or access mode not loaded")
|
||||
allowed_modes = _ALLOWED_MODES_BY_TOKEN_TYPE.get(data.token_type, frozenset())
|
||||
if data.app_access_mode not in allowed_modes:
|
||||
raise Forbidden("subject_not_allowed_for_access_mode")
|
||||
|
||||
|
||||
def check_private_app_permission(data: AuthData) -> None:
|
||||
if data.app is None:
|
||||
raise Forbidden("app not loaded")
|
||||
user_id = _resolve_user_id(data)
|
||||
if user_id is None:
|
||||
raise Forbidden("cannot resolve user for private app check")
|
||||
if not EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp(user_id=user_id, app_id=data.app.id):
|
||||
raise Forbidden("user_not_allowed_for_private_app")
|
||||
|
||||
|
||||
def _resolve_user_id(data: AuthData) -> str | None:
|
||||
if data.token_type == TokenType.OAUTH_ACCOUNT:
|
||||
return str(data.account_id) if data.account_id is not None else None
|
||||
if data.external_identity is None:
|
||||
return None
|
||||
account = AccountService.get_account_by_email(db.session, data.external_identity.email)
|
||||
return str(account.id) if account is not None else None
|
||||
@@ -17,11 +17,11 @@ from controllers.common.errors import (
|
||||
UnsupportedFileTypeError,
|
||||
)
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from extensions.ext_database import db
|
||||
from fields.file_fields import FileResponse
|
||||
from libs.oauth_bearer import Scope
|
||||
from models import Account, App
|
||||
from services.file_service import FileService
|
||||
|
||||
|
||||
@@ -39,8 +39,9 @@ class AppFileUploadApi(Resource):
|
||||
}
|
||||
)
|
||||
@openapi_ns.response(HTTPStatus.CREATED, "File uploaded", openapi_ns.models[FileResponse.__name__])
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, app_model: App, caller: Account, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, *, auth_data: AuthData):
|
||||
app_model, caller, _ = auth_data.require_app_context()
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
if len(request.files) > 1:
|
||||
|
||||
@@ -17,7 +17,8 @@ from werkzeug.exceptions import BadRequest, NotFound
|
||||
from controllers.common.human_input import HumanInputFormSubmitPayload, stringify_form_default_values
|
||||
from controllers.common.schema import register_schema_models
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from core.workflow.human_input_policy import HumanInputSurface, is_recipient_type_allowed_for_surface
|
||||
from extensions.ext_database import db
|
||||
from libs.helper import to_timestamp
|
||||
@@ -55,8 +56,9 @@ def _ensure_form_is_allowed_for_openapi(form) -> None:
|
||||
@openapi_ns.route("/apps/<string:app_id>/form/human_input/<string:form_token>")
|
||||
class OpenApiWorkflowHumanInputFormApi(Resource):
|
||||
@openapi_ns.response(200, "Form definition")
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def get(self, app_id: str, form_token: str, app_model: App, caller, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def get(self, app_id: str, form_token: str, *, auth_data: AuthData):
|
||||
app_model, caller, caller_kind = auth_data.require_app_context()
|
||||
service = HumanInputService(db.engine)
|
||||
form = service.get_form_by_token(form_token)
|
||||
if form is None:
|
||||
@@ -69,8 +71,9 @@ class OpenApiWorkflowHumanInputFormApi(Resource):
|
||||
|
||||
@openapi_ns.expect(openapi_ns.models[HumanInputFormSubmitPayload.__name__])
|
||||
@openapi_ns.response(200, "Form submitted")
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, form_token: str, app_model: App, caller, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def post(self, app_id: str, form_token: str, *, auth_data: AuthData):
|
||||
app_model, caller, caller_kind = auth_data.require_app_context()
|
||||
payload = HumanInputFormSubmitPayload.model_validate(request.get_json(silent=True) or {})
|
||||
|
||||
service = HumanInputService(db.engine)
|
||||
|
||||
@@ -17,7 +17,8 @@ from sqlalchemy.orm import sessionmaker
|
||||
from werkzeug.exceptions import NotFound, UnprocessableEntity
|
||||
|
||||
from controllers.openapi import openapi_ns
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from core.app.apps.advanced_chat.app_generator import AdvancedChatAppGenerator
|
||||
from core.app.apps.base_app_generator import BaseAppGenerator
|
||||
from core.app.apps.common.workflow_response_converter import WorkflowResponseConverter
|
||||
@@ -28,7 +29,7 @@ from core.workflow.human_input_policy import HumanInputSurface
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import Scope
|
||||
from models.enums import CreatorUserRole
|
||||
from models.model import App, AppMode
|
||||
from models.model import AppMode
|
||||
from repositories.factory import DifyAPIRepositoryFactory
|
||||
from services.workflow_event_snapshot_service import build_workflow_event_stream
|
||||
|
||||
@@ -36,8 +37,9 @@ from services.workflow_event_snapshot_service import build_workflow_event_stream
|
||||
@openapi_ns.route("/apps/<string:app_id>/tasks/<string:task_id>/events")
|
||||
class OpenApiWorkflowEventsApi(Resource):
|
||||
@openapi_ns.response(200, "SSE event stream")
|
||||
@OAUTH_BEARER_PIPELINE.guard(scope=Scope.APPS_RUN)
|
||||
def get(self, app_id: str, task_id: str, app_model: App, caller, caller_kind: str):
|
||||
@auth_router.guard(scope=Scope.APPS_RUN)
|
||||
def get(self, app_id: str, task_id: str, *, auth_data: AuthData):
|
||||
app_model, caller, caller_kind = auth_data.require_app_context()
|
||||
app_mode = AppMode.value_of(app_model.mode)
|
||||
if app_mode not in {AppMode.WORKFLOW, AppMode.ADVANCED_CHAT}:
|
||||
raise UnprocessableEntity("mode_not_supported_for_event_reconnect")
|
||||
|
||||
@@ -35,15 +35,11 @@ from controllers.openapi._models import (
|
||||
WorkspaceListResponse,
|
||||
WorkspaceSummaryResponse,
|
||||
)
|
||||
from controllers.openapi.auth.composition import auth_router
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from controllers.openapi.auth.role_gate import require_workspace_role
|
||||
from controllers.openapi.auth.surface_gate import accept_subjects
|
||||
from extensions.ext_database import db
|
||||
from libs.oauth_bearer import (
|
||||
ACCEPT_USER_ANY,
|
||||
SubjectType,
|
||||
get_auth_ctx,
|
||||
validate_bearer,
|
||||
)
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models import Account, Tenant, TenantAccountJoin
|
||||
from models.account import TenantAccountRole, TenantStatus
|
||||
from services.account_service import AccountService, RegisterService, TenantService
|
||||
@@ -60,11 +56,6 @@ from services.feature_service import FeatureService
|
||||
|
||||
|
||||
def _validate_body[M: BaseModel](model: type[M]) -> M:
|
||||
"""Validate JSON body against ``model``. Validation errors → HTTP 400.
|
||||
|
||||
The workspace spec is explicit that bad email / unknown role payloads
|
||||
are 400, not Pydantic's default 422 — handle uniformly here.
|
||||
"""
|
||||
body = request.get_json(silent=True) or {}
|
||||
try:
|
||||
return model.model_validate(body)
|
||||
@@ -91,7 +82,6 @@ def _load_tenant(workspace_id: str) -> Tenant:
|
||||
|
||||
|
||||
def _load_account(account_id: object) -> Account:
|
||||
"""Load the caller's Account. Missing == auth wiring bug, not user error."""
|
||||
account = AccountService.get_account_by_id(db.session, str(account_id)) if account_id else None
|
||||
if account is None:
|
||||
raise RuntimeError("authenticated account_id has no Account row")
|
||||
@@ -99,13 +89,6 @@ def _load_account(account_id: object) -> Account:
|
||||
|
||||
|
||||
def _quota_error(*, code: str, message: str, hint: str) -> Forbidden:
|
||||
"""Build a 403 with envelope ``{code, message, hint}``.
|
||||
|
||||
CLI ``error-mapper`` reads ``message`` and ``hint`` off the wire body
|
||||
verbatim — the structured envelope lets it surface remediation guidance
|
||||
(e.g. "upgrade your plan") without the CLI needing to know edition
|
||||
semantics.
|
||||
"""
|
||||
err = Forbidden(message)
|
||||
err.response = make_response(
|
||||
jsonify({"code": code, "message": message, "hint": hint}),
|
||||
@@ -115,16 +98,6 @@ def _quota_error(*, code: str, message: str, hint: str) -> Forbidden:
|
||||
|
||||
|
||||
def _check_member_invite_quota(tenant_id: str) -> None:
|
||||
"""Edition-aware member-count gate for invite.
|
||||
|
||||
Both branches self-disable on CE because ``FeatureService.get_features``
|
||||
leaves ``billing.enabled`` and ``workspace_members.enabled`` False by
|
||||
default; SaaS billing API and EE license activation are what flip them on.
|
||||
|
||||
Mirrors the two checks the console invite path performs (decorator at
|
||||
``console/wraps.py:106`` for billing + inline at
|
||||
``console/workspace/members.py:130`` for license).
|
||||
"""
|
||||
features = FeatureService.get_features(tenant_id)
|
||||
|
||||
if features.billing.enabled:
|
||||
@@ -148,12 +121,9 @@ def _check_member_invite_quota(tenant_id: str) -> None:
|
||||
@openapi_ns.route("/workspaces")
|
||||
class WorkspacesApi(Resource):
|
||||
@openapi_ns.response(200, "Workspace list", openapi_ns.models[WorkspaceListResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
def get(self):
|
||||
ctx = get_auth_ctx()
|
||||
|
||||
rows = TenantService.get_workspaces_for_account(db.session, str(ctx.account_id))
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, *, auth_data: AuthData):
|
||||
rows = TenantService.get_workspaces_for_account(db.session, str(auth_data.account_id))
|
||||
|
||||
return WorkspaceListResponse(workspaces=list(starmap(_workspace_summary, rows))).model_dump(mode="json"), 200
|
||||
|
||||
@@ -161,12 +131,9 @@ class WorkspacesApi(Resource):
|
||||
@openapi_ns.route("/workspaces/<string:workspace_id>")
|
||||
class WorkspaceByIdApi(Resource):
|
||||
@openapi_ns.response(200, "Workspace detail", openapi_ns.models[WorkspaceDetailResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
def get(self, workspace_id: str):
|
||||
ctx = get_auth_ctx()
|
||||
|
||||
row = TenantService.find_workspace_for_account(db.session, str(ctx.account_id), workspace_id)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def get(self, workspace_id: str, *, auth_data: AuthData):
|
||||
row = TenantService.find_workspace_for_account(db.session, str(auth_data.account_id), workspace_id)
|
||||
# 404 (not 403) on non-member so workspace IDs don't leak across tenants.
|
||||
if row is None:
|
||||
raise NotFound("workspace not found")
|
||||
@@ -185,21 +152,17 @@ class WorkspaceSwitchApi(Resource):
|
||||
"""
|
||||
|
||||
@openapi_ns.response(200, "Workspace detail", openapi_ns.models[WorkspaceDetailResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
@require_workspace_role()
|
||||
def post(self, workspace_id: str):
|
||||
ctx = get_auth_ctx()
|
||||
account = _load_account(ctx.account_id)
|
||||
def post(self, workspace_id: str, *, auth_data: AuthData):
|
||||
account = _load_account(auth_data.account_id)
|
||||
|
||||
try:
|
||||
TenantService.switch_tenant(account, workspace_id)
|
||||
except AccountNotLinkTenantError:
|
||||
# Membership existed at gate time but Tenant.status != NORMAL or
|
||||
# the row was just removed — treat as not-found.
|
||||
raise NotFound("workspace not found")
|
||||
|
||||
row = TenantService.find_workspace_for_account(db.session, str(ctx.account_id), workspace_id)
|
||||
row = TenantService.find_workspace_for_account(db.session, str(auth_data.account_id), workspace_id)
|
||||
if row is None:
|
||||
raise NotFound("workspace not found")
|
||||
tenant, membership = row
|
||||
@@ -216,20 +179,15 @@ class WorkspaceMembersApi(Resource):
|
||||
|
||||
@openapi_ns.doc(params=query_params_from_model(MemberListQuery))
|
||||
@openapi_ns.response(200, "Member list", openapi_ns.models[MemberListResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_READ, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
@require_workspace_role()
|
||||
def get(self, workspace_id: str):
|
||||
def get(self, workspace_id: str, *, auth_data: AuthData):
|
||||
try:
|
||||
query = MemberListQuery.model_validate(request.args.to_dict(flat=True))
|
||||
except ValidationError as exc:
|
||||
raise BadRequest(str(exc))
|
||||
|
||||
tenant = _load_tenant(workspace_id)
|
||||
# Members per workspace are bounded by SaaS plan caps (≤50) or EE
|
||||
# license seats (low thousands worst-case), so we materialize and
|
||||
# slice in-memory rather than push pagination into the service —
|
||||
# matches how the rest of the service exposes member lists.
|
||||
members = TenantService.get_tenant_members(tenant)
|
||||
total = len(members)
|
||||
start = (query.page - 1) * query.limit
|
||||
@@ -244,13 +202,11 @@ class WorkspaceMembersApi(Resource):
|
||||
|
||||
@openapi_ns.expect(openapi_ns.models[MemberInvitePayload.__name__])
|
||||
@openapi_ns.response(201, "Member invited", openapi_ns.models[MemberInviteResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_WRITE, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
@require_workspace_role(TenantAccountRole.OWNER, TenantAccountRole.ADMIN)
|
||||
def post(self, workspace_id: str):
|
||||
def post(self, workspace_id: str, *, auth_data: AuthData):
|
||||
payload = _validate_body(MemberInvitePayload)
|
||||
ctx = get_auth_ctx()
|
||||
inviter = _load_account(ctx.account_id)
|
||||
inviter = _load_account(auth_data.account_id)
|
||||
tenant = _load_tenant(workspace_id)
|
||||
|
||||
_check_member_invite_quota(str(tenant.id))
|
||||
@@ -297,12 +253,10 @@ class WorkspaceMemberApi(Resource):
|
||||
"""
|
||||
|
||||
@openapi_ns.response(200, "Member removed", openapi_ns.models[MemberActionResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_WRITE, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
@require_workspace_role(TenantAccountRole.OWNER, TenantAccountRole.ADMIN)
|
||||
def delete(self, workspace_id: str, member_id: str):
|
||||
ctx = get_auth_ctx()
|
||||
operator = _load_account(ctx.account_id)
|
||||
def delete(self, workspace_id: str, member_id: str, *, auth_data: AuthData):
|
||||
operator = _load_account(auth_data.account_id)
|
||||
tenant = _load_tenant(workspace_id)
|
||||
member = AccountService.get_account_by_id(db.session, member_id)
|
||||
if member is None:
|
||||
@@ -330,13 +284,11 @@ class WorkspaceMemberRoleApi(Resource):
|
||||
|
||||
@openapi_ns.expect(openapi_ns.models[MemberRoleUpdatePayload.__name__])
|
||||
@openapi_ns.response(200, "Role updated", openapi_ns.models[MemberActionResponse.__name__])
|
||||
@validate_bearer(accept=ACCEPT_USER_ANY)
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
@auth_router.guard(scope=Scope.WORKSPACE_WRITE, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
@require_workspace_role(TenantAccountRole.OWNER, TenantAccountRole.ADMIN)
|
||||
def put(self, workspace_id: str, member_id: str):
|
||||
def put(self, workspace_id: str, member_id: str, *, auth_data: AuthData):
|
||||
payload = _validate_body(MemberRoleUpdatePayload)
|
||||
ctx = get_auth_ctx()
|
||||
operator = _load_account(ctx.account_id)
|
||||
operator = _load_account(auth_data.account_id)
|
||||
tenant = _load_tenant(workspace_id)
|
||||
member = AccountService.get_account_by_id(db.session, member_id)
|
||||
if member is None:
|
||||
|
||||
@@ -27,6 +27,7 @@ from core.moderation.base import ModerationError
|
||||
from core.moderation.input_moderation import InputModeration
|
||||
from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
|
||||
from core.workflow.node_factory import get_default_root_node_id
|
||||
from core.workflow.nodes.agent_v2.session_cleanup_layer import build_workflow_agent_session_cleanup_layer
|
||||
from core.workflow.system_variables import (
|
||||
build_bootstrap_variables,
|
||||
build_system_variables,
|
||||
@@ -239,6 +240,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
|
||||
)
|
||||
|
||||
workflow_entry.graph_engine.layer(persistence_layer)
|
||||
workflow_entry.graph_engine.layer(build_workflow_agent_session_cleanup_layer())
|
||||
conversation_variable_layer = ConversationVariablePersistenceLayer(
|
||||
ConversationVariableUpdater(session_factory.get_session_maker())
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from core.app.entities.app_invoke_entities import InvokeFrom, WorkflowAppGenerat
|
||||
from core.app.workflow.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer
|
||||
from core.repositories.factory import WorkflowExecutionRepository, WorkflowNodeExecutionRepository
|
||||
from core.workflow.node_factory import get_default_root_node_id
|
||||
from core.workflow.nodes.agent_v2.session_cleanup_layer import build_workflow_agent_session_cleanup_layer
|
||||
from core.workflow.system_variables import build_bootstrap_variables, build_system_variables
|
||||
from core.workflow.variable_pool_initializer import add_node_inputs_to_pool, add_variables_to_pool
|
||||
from core.workflow.workflow_entry import WorkflowEntry
|
||||
@@ -166,6 +167,7 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
|
||||
)
|
||||
|
||||
workflow_entry.graph_engine.layer(persistence_layer)
|
||||
workflow_entry.graph_engine.layer(build_workflow_agent_session_cleanup_layer())
|
||||
for layer in self._graph_engine_layers:
|
||||
workflow_entry.graph_engine.layer(layer)
|
||||
|
||||
|
||||
@@ -863,7 +863,7 @@ class ToolManager:
|
||||
return controller
|
||||
|
||||
@classmethod
|
||||
def user_get_api_provider(cls, provider: str, tenant_id: str):
|
||||
def user_get_api_provider(cls, provider: str, tenant_id: str, mask: bool = True):
|
||||
"""
|
||||
get api provider
|
||||
"""
|
||||
@@ -902,8 +902,10 @@ class ToolManager:
|
||||
tenant_id=tenant_id,
|
||||
controller=controller,
|
||||
)
|
||||
|
||||
masked_credentials = encrypter.mask_plugin_credentials(encrypter.decrypt(credentials))
|
||||
if mask:
|
||||
masked_credentials = encrypter.mask_plugin_credentials(encrypter.decrypt(credentials))
|
||||
else:
|
||||
masked_credentials = encrypter.decrypt(credentials)
|
||||
|
||||
try:
|
||||
icon = emoji_icon_adapter.validate_json(provider_obj.icon)
|
||||
|
||||
@@ -6,7 +6,7 @@ from json.decoder import JSONDecodeError
|
||||
from typing import Any, TypedDict
|
||||
|
||||
import httpx
|
||||
from flask import request
|
||||
from flask import has_request_context, request
|
||||
from yaml import YAMLError, safe_load
|
||||
|
||||
from core.tools.entities.common_entities import I18nObject
|
||||
@@ -44,7 +44,7 @@ class ApiBasedToolSchemaParser:
|
||||
raise ToolProviderNotFoundError("No server found in the openapi yaml.")
|
||||
|
||||
server_url = openapi["servers"][0]["url"]
|
||||
request_env = request.headers.get("X-Request-Env")
|
||||
request_env = request.headers.get("X-Request-Env") if has_request_context() else None
|
||||
if request_env:
|
||||
matched_servers = [server["url"] for server in openapi["servers"] if server["env"] == request_env]
|
||||
server_url = matched_servers[0] if matched_servers else server_url
|
||||
|
||||
@@ -475,6 +475,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
from core.workflow.nodes.agent_v2.file_tenant_validator import UploadFileTenantValidator
|
||||
from core.workflow.nodes.agent_v2.output_failure_orchestrator import OutputFailureOrchestrator
|
||||
from core.workflow.nodes.agent_v2.output_type_checker import PerOutputTypeChecker
|
||||
from core.workflow.nodes.agent_v2.session_store import WorkflowAgentRuntimeSessionStore
|
||||
|
||||
return {
|
||||
"binding_resolver": WorkflowAgentBindingResolver(),
|
||||
@@ -494,6 +495,7 @@ class DifyNodeFactory(NodeFactory):
|
||||
# outputs contain no file refs.
|
||||
"type_checker": PerOutputTypeChecker(file_validator=UploadFileTenantValidator()),
|
||||
"failure_orchestrator": OutputFailureOrchestrator(),
|
||||
"session_store": WorkflowAgentRuntimeSessionStore(),
|
||||
}
|
||||
return {
|
||||
"strategy_resolver": self._agent_strategy_resolver,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
|
||||
from clients.agent_backend import (
|
||||
AgentBackendError,
|
||||
AgentBackendHTTPError,
|
||||
@@ -17,11 +20,14 @@ from clients.agent_backend import (
|
||||
AgentBackendStreamInternalEvent,
|
||||
AgentBackendTransportError,
|
||||
AgentBackendValidationError,
|
||||
CleanupLayerSpec,
|
||||
extract_cleanup_layer_specs,
|
||||
)
|
||||
from core.app.entities.app_invoke_entities import DIFY_RUN_CONTEXT_KEY, DifyRunContext
|
||||
from core.workflow.system_variables import SystemVariableKey, get_system_text
|
||||
from graphon.entities.pause_reason import SchedulingPause
|
||||
from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
|
||||
from graphon.node_events import NodeEventBase, NodeRunResult, StreamCompletedEvent
|
||||
from graphon.node_events import NodeEventBase, NodeRunResult, PauseRequestedEvent, StreamCompletedEvent
|
||||
from graphon.nodes.base.node import Node
|
||||
from models.agent_config_entities import WorkflowNodeJobConfig
|
||||
|
||||
@@ -40,11 +46,14 @@ from .runtime_request_builder import (
|
||||
WorkflowAgentRuntimeRequestBuilder,
|
||||
WorkflowAgentRuntimeRequestBuildError,
|
||||
)
|
||||
from .session_store import WorkflowAgentRuntimeSessionStore, WorkflowAgentSessionScope
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from graphon.entities import GraphInitParams
|
||||
from graphon.runtime import GraphRuntimeState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Stage 4 §5+§7: the terminal events that `_consume_event_stream` may return.
|
||||
# Stream + started events are filtered out before we yield; transport errors
|
||||
@@ -74,6 +83,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
output_adapter: WorkflowAgentOutputAdapter,
|
||||
type_checker: PerOutputTypeChecker,
|
||||
failure_orchestrator: OutputFailureOrchestrator,
|
||||
session_store: WorkflowAgentRuntimeSessionStore | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
node_id=node_id,
|
||||
@@ -88,6 +98,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
self._output_adapter = output_adapter
|
||||
self._type_checker = type_checker
|
||||
self._failure_orchestrator = failure_orchestrator
|
||||
self._session_store = session_store
|
||||
|
||||
@classmethod
|
||||
def version(cls) -> str:
|
||||
@@ -134,6 +145,17 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
"agent_config_snapshot_id": bundle.snapshot.id,
|
||||
"binding_id": bundle.binding.id,
|
||||
}
|
||||
session_scope = WorkflowAgentSessionScope(
|
||||
tenant_id=dify_ctx.tenant_id,
|
||||
app_id=dify_ctx.app_id,
|
||||
workflow_id=workflow_id,
|
||||
workflow_run_id=workflow_run_id,
|
||||
node_id=self._node_id,
|
||||
node_execution_id=self.id,
|
||||
binding_id=bundle.binding.id,
|
||||
agent_id=bundle.agent.id,
|
||||
agent_config_snapshot_id=bundle.snapshot.id,
|
||||
)
|
||||
|
||||
# Stage 4 §4.1 (D-3): use effective outputs so defaults flow through both
|
||||
# the backend request and the post-run type check.
|
||||
@@ -147,6 +169,9 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
session_snapshot = None
|
||||
if self._session_store is not None:
|
||||
session_snapshot = self._session_store.load_active_snapshot(session_scope)
|
||||
runtime_request = self._runtime_request_builder.build(
|
||||
WorkflowAgentRuntimeBuildContext(
|
||||
dify_context=dify_ctx,
|
||||
@@ -159,6 +184,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
agent=bundle.agent,
|
||||
snapshot=bundle.snapshot,
|
||||
attempt=attempt,
|
||||
session_snapshot=session_snapshot,
|
||||
)
|
||||
)
|
||||
except WorkflowAgentRuntimeRequestBuildError as error:
|
||||
@@ -221,9 +247,35 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
)
|
||||
return
|
||||
|
||||
# Non-success terminal (failed / cancelled / paused) skips per-output
|
||||
# post-processing — the backend itself already failed.
|
||||
if isinstance(terminal_event, AgentBackendRunPausedInternalEvent):
|
||||
self._save_session_snapshot(
|
||||
session_scope=session_scope,
|
||||
backend_run_id=terminal_event.run_id,
|
||||
snapshot=terminal_event.session_snapshot,
|
||||
composition_layer_specs=extract_cleanup_layer_specs(runtime_request.request.composition),
|
||||
metadata=metadata,
|
||||
)
|
||||
yield PauseRequestedEvent(
|
||||
reason=SchedulingPause(
|
||||
message=terminal_event.message
|
||||
or "Agent backend run requested workflow pause for external input."
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
# Non-success terminal (failed / cancelled) skips per-output
|
||||
# post-processing — the backend itself already failed. We also retire
|
||||
# the local ACTIVE session row so a workflow loop back into the same
|
||||
# Agent node cannot resume from a stale snapshot. The failed agent
|
||||
# backend layers (suspended per ``on_exit``) are left for agent
|
||||
# backend's own GC; this row will no longer be picked up by the
|
||||
# workflow-terminal cleanup layer.
|
||||
if not isinstance(terminal_event, AgentBackendRunSucceededInternalEvent):
|
||||
self._mark_session_cleaned_on_failure(
|
||||
session_scope=session_scope,
|
||||
backend_run_id=terminal_event.run_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
yield StreamCompletedEvent(
|
||||
node_run_result=self._output_adapter.build_failure_result(
|
||||
event=terminal_event,
|
||||
@@ -234,6 +286,14 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
)
|
||||
return
|
||||
|
||||
self._save_session_snapshot(
|
||||
session_scope=session_scope,
|
||||
backend_run_id=terminal_event.run_id,
|
||||
snapshot=terminal_event.session_snapshot,
|
||||
composition_layer_specs=extract_cleanup_layer_specs(runtime_request.request.composition),
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# ──── Stage 4: per-output type check ────
|
||||
type_check = self._type_checker.check(
|
||||
declared_outputs=effective_outputs,
|
||||
@@ -384,6 +444,75 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
],
|
||||
}
|
||||
|
||||
def _save_session_snapshot(
|
||||
self,
|
||||
*,
|
||||
session_scope: WorkflowAgentSessionScope,
|
||||
backend_run_id: str,
|
||||
snapshot: CompositorSessionSnapshot | None,
|
||||
composition_layer_specs: list[CleanupLayerSpec],
|
||||
metadata: dict[str, Any],
|
||||
) -> None:
|
||||
if self._session_store is None:
|
||||
return
|
||||
try:
|
||||
self._session_store.save_active_snapshot(
|
||||
scope=session_scope,
|
||||
backend_run_id=backend_run_id,
|
||||
snapshot=snapshot,
|
||||
composition_layer_specs=composition_layer_specs,
|
||||
)
|
||||
agent_backend = dict(metadata.get("agent_backend") or {})
|
||||
agent_backend["session_snapshot_persisted"] = snapshot is not None
|
||||
metadata["agent_backend"] = agent_backend
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to persist workflow Agent runtime session snapshot: "
|
||||
"tenant_id=%s workflow_run_id=%s node_id=%s binding_id=%s agent_id=%s backend_run_id=%s",
|
||||
session_scope.tenant_id,
|
||||
session_scope.workflow_run_id,
|
||||
session_scope.node_id,
|
||||
session_scope.binding_id,
|
||||
session_scope.agent_id,
|
||||
backend_run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
agent_backend = dict(metadata.get("agent_backend") or {})
|
||||
agent_backend["session_snapshot_persisted"] = False
|
||||
agent_backend["session_snapshot_persist_error"] = "workflow_agent_runtime_session_store_error"
|
||||
metadata["agent_backend"] = agent_backend
|
||||
|
||||
def _mark_session_cleaned_on_failure(
|
||||
self,
|
||||
*,
|
||||
session_scope: WorkflowAgentSessionScope,
|
||||
backend_run_id: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> None:
|
||||
if self._session_store is None:
|
||||
return
|
||||
try:
|
||||
self._session_store.mark_cleaned(scope=session_scope, backend_run_id=backend_run_id)
|
||||
agent_backend = dict(metadata.get("agent_backend") or {})
|
||||
agent_backend["session_snapshot_cleaned_on_failure"] = True
|
||||
metadata["agent_backend"] = agent_backend
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to mark workflow Agent runtime session cleaned on agent run failure: "
|
||||
"tenant_id=%s workflow_run_id=%s node_id=%s binding_id=%s agent_id=%s backend_run_id=%s",
|
||||
session_scope.tenant_id,
|
||||
session_scope.workflow_run_id,
|
||||
session_scope.node_id,
|
||||
session_scope.binding_id,
|
||||
session_scope.agent_id,
|
||||
backend_run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
agent_backend = dict(metadata.get("agent_backend") or {})
|
||||
agent_backend["session_snapshot_cleaned_on_failure"] = False
|
||||
agent_backend["session_snapshot_cleanup_error"] = "workflow_agent_runtime_session_store_error"
|
||||
metadata["agent_backend"] = agent_backend
|
||||
|
||||
@staticmethod
|
||||
def _patch_event_with_defaults(
|
||||
event: AgentBackendRunSucceededInternalEvent,
|
||||
|
||||
@@ -4,6 +4,7 @@ from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, Protocol, cast
|
||||
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
from dify_agent.layers.execution_context import DifyExecutionContextLayerConfig
|
||||
from dify_agent.protocol import CreateRunRequest
|
||||
|
||||
@@ -28,6 +29,7 @@ from models.agent_config_entities import (
|
||||
from models.agent_config_entities import (
|
||||
effective_declared_outputs as _effective_declared_outputs,
|
||||
)
|
||||
from models.provider_ids import ModelProviderID
|
||||
|
||||
from .output_failure_orchestrator import retry_idempotency_key
|
||||
from .plugin_tools_builder import WorkflowAgentPluginToolsBuilder, WorkflowAgentPluginToolsBuildError
|
||||
@@ -66,6 +68,7 @@ class WorkflowAgentRuntimeBuildContext:
|
||||
# Stage 4 §7 / D-4: 0 for the first run, then incremented per retry. Drives the
|
||||
# idempotency key so the backend treats each retry as a fresh request.
|
||||
attempt: int = 0
|
||||
session_snapshot: CompositorSessionSnapshot | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -129,11 +132,14 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
request = self._request_builder.build_for_workflow_node(
|
||||
AgentBackendWorkflowNodeRunInput(
|
||||
model=AgentBackendModelConfig(
|
||||
plugin_id=agent_soul.model.plugin_id,
|
||||
model_provider=agent_soul.model.model_provider,
|
||||
plugin_id=self._plugin_daemon_plugin_id(
|
||||
plugin_id=agent_soul.model.plugin_id,
|
||||
model_provider=agent_soul.model.model_provider,
|
||||
),
|
||||
model_provider=self._plugin_daemon_provider_name(agent_soul.model.model_provider),
|
||||
model=agent_soul.model.model,
|
||||
credentials=self._normalize_credentials(credentials),
|
||||
model_settings=cast(dict[str, Any], agent_soul.model.model_settings),
|
||||
model_settings=agent_soul.model.model_settings,
|
||||
),
|
||||
# The execution-context layer is now the only public protocol
|
||||
# carrier for Dify tenant/user/run identifiers. ``user_id`` must
|
||||
@@ -158,6 +164,7 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
user_prompt=user_prompt,
|
||||
output=self._build_output_config(node_job.declared_outputs),
|
||||
tools=tools_layer,
|
||||
session_snapshot=context.session_snapshot,
|
||||
idempotency_key=self._idempotency_key(context),
|
||||
metadata=metadata,
|
||||
)
|
||||
@@ -177,6 +184,20 @@ class WorkflowAgentRuntimeRequestBuilder:
|
||||
return "single_step"
|
||||
return "workflow_run"
|
||||
|
||||
@staticmethod
|
||||
def _plugin_daemon_plugin_id(*, plugin_id: str, model_provider: str) -> str:
|
||||
"""Return the transport plugin id expected by plugin-daemon headers."""
|
||||
if plugin_id.count("/") == 1:
|
||||
return plugin_id
|
||||
if plugin_id:
|
||||
return ModelProviderID(plugin_id).plugin_id
|
||||
return ModelProviderID(model_provider).plugin_id
|
||||
|
||||
@staticmethod
|
||||
def _plugin_daemon_provider_name(model_provider: str) -> str:
|
||||
"""Return the provider name expected by plugin-daemon dispatch payloads."""
|
||||
return ModelProviderID(model_provider).provider_name
|
||||
|
||||
@staticmethod
|
||||
def _idempotency_key(context: WorkflowAgentRuntimeBuildContext) -> str:
|
||||
# Stage 4 §7 / D-4: retries get distinct keys (``...:retry-{attempt}``) so
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import override
|
||||
|
||||
from clients.agent_backend import AgentBackendError, AgentBackendRunClient, AgentBackendRunRequestBuilder
|
||||
from clients.agent_backend.factory import create_agent_backend_run_client
|
||||
from configs import dify_config
|
||||
from core.workflow.system_variables import SystemVariableKey, get_system_text
|
||||
from graphon.graph_engine.layers import GraphEngineLayer
|
||||
from graphon.graph_events import (
|
||||
GraphEngineEvent,
|
||||
GraphRunAbortedEvent,
|
||||
GraphRunFailedEvent,
|
||||
GraphRunPartialSucceededEvent,
|
||||
GraphRunSucceededEvent,
|
||||
)
|
||||
|
||||
from .session_store import StoredWorkflowAgentSession, WorkflowAgentRuntimeSessionStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Upper bound on how long a cleanup-only run is allowed to settle before the
|
||||
# layer gives up and leaves the row ACTIVE so it can be retried later. Cleanup
|
||||
# work is mostly local agent-backend bookkeeping (no LLM inference), so 30s is
|
||||
# generous; a hung backend should never block workflow termination beyond this.
|
||||
_CLEANUP_WAIT_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
class WorkflowAgentSessionCleanupLayer(GraphEngineLayer):
|
||||
"""Retires workflow Agent session snapshots when a workflow reaches a terminal state.
|
||||
|
||||
Implementation notes — there are two failure modes the cleanup path has to
|
||||
avoid simultaneously:
|
||||
|
||||
1. The agenton compositor on the agent-backend side validates the cleanup
|
||||
request's session snapshot against the replayed composition before
|
||||
running any lifecycle hook. If the snapshot's layer names diverge from
|
||||
the composition, the run fails asynchronously with ``run_failed`` — but
|
||||
the initial ``POST /runs`` already returned 202, so the API side has no
|
||||
visibility of the failure unless it waits for terminal status. The
|
||||
``composition_layer_specs`` persistence in A.1–A.4 plus the
|
||||
``_filter_snapshot_to_specs`` shape in ``build_cleanup_request`` keeps
|
||||
the two name lists in sync.
|
||||
|
||||
2. The current agent backend's ``runner.py::_run_agent`` always invokes
|
||||
``run.get_layer("llm")`` and the structured-output / history validators
|
||||
before exiting any slot — there is no ``purpose: "cleanup"`` branch
|
||||
yet. A truly cleanup-only request (no LLM layer) therefore still
|
||||
crashes inside the runner with ``Layer 'llm' is not defined in this
|
||||
compositor run.``. Until the backend grows a cleanup-only purpose,
|
||||
this layer **does not issue an HTTP cleanup run**: it simply retires
|
||||
the local snapshot row so stale state cannot be re-resumed, and lets
|
||||
the agent backend's own retention TTL release the suspended layers.
|
||||
|
||||
The HTTP-cleanup machinery (``build_cleanup_request`` + ``wait_run``) is
|
||||
intentionally still wired into the request builder + integration tests so
|
||||
that when the agent backend supports cleanup runs we can flip the switch
|
||||
here with a one-line change (see ``_HTTP_CLEANUP_SUPPORTED``).
|
||||
"""
|
||||
|
||||
# Flip to True once dify-agent's runner has a ``purpose=cleanup`` branch
|
||||
# that skips the LLM/output/user-prompt invariants. Until then we only
|
||||
# update the local row; the spec list is still persisted so the future
|
||||
# HTTP cleanup path has everything it needs.
|
||||
_HTTP_CLEANUP_SUPPORTED: bool = False
|
||||
|
||||
_TERMINAL_EVENTS = (
|
||||
GraphRunSucceededEvent,
|
||||
GraphRunPartialSucceededEvent,
|
||||
GraphRunFailedEvent,
|
||||
GraphRunAbortedEvent,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
session_store: WorkflowAgentRuntimeSessionStore,
|
||||
request_builder: AgentBackendRunRequestBuilder,
|
||||
agent_backend_client: AgentBackendRunClient | None,
|
||||
cleanup_wait_timeout_seconds: float = _CLEANUP_WAIT_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self._session_store = session_store
|
||||
self._request_builder = request_builder
|
||||
self._agent_backend_client = agent_backend_client
|
||||
self._cleanup_wait_timeout_seconds = cleanup_wait_timeout_seconds
|
||||
|
||||
@override
|
||||
def on_graph_start(self) -> None:
|
||||
return
|
||||
|
||||
@override
|
||||
def on_event(self, event: GraphEngineEvent) -> None:
|
||||
if not isinstance(event, self._TERMINAL_EVENTS):
|
||||
return
|
||||
workflow_run_id = get_system_text(
|
||||
self.graph_runtime_state.variable_pool,
|
||||
SystemVariableKey.WORKFLOW_EXECUTION_ID,
|
||||
)
|
||||
if not workflow_run_id:
|
||||
logger.warning("Skipping workflow Agent session cleanup: workflow_run_id is missing.")
|
||||
return
|
||||
|
||||
for stored_session in self._session_store.list_active_sessions(workflow_run_id=workflow_run_id):
|
||||
self._cleanup_session(stored_session)
|
||||
|
||||
@override
|
||||
def on_graph_end(self, error: Exception | None) -> None:
|
||||
return
|
||||
|
||||
def _cleanup_session(self, stored_session: StoredWorkflowAgentSession) -> None:
|
||||
scope = stored_session.scope
|
||||
if not self._HTTP_CLEANUP_SUPPORTED:
|
||||
# Agent backend has no cleanup-only run mode yet (see class
|
||||
# docstring). Retire the local row so future re-entries do not
|
||||
# resume from stale state, and let the backend's retention TTL
|
||||
# release the suspended layers on its own schedule.
|
||||
logger.info(
|
||||
"Workflow Agent session retired locally; HTTP cleanup is disabled "
|
||||
"until the agent backend supports a cleanup-only run mode. "
|
||||
"workflow_run_id=%s node_id=%s binding_id=%s agent_id=%s previous_run_id=%s",
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.binding_id,
|
||||
scope.agent_id,
|
||||
stored_session.backend_run_id,
|
||||
)
|
||||
self._session_store.mark_cleaned(scope=scope, backend_run_id=stored_session.backend_run_id)
|
||||
return
|
||||
|
||||
if self._agent_backend_client is None:
|
||||
# HTTP cleanup was enabled by the caller but no client was wired
|
||||
# in (e.g. the API runs without AGENT_BACKEND_BASE_URL configured).
|
||||
# Leave the row ACTIVE so an operator restart with proper config
|
||||
# can drive the cleanup; do not silently retire it.
|
||||
logger.warning(
|
||||
"Skipping Agent backend cleanup: HTTP cleanup is enabled but no agent "
|
||||
"backend client is wired in. workflow_run_id=%s node_id=%s agent_id=%s",
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.agent_id,
|
||||
)
|
||||
return
|
||||
|
||||
if not stored_session.composition_layer_specs:
|
||||
# Sessions persisted before A.1 landed do not carry the spec list,
|
||||
# so we cannot replay a valid cleanup composition. Leave the row
|
||||
# ACTIVE and warn so the absence shows up in observability rather
|
||||
# than being silently swallowed by a doomed cleanup run.
|
||||
logger.warning(
|
||||
"Skipping Agent backend cleanup: no composition_layer_specs persisted. "
|
||||
"workflow_run_id=%s node_id=%s agent_id=%s",
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.agent_id,
|
||||
)
|
||||
return
|
||||
|
||||
request = self._request_builder.build_cleanup_request(
|
||||
session_snapshot=stored_session.session_snapshot,
|
||||
composition_layer_specs=stored_session.composition_layer_specs,
|
||||
idempotency_key=f"{scope.workflow_run_id}:{scope.node_id}:{scope.binding_id}:agent-session-cleanup",
|
||||
metadata={
|
||||
"tenant_id": scope.tenant_id,
|
||||
"app_id": scope.app_id,
|
||||
"workflow_id": scope.workflow_id,
|
||||
"workflow_run_id": scope.workflow_run_id,
|
||||
"node_id": scope.node_id,
|
||||
"node_execution_id": scope.node_execution_id,
|
||||
"binding_id": scope.binding_id,
|
||||
"agent_id": scope.agent_id,
|
||||
"agent_config_snapshot_id": scope.agent_config_snapshot_id,
|
||||
"previous_agent_backend_run_id": stored_session.backend_run_id,
|
||||
},
|
||||
)
|
||||
try:
|
||||
response = self._agent_backend_client.create_run(request)
|
||||
except AgentBackendError:
|
||||
logger.warning(
|
||||
"Agent backend session cleanup request failed: workflow_run_id=%s node_id=%s agent_id=%s",
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.agent_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
status_response = self._agent_backend_client.wait_run(
|
||||
response.run_id, timeout_seconds=self._cleanup_wait_timeout_seconds
|
||||
)
|
||||
except AgentBackendError:
|
||||
logger.warning(
|
||||
"Agent backend session cleanup wait_run failed: "
|
||||
"workflow_run_id=%s node_id=%s agent_id=%s cleanup_run_id=%s",
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.agent_id,
|
||||
response.run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
if status_response.status != "succeeded":
|
||||
logger.warning(
|
||||
"Agent backend session cleanup did not succeed: status=%s error=%s "
|
||||
"workflow_run_id=%s node_id=%s agent_id=%s cleanup_run_id=%s",
|
||||
status_response.status,
|
||||
status_response.error,
|
||||
scope.workflow_run_id,
|
||||
scope.node_id,
|
||||
scope.agent_id,
|
||||
response.run_id,
|
||||
)
|
||||
return
|
||||
|
||||
self._session_store.mark_cleaned(scope=scope, backend_run_id=response.run_id)
|
||||
|
||||
|
||||
def build_workflow_agent_session_cleanup_layer() -> WorkflowAgentSessionCleanupLayer:
|
||||
"""Wire the cleanup layer with the standard production dependencies.
|
||||
|
||||
The agent backend client is constructed only when ``AGENT_BACKEND_BASE_URL``
|
||||
is configured (or the deterministic fake is explicitly enabled). When
|
||||
neither is set — for example unit tests that bring up the workflow runner
|
||||
without an Agent node — we pass ``None`` so the layer stays harmless. With
|
||||
``_HTTP_CLEANUP_SUPPORTED = False`` the local-retire branch never touches
|
||||
the client anyway, but keeping it ``None`` avoids importing httpx and lets
|
||||
test harnesses skip backend configuration.
|
||||
"""
|
||||
agent_backend_client: AgentBackendRunClient | None
|
||||
if dify_config.AGENT_BACKEND_USE_FAKE or dify_config.AGENT_BACKEND_BASE_URL:
|
||||
agent_backend_client = create_agent_backend_run_client(
|
||||
base_url=dify_config.AGENT_BACKEND_BASE_URL,
|
||||
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
|
||||
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
|
||||
)
|
||||
else:
|
||||
agent_backend_client = None
|
||||
|
||||
return WorkflowAgentSessionCleanupLayer(
|
||||
session_store=WorkflowAgentRuntimeSessionStore(),
|
||||
request_builder=AgentBackendRunRequestBuilder(),
|
||||
agent_backend_client=agent_backend_client,
|
||||
)
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
from pydantic import TypeAdapter
|
||||
from sqlalchemy import select
|
||||
|
||||
from clients.agent_backend.request_builder import CleanupLayerSpec
|
||||
from core.db.session_factory import session_factory
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models.agent import (
|
||||
WorkflowAgentRuntimeSession,
|
||||
WorkflowAgentRuntimeSessionStatus,
|
||||
)
|
||||
|
||||
_SPECS_ADAPTER: TypeAdapter[list[CleanupLayerSpec]] = TypeAdapter(list[CleanupLayerSpec])
|
||||
|
||||
|
||||
def _serialize_specs(specs: list[CleanupLayerSpec]) -> str:
|
||||
return _SPECS_ADAPTER.dump_json(specs).decode()
|
||||
|
||||
|
||||
def _deserialize_specs(value: str | None) -> list[CleanupLayerSpec]:
|
||||
if not value:
|
||||
return []
|
||||
return _SPECS_ADAPTER.validate_json(value)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowAgentSessionScope:
|
||||
tenant_id: str
|
||||
app_id: str
|
||||
workflow_id: str
|
||||
workflow_run_id: str | None
|
||||
node_id: str
|
||||
node_execution_id: str
|
||||
binding_id: str
|
||||
agent_id: str
|
||||
agent_config_snapshot_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StoredWorkflowAgentSession:
|
||||
scope: WorkflowAgentSessionScope
|
||||
session_snapshot: CompositorSessionSnapshot
|
||||
backend_run_id: str | None
|
||||
composition_layer_specs: list[CleanupLayerSpec] = field(default_factory=list)
|
||||
|
||||
|
||||
class WorkflowAgentRuntimeSessionStore:
|
||||
"""Stores Agent backend session snapshots for workflow Agent node re-entry."""
|
||||
|
||||
def load_active_snapshot(self, scope: WorkflowAgentSessionScope) -> CompositorSessionSnapshot | None:
|
||||
if scope.workflow_run_id is None:
|
||||
return None
|
||||
|
||||
with session_factory.create_session() as session:
|
||||
row = session.scalar(
|
||||
select(WorkflowAgentRuntimeSession).where(
|
||||
WorkflowAgentRuntimeSession.tenant_id == scope.tenant_id,
|
||||
WorkflowAgentRuntimeSession.workflow_run_id == scope.workflow_run_id,
|
||||
WorkflowAgentRuntimeSession.node_id == scope.node_id,
|
||||
WorkflowAgentRuntimeSession.binding_id == scope.binding_id,
|
||||
WorkflowAgentRuntimeSession.agent_id == scope.agent_id,
|
||||
WorkflowAgentRuntimeSession.status == WorkflowAgentRuntimeSessionStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
return CompositorSessionSnapshot.model_validate_json(row.session_snapshot)
|
||||
|
||||
def list_active_sessions(self, *, workflow_run_id: str) -> list[StoredWorkflowAgentSession]:
|
||||
with session_factory.create_session() as session:
|
||||
rows = session.scalars(
|
||||
select(WorkflowAgentRuntimeSession).where(
|
||||
WorkflowAgentRuntimeSession.workflow_run_id == workflow_run_id,
|
||||
WorkflowAgentRuntimeSession.status == WorkflowAgentRuntimeSessionStatus.ACTIVE,
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
StoredWorkflowAgentSession(
|
||||
scope=WorkflowAgentSessionScope(
|
||||
tenant_id=row.tenant_id,
|
||||
app_id=row.app_id,
|
||||
workflow_id=row.workflow_id,
|
||||
workflow_run_id=row.workflow_run_id,
|
||||
node_id=row.node_id,
|
||||
node_execution_id=row.node_execution_id or "",
|
||||
binding_id=row.binding_id,
|
||||
agent_id=row.agent_id,
|
||||
agent_config_snapshot_id=row.agent_config_snapshot_id,
|
||||
),
|
||||
session_snapshot=CompositorSessionSnapshot.model_validate_json(row.session_snapshot),
|
||||
backend_run_id=row.backend_run_id,
|
||||
composition_layer_specs=_deserialize_specs(row.composition_layer_specs),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def save_active_snapshot(
|
||||
self,
|
||||
*,
|
||||
scope: WorkflowAgentSessionScope,
|
||||
backend_run_id: str,
|
||||
snapshot: CompositorSessionSnapshot | None,
|
||||
composition_layer_specs: list[CleanupLayerSpec],
|
||||
) -> None:
|
||||
if scope.workflow_run_id is None or snapshot is None:
|
||||
return
|
||||
|
||||
snapshot_json = snapshot.model_dump_json()
|
||||
specs_json = _serialize_specs(composition_layer_specs)
|
||||
with session_factory.create_session() as session:
|
||||
row = session.scalar(
|
||||
select(WorkflowAgentRuntimeSession).where(
|
||||
WorkflowAgentRuntimeSession.tenant_id == scope.tenant_id,
|
||||
WorkflowAgentRuntimeSession.workflow_run_id == scope.workflow_run_id,
|
||||
WorkflowAgentRuntimeSession.node_id == scope.node_id,
|
||||
WorkflowAgentRuntimeSession.binding_id == scope.binding_id,
|
||||
WorkflowAgentRuntimeSession.agent_id == scope.agent_id,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
row = WorkflowAgentRuntimeSession(
|
||||
tenant_id=scope.tenant_id,
|
||||
app_id=scope.app_id,
|
||||
workflow_id=scope.workflow_id,
|
||||
workflow_run_id=scope.workflow_run_id,
|
||||
node_id=scope.node_id,
|
||||
node_execution_id=scope.node_execution_id,
|
||||
binding_id=scope.binding_id,
|
||||
agent_id=scope.agent_id,
|
||||
agent_config_snapshot_id=scope.agent_config_snapshot_id,
|
||||
backend_run_id=backend_run_id,
|
||||
session_snapshot=snapshot_json,
|
||||
composition_layer_specs=specs_json,
|
||||
status=WorkflowAgentRuntimeSessionStatus.ACTIVE,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.node_execution_id = scope.node_execution_id
|
||||
row.agent_config_snapshot_id = scope.agent_config_snapshot_id
|
||||
row.backend_run_id = backend_run_id
|
||||
row.session_snapshot = snapshot_json
|
||||
row.composition_layer_specs = specs_json
|
||||
row.status = WorkflowAgentRuntimeSessionStatus.ACTIVE
|
||||
row.cleaned_at = None
|
||||
session.commit()
|
||||
|
||||
def mark_cleaned(self, *, scope: WorkflowAgentSessionScope, backend_run_id: str | None = None) -> None:
|
||||
if scope.workflow_run_id is None:
|
||||
return
|
||||
|
||||
with session_factory.create_session() as session:
|
||||
row = session.scalar(
|
||||
select(WorkflowAgentRuntimeSession).where(
|
||||
WorkflowAgentRuntimeSession.tenant_id == scope.tenant_id,
|
||||
WorkflowAgentRuntimeSession.workflow_run_id == scope.workflow_run_id,
|
||||
WorkflowAgentRuntimeSession.node_id == scope.node_id,
|
||||
WorkflowAgentRuntimeSession.binding_id == scope.binding_id,
|
||||
WorkflowAgentRuntimeSession.agent_id == scope.agent_id,
|
||||
WorkflowAgentRuntimeSession.status == WorkflowAgentRuntimeSessionStatus.ACTIVE,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
return
|
||||
if backend_run_id is not None:
|
||||
row.backend_run_id = backend_run_id
|
||||
row.status = WorkflowAgentRuntimeSessionStatus.CLEANED
|
||||
row.cleaned_at = naive_utc_now()
|
||||
session.commit()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"StoredWorkflowAgentSession",
|
||||
"WorkflowAgentRuntimeSessionStore",
|
||||
"WorkflowAgentSessionScope",
|
||||
]
|
||||
@@ -1,6 +1,7 @@
|
||||
import logging
|
||||
|
||||
from core.tools.entities.tool_entities import ToolProviderType
|
||||
from core.tools.errors import ToolProviderNotFoundError
|
||||
from core.tools.tool_manager import ToolManager
|
||||
from core.tools.utils.configuration import ToolParameterConfigurationManager
|
||||
from core.workflow.human_input_adapter import adapt_node_config_for_graph
|
||||
@@ -38,6 +39,14 @@ def handle(sender, **kwargs):
|
||||
identity_id=f"WORKFLOW.{app.id}.{node_data.get('id')}",
|
||||
)
|
||||
manager.delete_tool_parameters_cache()
|
||||
except ToolProviderNotFoundError as exc:
|
||||
logger.info(
|
||||
"Skipped deleting tool parameters cache for workflow %s node %s "
|
||||
"because tool provider is missing: %s",
|
||||
app.id,
|
||||
node_data.get("id"),
|
||||
exc,
|
||||
)
|
||||
except Exception:
|
||||
# tool dose not exist
|
||||
logger.exception(
|
||||
|
||||
@@ -15,14 +15,18 @@ def init_app(app: DifyApp):
|
||||
data_migrate,
|
||||
delete_archived_workflow_runs,
|
||||
export_app_messages,
|
||||
export_migration_data,
|
||||
export_migration_data_template,
|
||||
extract_plugins,
|
||||
extract_unique_plugins,
|
||||
file_usage,
|
||||
fix_app_site_missing,
|
||||
import_migration_data,
|
||||
install_plugins,
|
||||
install_rag_pipeline_plugins,
|
||||
migrate_data_for_plugin,
|
||||
migrate_oss,
|
||||
migration_data_wizard,
|
||||
old_metadata_migration,
|
||||
remove_orphaned_files_on_storage,
|
||||
reset_email,
|
||||
@@ -70,6 +74,10 @@ def init_app(app: DifyApp):
|
||||
clean_workflow_runs,
|
||||
clean_expired_messages,
|
||||
export_app_messages,
|
||||
export_migration_data,
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
migration_data_wizard,
|
||||
]
|
||||
for cmd in cmds_to_register:
|
||||
app.cli.add_command(cmd)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import posixpath
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
import oss2 as aliyun_s3
|
||||
|
||||
@@ -29,9 +30,11 @@ class AliyunOssStorage(BaseStorage):
|
||||
cloudbox_id=dify_config.ALIYUN_CLOUDBOX_ID,
|
||||
)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.put_object(self.__wrapper_folder_filename(filename), data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
obj = self.client.get_object(self.__wrapper_folder_filename(filename))
|
||||
data = obj.read()
|
||||
@@ -39,17 +42,21 @@ class AliyunOssStorage(BaseStorage):
|
||||
return b""
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
obj = self.client.get_object(self.__wrapper_folder_filename(filename))
|
||||
while chunk := obj.read(4096):
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename: str, target_filepath):
|
||||
self.client.get_object_to_file(self.__wrapper_folder_filename(filename), target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename: str):
|
||||
return self.client.object_exists(self.__wrapper_folder_filename(filename))
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(self.__wrapper_folder_filename(filename))
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
@@ -48,9 +49,11 @@ class AwsS3Storage(BaseStorage):
|
||||
# other error, raise exception
|
||||
raise
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
try:
|
||||
data: bytes = self.client.get_object(Bucket=self.bucket_name, Key=filename)["Body"].read()
|
||||
@@ -61,6 +64,7 @@ class AwsS3Storage(BaseStorage):
|
||||
raise
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket_name, Key=filename)
|
||||
@@ -73,9 +77,11 @@ class AwsS3Storage(BaseStorage):
|
||||
else:
|
||||
raise
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
self.client.download_file(self.bucket_name, filename, target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket_name, Key=filename)
|
||||
@@ -83,5 +89,6 @@ class AwsS3Storage(BaseStorage):
|
||||
except:
|
||||
return False
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections.abc import Generator
|
||||
from datetime import timedelta
|
||||
from typing import override
|
||||
|
||||
from azure.identity import ChainedTokenCredential, DefaultAzureCredential
|
||||
from azure.storage.blob import AccountSasPermissions, BlobServiceClient, ResourceTypes, generate_account_sas
|
||||
@@ -26,6 +27,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
else:
|
||||
self.credential = None
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
if not self.bucket_name:
|
||||
return
|
||||
@@ -34,6 +36,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
blob_container = client.get_container_client(container=self.bucket_name)
|
||||
blob_container.upload_blob(filename, data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
if not self.bucket_name:
|
||||
raise FileNotFoundError("Azure bucket name is not configured.")
|
||||
@@ -46,6 +49,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
raise TypeError(f"Expected bytes from blob.readall(), got {type(data).__name__}")
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
if not self.bucket_name:
|
||||
raise FileNotFoundError("Azure bucket name is not configured.")
|
||||
@@ -55,6 +59,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
blob_data = blob.download_blob()
|
||||
yield from blob_data.chunks()
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
if not self.bucket_name:
|
||||
return
|
||||
@@ -66,6 +71,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
blob_data = blob.download_blob()
|
||||
blob_data.readinto(my_blob)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
if not self.bucket_name:
|
||||
return False
|
||||
@@ -75,6 +81,7 @@ class AzureBlobStorage(BaseStorage):
|
||||
blob = client.get_blob_client(container=self.bucket_name, blob=filename)
|
||||
return blob.exists()
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
if not self.bucket_name:
|
||||
return
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import base64
|
||||
import hashlib
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
from baidubce.auth.bce_credentials import BceCredentials
|
||||
from baidubce.bce_client_configuration import BceClientConfiguration
|
||||
@@ -26,6 +27,7 @@ class BaiduObsStorage(BaseStorage):
|
||||
|
||||
self.client = BosClient(config=client_config)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
md5 = hashlib.md5()
|
||||
md5.update(data)
|
||||
@@ -34,24 +36,29 @@ class BaiduObsStorage(BaseStorage):
|
||||
bucket_name=self.bucket_name, key=filename, data=data, content_length=len(data), content_md5=content_md5
|
||||
)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
response = self.client.get_object(bucket_name=self.bucket_name, key=filename)
|
||||
data: bytes = response.data.read()
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
response = self.client.get_object(bucket_name=self.bucket_name, key=filename).data
|
||||
while chunk := response.read(4096):
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
self.client.get_object_to_file(bucket_name=self.bucket_name, key=filename, file_name=target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
res = self.client.get_object_meta_data(bucket_name=self.bucket_name, key=filename)
|
||||
if res is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(bucket_name=self.bucket_name, key=filename)
|
||||
|
||||
@@ -10,7 +10,7 @@ import tempfile
|
||||
from collections.abc import Generator
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, override
|
||||
|
||||
import clickzetta
|
||||
from pydantic import BaseModel, model_validator
|
||||
@@ -251,6 +251,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
# Don't raise exception, let the operation continue
|
||||
# The table might exist but not be visible due to permissions
|
||||
|
||||
@override
|
||||
def save(self, filename: str, data: bytes):
|
||||
"""Save data to ClickZetta Volume.
|
||||
|
||||
@@ -304,6 +305,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
# Clean up temporary file
|
||||
Path(temp_file_path).unlink(missing_ok=True)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
"""Load file content from ClickZetta Volume.
|
||||
|
||||
@@ -364,6 +366,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
logger.debug("File %s loaded from ClickZetta Volume", filename)
|
||||
return content
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
"""Load file as stream from ClickZetta Volume.
|
||||
|
||||
@@ -382,6 +385,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
|
||||
logger.debug("File %s loaded as stream from ClickZetta Volume", filename)
|
||||
|
||||
@override
|
||||
def download(self, filename: str, target_filepath: str):
|
||||
"""Download file from ClickZetta Volume to local path.
|
||||
|
||||
@@ -395,6 +399,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
|
||||
logger.debug("File %s downloaded from ClickZetta Volume to %s", filename, target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename: str) -> bool:
|
||||
"""Check if file exists in ClickZetta Volume.
|
||||
|
||||
@@ -436,6 +441,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
logger.warning("Error checking file existence for %s: %s", filename, e)
|
||||
return False
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
"""Delete file from ClickZetta Volume.
|
||||
|
||||
@@ -472,6 +478,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||
|
||||
logger.debug("File %s deleted from ClickZetta Volume", filename)
|
||||
|
||||
@override
|
||||
def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
|
||||
"""Scan files and directories in ClickZetta Volume.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import base64
|
||||
import io
|
||||
from collections.abc import Generator
|
||||
from typing import Any
|
||||
from typing import Any, override
|
||||
|
||||
from google.cloud import storage as google_cloud_storage # type: ignore
|
||||
from pydantic import TypeAdapter
|
||||
@@ -29,12 +29,14 @@ class GoogleCloudStorage(BaseStorage):
|
||||
else:
|
||||
self.client = google_cloud_storage.Client()
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(filename)
|
||||
with io.BytesIO(data) as stream:
|
||||
blob.upload_from_file(stream)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
blob = bucket.get_blob(filename)
|
||||
@@ -43,6 +45,7 @@ class GoogleCloudStorage(BaseStorage):
|
||||
data: bytes = blob.download_as_bytes()
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
blob = bucket.get_blob(filename)
|
||||
@@ -52,6 +55,7 @@ class GoogleCloudStorage(BaseStorage):
|
||||
while chunk := blob_stream.read(4096):
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
blob = bucket.get_blob(filename)
|
||||
@@ -59,11 +63,13 @@ class GoogleCloudStorage(BaseStorage):
|
||||
raise FileNotFoundError("File not found")
|
||||
blob.download_to_filename(target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(filename)
|
||||
return blob.exists()
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
bucket = self.client.get_bucket(self.bucket_name)
|
||||
bucket.delete_blob(filename)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
from obs import ObsClient
|
||||
|
||||
@@ -20,27 +21,33 @@ class HuaweiObsStorage(BaseStorage):
|
||||
path_style=dify_config.HUAWEI_OBS_PATH_STYLE,
|
||||
)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.putObject(bucketName=self.bucket_name, objectKey=filename, content=data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
data: bytes = self.client.getObject(bucketName=self.bucket_name, objectKey=filename)["body"].response.read()
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
response = self.client.getObject(bucketName=self.bucket_name, objectKey=filename)["body"].response
|
||||
while chunk := response.read(4096):
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
self.client.getObject(bucketName=self.bucket_name, objectKey=filename, downloadPath=target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
res = self._get_meta(filename)
|
||||
if res is None:
|
||||
return False
|
||||
return True
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.deleteObject(bucketName=self.bucket_name, objectKey=filename)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
import os
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, override
|
||||
|
||||
import opendal
|
||||
from dotenv import dotenv_values
|
||||
@@ -41,10 +41,12 @@ class OpenDALStorage(BaseStorage):
|
||||
logger.debug("opendal operator created with scheme %s", scheme)
|
||||
logger.debug("added retry layer to opendal operator")
|
||||
|
||||
@override
|
||||
def save(self, filename: str, data: bytes):
|
||||
self.op.write(path=filename, bs=data)
|
||||
logger.debug("file %s saved", filename)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
if not self.exists(filename):
|
||||
raise FileNotFoundError("File not found")
|
||||
@@ -53,6 +55,7 @@ class OpenDALStorage(BaseStorage):
|
||||
logger.debug("file %s loaded", filename)
|
||||
return content
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
if not self.exists(filename):
|
||||
raise FileNotFoundError("File not found")
|
||||
@@ -67,6 +70,7 @@ class OpenDALStorage(BaseStorage):
|
||||
yield chunk
|
||||
logger.debug("file %s loaded as stream", filename)
|
||||
|
||||
@override
|
||||
def download(self, filename: str, target_filepath: str):
|
||||
if not self.exists(filename):
|
||||
raise FileNotFoundError("File not found")
|
||||
@@ -74,9 +78,11 @@ class OpenDALStorage(BaseStorage):
|
||||
Path(target_filepath).write_bytes(self.op.read(path=filename))
|
||||
logger.debug("file %s downloaded to %s", filename, target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename: str) -> bool:
|
||||
return self.op.exists(path=filename)
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
if self.exists(filename):
|
||||
self.op.delete(path=filename)
|
||||
@@ -84,6 +90,7 @@ class OpenDALStorage(BaseStorage):
|
||||
return
|
||||
logger.debug("file %s not found, skip delete", filename)
|
||||
|
||||
@override
|
||||
def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
|
||||
if not self.exists(path):
|
||||
raise FileNotFoundError("Path not found")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError
|
||||
@@ -22,9 +23,11 @@ class OracleOCIStorage(BaseStorage):
|
||||
region_name=dify_config.OCI_REGION,
|
||||
)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.put_object(Bucket=self.bucket_name, Key=filename, Body=data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
try:
|
||||
data: bytes = self.client.get_object(Bucket=self.bucket_name, Key=filename)["Body"].read()
|
||||
@@ -35,6 +38,7 @@ class OracleOCIStorage(BaseStorage):
|
||||
raise
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=self.bucket_name, Key=filename)
|
||||
@@ -45,9 +49,11 @@ class OracleOCIStorage(BaseStorage):
|
||||
else:
|
||||
raise
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
self.client.download_file(self.bucket_name, filename, target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket_name, Key=filename)
|
||||
@@ -55,5 +61,6 @@ class OracleOCIStorage(BaseStorage):
|
||||
except:
|
||||
return False
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import io
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
from typing import override
|
||||
|
||||
from supabase import Client
|
||||
|
||||
@@ -28,29 +29,35 @@ class SupabaseStorage(BaseStorage):
|
||||
if not self.bucket_exists():
|
||||
self.client.storage.create_bucket(id=id, name=bucket_name)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.storage.from_(self.bucket_name).upload(filename, data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
content: bytes = self.client.storage.from_(self.bucket_name).download(filename)
|
||||
return content
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
result = self.client.storage.from_(self.bucket_name).download(filename)
|
||||
byte_stream = io.BytesIO(result)
|
||||
while chunk := byte_stream.read(4096): # Read in chunks of 4KB
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
result = self.client.storage.from_(self.bucket_name).download(filename)
|
||||
Path(target_filepath).write_bytes(result)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
result = self.client.storage.from_(self.bucket_name).list(path=filename)
|
||||
if len(result) > 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.storage.from_(self.bucket_name).remove([filename])
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
@@ -29,23 +30,29 @@ class TencentCosStorage(BaseStorage):
|
||||
)
|
||||
self.client = CosS3Client(config)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
self.client.put_object(Bucket=self.bucket_name, Body=data, Key=filename)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
data: bytes = self.client.get_object(Bucket=self.bucket_name, Key=filename)["Body"].get_raw_stream().read()
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
response = self.client.get_object(Bucket=self.bucket_name, Key=filename)
|
||||
yield from response["Body"].get_stream(chunk_size=4096)
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
response = self.client.get_object(Bucket=self.bucket_name, Key=filename)
|
||||
response["Body"].get_stream_to_file(target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
return self.client.object_exists(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from collections.abc import Generator
|
||||
from typing import override
|
||||
|
||||
import tos
|
||||
|
||||
@@ -27,11 +28,13 @@ class VolcengineTosStorage(BaseStorage):
|
||||
region=dify_config.VOLCENGINE_TOS_REGION,
|
||||
)
|
||||
|
||||
@override
|
||||
def save(self, filename, data):
|
||||
if not self.bucket_name:
|
||||
raise ValueError("VOLCENGINE_TOS_BUCKET_NAME is not set")
|
||||
self.client.put_object(bucket=self.bucket_name, key=filename, content=data)
|
||||
|
||||
@override
|
||||
def load_once(self, filename: str) -> bytes:
|
||||
if not self.bucket_name:
|
||||
raise FileNotFoundError("VOLCENGINE_TOS_BUCKET_NAME is not set")
|
||||
@@ -40,6 +43,7 @@ class VolcengineTosStorage(BaseStorage):
|
||||
raise TypeError(f"Expected bytes, got {type(data).__name__}")
|
||||
return data
|
||||
|
||||
@override
|
||||
def load_stream(self, filename: str) -> Generator:
|
||||
if not self.bucket_name:
|
||||
raise FileNotFoundError("VOLCENGINE_TOS_BUCKET_NAME is not set")
|
||||
@@ -47,11 +51,13 @@ class VolcengineTosStorage(BaseStorage):
|
||||
while chunk := response.read(4096):
|
||||
yield chunk
|
||||
|
||||
@override
|
||||
def download(self, filename, target_filepath):
|
||||
if not self.bucket_name:
|
||||
raise ValueError("VOLCENGINE_TOS_BUCKET_NAME is not set")
|
||||
self.client.get_object_to_file(bucket=self.bucket_name, key=filename, file_path=target_filepath)
|
||||
|
||||
@override
|
||||
def exists(self, filename):
|
||||
if not self.bucket_name:
|
||||
return False
|
||||
@@ -60,6 +66,7 @@ class VolcengineTosStorage(BaseStorage):
|
||||
return False
|
||||
return True
|
||||
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
if not self.bucket_name:
|
||||
return
|
||||
|
||||
@@ -43,6 +43,11 @@ class SubjectType(StrEnum):
|
||||
EXTERNAL_SSO = "external_sso"
|
||||
|
||||
|
||||
class TokenType(StrEnum):
|
||||
OAUTH_ACCOUNT = "oauth_account"
|
||||
OAUTH_EXTERNAL_SSO = "oauth_external_sso"
|
||||
|
||||
|
||||
class Scope(StrEnum):
|
||||
"""Catalog of bearer scopes recognised by the openapi surface.
|
||||
|
||||
@@ -55,6 +60,8 @@ class Scope(StrEnum):
|
||||
APPS_READ = "apps:read"
|
||||
APPS_READ_PERMITTED_EXTERNAL = "apps:read:permitted-external"
|
||||
APPS_RUN = "apps:run"
|
||||
WORKSPACE_READ = "workspace:read"
|
||||
WORKSPACE_WRITE = "workspace:write"
|
||||
|
||||
|
||||
class Accepts(StrEnum):
|
||||
@@ -77,7 +84,7 @@ _SUBJECT_TO_ACCEPT: dict[SubjectType, Accepts] = {
|
||||
class AuthContext:
|
||||
"""Per-request identity published via :data:`_auth_ctx_var`
|
||||
(see :func:`set_auth_ctx` / :func:`get_auth_ctx`). ``scopes`` /
|
||||
``subject_type`` / ``source`` come from the TokenKind, not the DB —
|
||||
``subject_type`` / ``token_type`` come from the TokenKind, not the DB —
|
||||
corrupt rows can't elevate scope.
|
||||
|
||||
`verified_tenants` is a snapshot of the Layer-0 verdict cache at
|
||||
@@ -92,7 +99,7 @@ class AuthContext:
|
||||
client_id: str | None
|
||||
scopes: frozenset[Scope]
|
||||
token_id: uuid.UUID
|
||||
source: str
|
||||
token_type: TokenType
|
||||
expires_at: datetime | None
|
||||
token_hash: str
|
||||
verified_tenants: dict[str, bool] = field(default_factory=dict)
|
||||
@@ -180,7 +187,7 @@ class TokenKind:
|
||||
prefix: str
|
||||
subject_type: SubjectType
|
||||
scopes: frozenset[Scope]
|
||||
source: str
|
||||
token_type: TokenType
|
||||
resolver: Resolver
|
||||
|
||||
def matches(self, token: str) -> bool:
|
||||
@@ -291,7 +298,7 @@ class BearerAuthenticator:
|
||||
client_id=row.client_id,
|
||||
scopes=kind.scopes,
|
||||
token_id=row.token_id,
|
||||
source=kind.source,
|
||||
token_type=kind.token_type,
|
||||
expires_at=row.expires_at,
|
||||
token_hash=token_hash,
|
||||
verified_tenants=dict(row.verified_tenants),
|
||||
@@ -483,7 +490,7 @@ def check_workspace_membership(
|
||||
account_id: uuid.UUID | str,
|
||||
tenant_id: str,
|
||||
token_hash: str,
|
||||
cached_verdicts: dict[str, bool],
|
||||
membership_cache: dict[str, bool],
|
||||
) -> None:
|
||||
"""Layer-0 enforcement core. Raises `Forbidden` on deny, returns on allow.
|
||||
|
||||
@@ -492,7 +499,7 @@ def check_workspace_membership(
|
||||
short-circuiting on EE / SSO subjects before invoking — this function
|
||||
runs the membership + active-status checks unconditionally.
|
||||
"""
|
||||
cached = cached_verdicts.get(tenant_id)
|
||||
cached = membership_cache.get(tenant_id)
|
||||
if cached is True:
|
||||
return
|
||||
if cached is False:
|
||||
@@ -530,7 +537,7 @@ def require_workspace_member(ctx: AuthContext, tenant_id: str) -> None:
|
||||
account_id=ctx.account_id,
|
||||
tenant_id=tenant_id,
|
||||
token_hash=ctx.token_hash,
|
||||
cached_verdicts=ctx.verified_tenants,
|
||||
membership_cache=ctx.verified_tenants,
|
||||
)
|
||||
|
||||
|
||||
@@ -664,14 +671,14 @@ def build_registry(session_factory, redis_client) -> TokenKindRegistry:
|
||||
prefix=account.prefix,
|
||||
subject_type=account.subject_type,
|
||||
scopes=account.scopes,
|
||||
source="oauth_account",
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
resolver=oauth.for_account(),
|
||||
),
|
||||
TokenKind(
|
||||
prefix=external.prefix,
|
||||
subject_type=external.subject_type,
|
||||
scopes=external.scopes,
|
||||
source="oauth_external_sso",
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
resolver=oauth.for_external_sso(),
|
||||
),
|
||||
]
|
||||
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
"""add workflow agent runtime sessions
|
||||
|
||||
Revision ID: 7885bd53f9a9
|
||||
Revises: d4a5e1f3c9b7
|
||||
Create Date: 2026-05-27 09:53:54.711805
|
||||
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
import models as models
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "7885bd53f9a9"
|
||||
down_revision = "d4a5e1f3c9b7"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _is_pg() -> bool:
|
||||
return op.get_bind().dialect.name == "postgresql"
|
||||
|
||||
|
||||
def _uuid_column(name: str, *, nullable: bool = False, primary_key: bool = False) -> sa.Column:
|
||||
"""Match the ``uuidv7()`` default that other tables on Postgres rely on,
|
||||
while staying portable on MySQL where the ORM supplies the id."""
|
||||
kwargs: dict[str, object] = {"nullable": nullable, "primary_key": primary_key}
|
||||
if primary_key and _is_pg():
|
||||
kwargs["server_default"] = sa.text("uuidv7()")
|
||||
return sa.Column(name, models.types.StringUUID(), **kwargs)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"workflow_agent_runtime_sessions",
|
||||
_uuid_column("id", primary_key=True),
|
||||
sa.Column("tenant_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("app_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("workflow_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("workflow_run_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=255), nullable=False),
|
||||
sa.Column("node_execution_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("binding_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("agent_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("agent_config_snapshot_id", models.types.StringUUID(), nullable=False),
|
||||
sa.Column("backend_run_id", sa.String(length=255), nullable=True),
|
||||
sa.Column("session_snapshot", models.types.LongText(), nullable=False),
|
||||
# MySQL rejects ``server_default`` on TEXT/BLOB columns. The JSON
|
||||
# payload is always populated at the ORM layer via
|
||||
# ``WorkflowAgentRuntimeSessionStore.save_active_snapshot`` so the
|
||||
# missing DB-level default cannot leave new rows uninitialized.
|
||||
sa.Column("composition_layer_specs", models.types.LongText(), nullable=False),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.String(length=32),
|
||||
server_default=sa.text("'active'"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("cleaned_at", sa.DateTime(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("workflow_agent_runtime_session_pkey")),
|
||||
sa.UniqueConstraint(
|
||||
"tenant_id",
|
||||
"workflow_run_id",
|
||||
"node_id",
|
||||
"binding_id",
|
||||
"agent_id",
|
||||
name=op.f("workflow_agent_runtime_session_scope_unique"),
|
||||
),
|
||||
)
|
||||
with op.batch_alter_table("workflow_agent_runtime_sessions", schema=None) as batch_op:
|
||||
batch_op.create_index(
|
||||
"workflow_agent_runtime_session_lookup_idx",
|
||||
["tenant_id", "workflow_run_id", "node_id", "status"],
|
||||
unique=False,
|
||||
)
|
||||
batch_op.create_index(
|
||||
"workflow_agent_runtime_session_backend_run_idx",
|
||||
["backend_run_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.batch_alter_table("workflow_agent_runtime_sessions", schema=None) as batch_op:
|
||||
batch_op.drop_index("workflow_agent_runtime_session_backend_run_idx")
|
||||
batch_op.drop_index("workflow_agent_runtime_session_lookup_idx")
|
||||
op.drop_table("workflow_agent_runtime_sessions")
|
||||
@@ -20,6 +20,8 @@ from .agent import (
|
||||
AgentStatus,
|
||||
WorkflowAgentBindingType,
|
||||
WorkflowAgentNodeBinding,
|
||||
WorkflowAgentRuntimeSession,
|
||||
WorkflowAgentRuntimeSessionStatus,
|
||||
)
|
||||
from .api_based_extension import APIBasedExtension, APIBasedExtensionPoint
|
||||
from .comment import (
|
||||
@@ -235,6 +237,8 @@ __all__ = [
|
||||
"Workflow",
|
||||
"WorkflowAgentBindingType",
|
||||
"WorkflowAgentNodeBinding",
|
||||
"WorkflowAgentRuntimeSession",
|
||||
"WorkflowAgentRuntimeSessionStatus",
|
||||
"WorkflowAppLog",
|
||||
"WorkflowAppLogCreatedFrom",
|
||||
"WorkflowArchiveLog",
|
||||
|
||||
@@ -92,6 +92,15 @@ class WorkflowAgentBindingType(StrEnum):
|
||||
INLINE_AGENT = "inline_agent"
|
||||
|
||||
|
||||
class WorkflowAgentRuntimeSessionStatus(StrEnum):
|
||||
"""Lifecycle state of an Agent backend session snapshot owned by a workflow run."""
|
||||
|
||||
# Snapshot can be reused by a later Agent run in the same workflow run.
|
||||
ACTIVE = "active"
|
||||
# Snapshot has been retired and must not be submitted to Agent backend again.
|
||||
CLEANED = "cleaned"
|
||||
|
||||
|
||||
class Agent(DefaultFieldsMixin, Base):
|
||||
"""Workspace-scoped Agent identity used by Agent Roster and workflow-only agents."""
|
||||
|
||||
@@ -273,3 +282,56 @@ class WorkflowAgentNodeBinding(DefaultFieldsMixin, Base):
|
||||
if isinstance(self.node_job_config, str):
|
||||
return json.loads(self.node_job_config)
|
||||
return dict(self.node_job_config)
|
||||
|
||||
|
||||
class WorkflowAgentRuntimeSession(DefaultFieldsMixin, Base):
|
||||
"""Persisted Agent backend session snapshot for one workflow Agent node execution scope.
|
||||
|
||||
The snapshot is runtime state returned by Agent backend. It is intentionally
|
||||
separate from Agent Soul snapshots and workflow node-job config.
|
||||
"""
|
||||
|
||||
__tablename__ = "workflow_agent_runtime_sessions"
|
||||
__table_args__ = (
|
||||
sa.PrimaryKeyConstraint("id", name="workflow_agent_runtime_session_pkey"),
|
||||
UniqueConstraint(
|
||||
"tenant_id",
|
||||
"workflow_run_id",
|
||||
"node_id",
|
||||
"binding_id",
|
||||
"agent_id",
|
||||
name="workflow_agent_runtime_session_scope_unique",
|
||||
),
|
||||
Index(
|
||||
"workflow_agent_runtime_session_lookup_idx",
|
||||
"tenant_id",
|
||||
"workflow_run_id",
|
||||
"node_id",
|
||||
"status",
|
||||
),
|
||||
Index("workflow_agent_runtime_session_backend_run_idx", "backend_run_id"),
|
||||
)
|
||||
|
||||
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
workflow_run_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
node_id: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
node_execution_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
binding_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
agent_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
agent_config_snapshot_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
||||
backend_run_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
session_snapshot: Mapped[str] = mapped_column(LongText, nullable=False)
|
||||
# JSON-encoded list of ``WorkflowAgentSessionLayerSpec`` ({name, type, deps,
|
||||
# config}). Drives Agent backend cleanup-only runs: the agenton compositor
|
||||
# rejects a session snapshot whose layer names do not match the cleanup
|
||||
# composition, so we must replay the same layer graph (minus credential-
|
||||
# bearing plugin layers) when issuing the cleanup request.
|
||||
composition_layer_specs: Mapped[str] = mapped_column(LongText, nullable=False, server_default="[]")
|
||||
status: Mapped[WorkflowAgentRuntimeSessionStatus] = mapped_column(
|
||||
EnumText(WorkflowAgentRuntimeSessionStatus, length=32),
|
||||
nullable=False,
|
||||
default=WorkflowAgentRuntimeSessionStatus.ACTIVE,
|
||||
)
|
||||
cleaned_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
|
||||
|
||||
+1
-1
@@ -59,7 +59,7 @@ members = ["providers/vdb/*", "providers/trace/*"]
|
||||
exclude = ["providers/vdb/__pycache__", "providers/trace/__pycache__"]
|
||||
|
||||
[tool.uv.sources]
|
||||
dify-agent = { path = "../dify-agent" }
|
||||
dify-agent = { path = "../dify-agent", editable = true }
|
||||
dify-vdb-alibabacloud-mysql = { workspace = true }
|
||||
dify-vdb-analyticdb = { workspace = true }
|
||||
dify-vdb-baidu = { workspace = true }
|
||||
|
||||
@@ -97,6 +97,7 @@ class AppDslService:
|
||||
icon: str | None = None,
|
||||
icon_background: str | None = None,
|
||||
app_id: str | None = None,
|
||||
import_app_id: str | None = None,
|
||||
) -> Import:
|
||||
"""Import an app from YAML content or URL."""
|
||||
import_id = str(uuid.uuid4())
|
||||
@@ -262,6 +263,7 @@ class AppDslService:
|
||||
icon=icon,
|
||||
icon_background=icon_background,
|
||||
dependencies=check_dependencies_pending_data,
|
||||
import_app_id=import_app_id,
|
||||
)
|
||||
|
||||
draft_var_srv = WorkflowDraftVariableService(session=self._session)
|
||||
@@ -385,6 +387,7 @@ class AppDslService:
|
||||
icon: str | None = None,
|
||||
icon_background: str | None = None,
|
||||
dependencies: list[PluginDependency] | None = None,
|
||||
import_app_id: str | None = None,
|
||||
) -> App:
|
||||
"""Create a new app or update an existing one."""
|
||||
app_data = data.get("app", {})
|
||||
@@ -417,7 +420,7 @@ class AppDslService:
|
||||
|
||||
# Create new app
|
||||
app = App()
|
||||
app.id = str(uuid4())
|
||||
app.id = import_app_id or str(uuid4())
|
||||
app.tenant_id = account.current_tenant_id
|
||||
app.mode = app_mode
|
||||
app.name = name or app_data.get("name", "")
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from services.data_migration.entities import (
|
||||
ConflictStrategy,
|
||||
ExportSelection,
|
||||
IdStrategy,
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
MigrationPackage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ConflictStrategy",
|
||||
"ExportSelection",
|
||||
"IdStrategy",
|
||||
"ImportOptions",
|
||||
"MigrationDataError",
|
||||
"MigrationPackage",
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from services.data_migration.entities import DependencyKind
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveredDependency:
|
||||
kind: DependencyKind
|
||||
provider_id: str
|
||||
provider_name: str | None = None
|
||||
source: str | None = None
|
||||
|
||||
|
||||
class DependencyDiscoveryService:
|
||||
def discover_from_dsl(self, dsl: dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
seen: set[tuple[DependencyKind, str]] = set()
|
||||
result: list[DiscoveredDependency] = []
|
||||
for node in self._nodes_from_dsl(dsl):
|
||||
data = node.get("data", {}) if isinstance(node, dict) else {}
|
||||
for dependency in self._dependencies_from_node(data):
|
||||
key = (dependency.kind, dependency.provider_id)
|
||||
if dependency.provider_id and key not in seen:
|
||||
seen.add(key)
|
||||
result.append(dependency)
|
||||
return result
|
||||
|
||||
def _nodes_from_dsl(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
nodes: list[dict[str, Any]] = []
|
||||
graph = dsl.get("graph") if isinstance(dsl, dict) else None
|
||||
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
|
||||
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
|
||||
workflow = dsl.get("workflow") if isinstance(dsl, dict) else None
|
||||
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
|
||||
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
|
||||
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
|
||||
return nodes
|
||||
|
||||
def _dependencies_from_node(self, data: dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
dependencies: list[DiscoveredDependency] = []
|
||||
node_type = data.get("type")
|
||||
if node_type == "tool":
|
||||
dependency = self._from_tool_config(data, source="tool_node")
|
||||
if dependency:
|
||||
dependencies.append(dependency)
|
||||
if node_type == "agent":
|
||||
for tool_config in self._agent_tool_configs(data):
|
||||
if isinstance(tool_config, dict):
|
||||
dependency = self._from_tool_config(tool_config, source="agent_node")
|
||||
if dependency:
|
||||
dependencies.append(dependency)
|
||||
return dependencies
|
||||
|
||||
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
configs = data.get("tools")
|
||||
if isinstance(configs, list):
|
||||
return [config for config in configs if isinstance(config, dict)]
|
||||
agent_parameters = data.get("agent_parameters")
|
||||
if not isinstance(agent_parameters, dict):
|
||||
return []
|
||||
tools_parameter = agent_parameters.get("tools")
|
||||
if not isinstance(tools_parameter, dict):
|
||||
return []
|
||||
value = tools_parameter.get("value", [])
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [config for config in value if isinstance(config, dict)]
|
||||
|
||||
def _from_tool_config(self, config: dict[str, Any], *, source: str) -> DiscoveredDependency | None:
|
||||
provider_id = config.get("provider_id") or config.get("provider_name") or config.get("provider")
|
||||
if not provider_id:
|
||||
return None
|
||||
provider_type = str(config.get("provider_type") or config.get("type") or "")
|
||||
kind = self._kind_from_provider_type(provider_type)
|
||||
return DiscoveredDependency(
|
||||
kind=kind,
|
||||
provider_id=str(provider_id),
|
||||
provider_name=config.get("provider_name"),
|
||||
source=source,
|
||||
)
|
||||
|
||||
def _kind_from_provider_type(self, provider_type: str) -> DependencyKind:
|
||||
normalized = provider_type.lower()
|
||||
if normalized in {"api", "custom", "api_tool"}:
|
||||
return DependencyKind.API_TOOL
|
||||
if normalized in {"workflow", "workflow_tool"}:
|
||||
return DependencyKind.WORKFLOW_TOOL
|
||||
if normalized == "mcp":
|
||||
return DependencyKind.MCP_TOOL
|
||||
return DependencyKind.BUILTIN_OR_PLUGIN_TOOL
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Typed entities for versioned cross-environment migration packages.
|
||||
|
||||
This module is intentionally side-effect free. It owns only value objects and
|
||||
validation for migration package/config shapes; command output and database I/O
|
||||
belong in adapter and service modules built on top of these entities.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import StrEnum
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
|
||||
class MigrationDataError(ValueError):
|
||||
"""Raised when migration config or package data is invalid."""
|
||||
|
||||
|
||||
class IdStrategy(StrEnum):
|
||||
PRESERVE_ID = "preserve-id"
|
||||
GENERATE_NEW_ID = "generate-new-id"
|
||||
|
||||
|
||||
class ConflictStrategy(StrEnum):
|
||||
FAIL = "fail"
|
||||
SKIP = "skip"
|
||||
UPDATE = "update"
|
||||
|
||||
|
||||
class ResourceType(StrEnum):
|
||||
WORKFLOW = "workflow"
|
||||
API_TOOL = "api_tool"
|
||||
WORKFLOW_TOOL = "workflow_tool"
|
||||
MCP_TOOL = "mcp_tool"
|
||||
DEPENDENCY = "dependency"
|
||||
|
||||
|
||||
class DependencyKind(StrEnum):
|
||||
API_TOOL = "api_tool"
|
||||
WORKFLOW_TOOL = "workflow_tool"
|
||||
MCP_TOOL = "mcp_tool"
|
||||
BUILTIN_OR_PLUGIN_TOOL = "builtin_or_plugin_tool"
|
||||
UNRESOLVED = "unresolved"
|
||||
|
||||
|
||||
class TargetTenantSelector(TypedDict, total=False):
|
||||
id: str
|
||||
name: str
|
||||
|
||||
|
||||
def _parse_target_tenant(value: Any) -> TargetTenantSelector | None:
|
||||
if value is None:
|
||||
return None
|
||||
if not isinstance(value, dict):
|
||||
raise MigrationDataError("metadata.target_tenant must be an object when provided.")
|
||||
target: TargetTenantSelector = {}
|
||||
target_id = value.get("id")
|
||||
if target_id is not None:
|
||||
if not isinstance(target_id, str):
|
||||
raise MigrationDataError("metadata.target_tenant.id must be a string.")
|
||||
target["id"] = target_id
|
||||
target_name = value.get("name")
|
||||
if target_name is not None:
|
||||
if not isinstance(target_name, str):
|
||||
raise MigrationDataError("metadata.target_tenant.name must be a string.")
|
||||
target["name"] = target_name
|
||||
unsupported_keys = sorted(set(value.keys()) - {"id", "name"})
|
||||
if unsupported_keys:
|
||||
raise MigrationDataError(f"metadata.target_tenant contains unsupported fields: {unsupported_keys}")
|
||||
return target
|
||||
|
||||
|
||||
def _parse_package_section(value: Any, section: str) -> list[dict[str, Any]]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise MigrationDataError(f"Migration package field '{section}' must be a list.")
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
raise MigrationDataError(f"Migration package field '{section}' must contain only objects.")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SourceTenant:
|
||||
id: str
|
||||
name: str
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> SourceTenant:
|
||||
return cls(id=str(value.get("id", "")), name=str(value.get("name", "")))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportOptions:
|
||||
create_app_api_token_on_import: bool = False
|
||||
id_strategy: IdStrategy = IdStrategy.PRESERVE_ID
|
||||
conflict_strategy: ConflictStrategy = ConflictStrategy.FAIL
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any] | None) -> ImportOptions:
|
||||
value = value or {}
|
||||
try:
|
||||
id_strategy = IdStrategy(value.get("id_strategy", IdStrategy.PRESERVE_ID))
|
||||
except ValueError as exc:
|
||||
raise MigrationDataError(f"Unsupported import_options.id_strategy: {value.get('id_strategy')}") from exc
|
||||
try:
|
||||
conflict_strategy = ConflictStrategy(value.get("conflict_strategy", ConflictStrategy.FAIL))
|
||||
except ValueError as exc:
|
||||
raise MigrationDataError(
|
||||
f"Unsupported import_options.conflict_strategy: {value.get('conflict_strategy')}"
|
||||
) from exc
|
||||
return cls(
|
||||
create_app_api_token_on_import=bool(value.get("create_app_api_token_on_import", False)),
|
||||
id_strategy=id_strategy,
|
||||
conflict_strategy=conflict_strategy,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationMetadata:
|
||||
version: str
|
||||
source_scope: Literal["single"]
|
||||
source_tenants: list[SourceTenant]
|
||||
target_tenant: TargetTenantSelector | None = None
|
||||
created_at: str | None = None
|
||||
include_secrets: bool = False
|
||||
import_options: ImportOptions = field(default_factory=ImportOptions)
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> MigrationMetadata:
|
||||
version = value.get("version")
|
||||
if not version:
|
||||
raise MigrationDataError("Migration package must include metadata.version.")
|
||||
source_scope = value.get("source_scope", "single")
|
||||
if source_scope != "single":
|
||||
raise MigrationDataError(f"Unsupported source_scope: {source_scope}")
|
||||
source_tenants = [
|
||||
SourceTenant.from_mapping(item) for item in value.get("source_tenants", []) if isinstance(item, dict)
|
||||
]
|
||||
return cls(
|
||||
version=str(version),
|
||||
source_scope="single",
|
||||
source_tenants=source_tenants,
|
||||
target_tenant=_parse_target_tenant(value.get("target_tenant")),
|
||||
created_at=value.get("created_at"),
|
||||
include_secrets=bool(value.get("include_secrets", False)),
|
||||
import_options=ImportOptions.from_mapping(value.get("import_options")),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MigrationPackage:
|
||||
metadata: MigrationMetadata
|
||||
workflows: list[dict[str, Any]] = field(default_factory=list)
|
||||
tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
workflow_tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
mcp_tools: list[dict[str, Any]] = field(default_factory=list)
|
||||
dependencies: list[dict[str, Any]] = field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> MigrationPackage:
|
||||
metadata_value = value.get("metadata")
|
||||
if not isinstance(metadata_value, dict):
|
||||
raise MigrationDataError("Migration package must include metadata.version.")
|
||||
return cls(
|
||||
metadata=MigrationMetadata.from_mapping(metadata_value),
|
||||
workflows=_parse_package_section(value.get("workflows"), "workflows"),
|
||||
tools=_parse_package_section(value.get("tools"), "tools"),
|
||||
workflow_tools=_parse_package_section(value.get("workflow_tools"), "workflow_tools"),
|
||||
mcp_tools=_parse_package_section(value.get("mcp_tools"), "mcp_tools"),
|
||||
dependencies=_parse_package_section(value.get("dependencies"), "dependencies"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExportSelection:
|
||||
source_tenant_name: str
|
||||
app_ids: list[str]
|
||||
source_tenant_id: str | None = None
|
||||
export_all_apps: bool = False
|
||||
include_referenced_tools: bool = True
|
||||
additional_api_tools: list[str] = field(default_factory=list)
|
||||
additional_workflow_tools: list[str] = field(default_factory=list)
|
||||
additional_mcp_tools: list[str] = field(default_factory=list)
|
||||
include_secrets: bool = False
|
||||
import_options: ImportOptions = field(default_factory=ImportOptions)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResourceReportItem:
|
||||
resource_type: ResourceType
|
||||
identifier: str
|
||||
name: str | None
|
||||
status: str
|
||||
message: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResourceIdMapping:
|
||||
resource_type: ResourceType
|
||||
name: str | None
|
||||
source_id: str
|
||||
target_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExportResult:
|
||||
package: MigrationPackage
|
||||
report_items: list[ResourceReportItem]
|
||||
report_context: ReportContext | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportTarget:
|
||||
tenant_id: str
|
||||
tenant_name: str
|
||||
operator_id: str
|
||||
operator_email: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportResult:
|
||||
report_items: list[ResourceReportItem]
|
||||
id_mapping: dict[str, str] = field(default_factory=dict)
|
||||
report_context: ReportContext | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReportContext:
|
||||
output_path: str | None = None
|
||||
source_scope: str | None = None
|
||||
selected_app_count: int | None = None
|
||||
include_secrets: bool | None = None
|
||||
target_tenant: str | None = None
|
||||
operator_email: str | None = None
|
||||
app_api_tokens_created: int = 0
|
||||
app_api_tokens_reused: int = 0
|
||||
id_mapping_count: int = 0
|
||||
id_mappings: dict[str, str] = field(default_factory=dict)
|
||||
id_mapping_details: list[ResourceIdMapping] = field(default_factory=list)
|
||||
@@ -0,0 +1,492 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import sqlalchemy as sa
|
||||
import yaml
|
||||
|
||||
from core.tools.tool_manager import ToolManager
|
||||
from extensions.ext_database import db
|
||||
from graphon.model_runtime.utils.encoders import jsonable_encoder
|
||||
from models import Account, Tenant
|
||||
from models.account import TenantAccountJoin
|
||||
from models.model import App
|
||||
from models.tools import MCPToolProvider
|
||||
from services.app_dsl_service import AppDslService
|
||||
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService, DiscoveredDependency
|
||||
from services.data_migration.entities import (
|
||||
DependencyKind,
|
||||
ExportResult,
|
||||
ExportSelection,
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
ReportContext,
|
||||
ResourceReportItem,
|
||||
ResourceType,
|
||||
)
|
||||
from services.data_migration.package_service import MigrationPackageService
|
||||
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
|
||||
|
||||
SUPPORTED_APP_MODES = {"workflow", "advanced-chat"}
|
||||
|
||||
|
||||
class ExportConfigParser:
|
||||
def parse(self, data: dict[str, Any]) -> ExportSelection:
|
||||
if not isinstance(data, dict):
|
||||
raise MigrationDataError("Export config JSON must be an object.")
|
||||
|
||||
source_tenant = self._source_tenant(data)
|
||||
source_tenant_name = self._source_tenant_name(source_tenant, data)
|
||||
apps = self._mapping(data.get("apps"), field_name="apps")
|
||||
self._validate_source_scope(data)
|
||||
self._validate_app_modes(apps.get("modes", []))
|
||||
|
||||
additional_tools = self._mapping(data.get("additional_tools"), field_name="additional_tools")
|
||||
return ExportSelection(
|
||||
source_tenant_name=source_tenant_name,
|
||||
app_ids=self._string_list(apps.get("ids", data.get("workflows", [])), field_name="apps.ids"),
|
||||
source_tenant_id=source_tenant.get("id"),
|
||||
export_all_apps=bool(apps.get("all", data.get("export_all_workflows", False))),
|
||||
include_referenced_tools=bool(data.get("include_referenced_tools", True)),
|
||||
additional_api_tools=self._string_list(
|
||||
additional_tools.get("api_tools", data.get("tools", [])), field_name="additional_tools.api_tools"
|
||||
),
|
||||
additional_workflow_tools=self._string_list(
|
||||
additional_tools.get("workflow_tools", data.get("workflow_tools", [])),
|
||||
field_name="additional_tools.workflow_tools",
|
||||
),
|
||||
additional_mcp_tools=self._string_list(
|
||||
additional_tools.get("mcp_tools", data.get("mcp_tools", [])),
|
||||
field_name="additional_tools.mcp_tools",
|
||||
),
|
||||
include_secrets=bool(data.get("include_secrets", False)),
|
||||
import_options=ImportOptions.from_mapping(data.get("import_options")),
|
||||
)
|
||||
|
||||
def _source_tenant(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
if "source_tenant" in data:
|
||||
return self._mapping(data.get("source_tenant"), field_name="source_tenant")
|
||||
return {}
|
||||
|
||||
def _source_tenant_name(self, source_tenant: dict[str, Any], data: dict[str, Any]) -> str:
|
||||
if source_tenant:
|
||||
source_tenant_name = source_tenant.get("name")
|
||||
if not source_tenant_name:
|
||||
raise MigrationDataError("Export config must include source_tenant.name.")
|
||||
return str(source_tenant_name)
|
||||
source_tenant_name = data.get("tenant_name")
|
||||
if not source_tenant_name:
|
||||
raise MigrationDataError("Export config must include source_tenant.name.")
|
||||
return str(source_tenant_name)
|
||||
|
||||
def _validate_source_scope(self, data: dict[str, Any]) -> None:
|
||||
source_tenant = data.get("source_tenant")
|
||||
if not isinstance(source_tenant, dict):
|
||||
return
|
||||
mode = source_tenant.get("mode", "single")
|
||||
if mode != "single":
|
||||
raise MigrationDataError(f"Unsupported source_tenant.mode: {mode}")
|
||||
|
||||
def _validate_app_modes(self, modes: Any) -> None:
|
||||
app_modes = self._string_list(modes, field_name="apps.modes") if modes else []
|
||||
unsupported_modes = sorted(set(app_modes) - SUPPORTED_APP_MODES)
|
||||
if unsupported_modes:
|
||||
raise MigrationDataError(f"Unsupported app modes for export: {unsupported_modes}")
|
||||
|
||||
def _mapping(self, value: Any, *, field_name: str) -> dict[str, Any]:
|
||||
if value is None:
|
||||
return {}
|
||||
if not isinstance(value, dict):
|
||||
raise MigrationDataError(f"Export config field '{field_name}' must be an object.")
|
||||
return value
|
||||
|
||||
def _string_list(self, value: Any, *, field_name: str) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if not isinstance(value, list):
|
||||
raise MigrationDataError(f"Export config field '{field_name}' must be a list.")
|
||||
return [str(item) for item in value]
|
||||
|
||||
|
||||
class MigrationExportService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
package_service: MigrationPackageService | None = None,
|
||||
dependency_discovery_service: DependencyDiscoveryService | None = None,
|
||||
) -> None:
|
||||
self.package_service = package_service or MigrationPackageService()
|
||||
self.dependency_discovery_service = dependency_discovery_service or DependencyDiscoveryService()
|
||||
|
||||
def export(self, selection: ExportSelection) -> ExportResult:
|
||||
tenant = self._get_tenant(selection)
|
||||
package = self.package_service.build_empty_package(
|
||||
source_tenant_id=tenant.id,
|
||||
source_tenant_name=tenant.name,
|
||||
include_secrets=selection.include_secrets,
|
||||
import_options=selection.import_options,
|
||||
)
|
||||
report_items: list[ResourceReportItem] = []
|
||||
discovered_dependencies: list[DiscoveredDependency] = []
|
||||
|
||||
apps = self._selected_apps(tenant.id, selection)
|
||||
exported_app_ids = {app.id for app in apps}
|
||||
for app in apps:
|
||||
dsl_content = AppDslService.export_dsl(app_model=app, include_secret=selection.include_secrets)
|
||||
package.workflows.append(
|
||||
{
|
||||
"id": app.id,
|
||||
"name": app.name,
|
||||
"mode": app.mode.value if hasattr(app.mode, "value") else app.mode,
|
||||
"dsl": dsl_content,
|
||||
"source_tenant_id": tenant.id,
|
||||
"create_app_api_token_on_import": selection.import_options.create_app_api_token_on_import,
|
||||
}
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW, app.id, app.name, "exported"))
|
||||
if selection.include_referenced_tools:
|
||||
discovered_dependencies.extend(self._discover_dependencies(dsl_content))
|
||||
|
||||
self._export_api_tools(
|
||||
tenant.id,
|
||||
self._provider_ids(selection.additional_api_tools, discovered_dependencies, DependencyKind.API_TOOL),
|
||||
include_secrets=selection.include_secrets,
|
||||
exported_tools=package.tools,
|
||||
report_items=report_items,
|
||||
)
|
||||
self._export_workflow_tools(
|
||||
tenant,
|
||||
self._provider_ids(
|
||||
selection.additional_workflow_tools, discovered_dependencies, DependencyKind.WORKFLOW_TOOL
|
||||
),
|
||||
exported_app_ids=exported_app_ids,
|
||||
exported_workflow_tools=package.workflow_tools,
|
||||
dependencies=package.dependencies,
|
||||
report_items=report_items,
|
||||
)
|
||||
self._export_mcp_tools(
|
||||
tenant_id=tenant.id,
|
||||
provider_ids=self._provider_ids(
|
||||
selection.additional_mcp_tools,
|
||||
discovered_dependencies,
|
||||
DependencyKind.MCP_TOOL,
|
||||
),
|
||||
include_secrets=selection.include_secrets,
|
||||
exported_mcp_tools=package.mcp_tools,
|
||||
dependencies=package.dependencies,
|
||||
report_items=report_items,
|
||||
)
|
||||
self._record_dependency_metadata(
|
||||
self._dependencies_by_kind(discovered_dependencies, DependencyKind.BUILTIN_OR_PLUGIN_TOOL),
|
||||
package.dependencies,
|
||||
report_items,
|
||||
)
|
||||
return ExportResult(
|
||||
package=package,
|
||||
report_items=report_items,
|
||||
report_context=ReportContext(
|
||||
source_scope=package.metadata.source_scope,
|
||||
selected_app_count=len(apps),
|
||||
include_secrets=selection.include_secrets,
|
||||
),
|
||||
)
|
||||
|
||||
def _get_tenant(self, selection: ExportSelection) -> Tenant:
|
||||
if selection.source_tenant_id:
|
||||
tenant = db.session.get(Tenant, selection.source_tenant_id)
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_id}")
|
||||
if tenant.name != selection.source_tenant_name:
|
||||
raise MigrationDataError(
|
||||
f"Source tenant id/name mismatch: {selection.source_tenant_id} / {selection.source_tenant_name}"
|
||||
)
|
||||
return tenant
|
||||
tenants = list(db.session.scalars(sa.select(Tenant).where(Tenant.name == selection.source_tenant_name)).all())
|
||||
if not tenants:
|
||||
raise MigrationDataError(f"Source tenant not found: {selection.source_tenant_name}")
|
||||
if len(tenants) > 1:
|
||||
raise MigrationDataError(
|
||||
f"Source tenant name is ambiguous; use source_tenant.id: {selection.source_tenant_name}"
|
||||
)
|
||||
return tenants[0]
|
||||
|
||||
def _selected_apps(self, tenant_id: str, selection: ExportSelection) -> list[App]:
|
||||
query = sa.select(App).where(App.tenant_id == tenant_id, App.mode.in_(SUPPORTED_APP_MODES))
|
||||
if not selection.export_all_apps:
|
||||
if not selection.app_ids:
|
||||
return []
|
||||
query = query.where(App.id.in_(selection.app_ids))
|
||||
apps = list(db.session.scalars(query).all())
|
||||
if not selection.export_all_apps and len(apps) != len(set(selection.app_ids)):
|
||||
found_ids = {app.id for app in apps}
|
||||
missing_ids = [app_id for app_id in selection.app_ids if app_id not in found_ids]
|
||||
raise MigrationDataError(
|
||||
f"Selected app IDs not found in source tenant or unsupported app mode: {missing_ids}"
|
||||
)
|
||||
return apps
|
||||
|
||||
def _discover_dependencies(self, dsl_content: str | dict[str, Any]) -> list[DiscoveredDependency]:
|
||||
if isinstance(dsl_content, dict):
|
||||
dsl = dsl_content
|
||||
else:
|
||||
raw_dsl = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
dsl = raw_dsl if isinstance(raw_dsl, dict) else {}
|
||||
return self.dependency_discovery_service.discover_from_dsl(dsl)
|
||||
|
||||
def _export_api_tools(
|
||||
self,
|
||||
tenant_id: str,
|
||||
provider_ids: Iterable[str],
|
||||
*,
|
||||
include_secrets: bool,
|
||||
exported_tools: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
for provider_id in self._dedupe(provider_ids):
|
||||
try:
|
||||
tool_data = ToolManager.user_get_api_provider(
|
||||
provider=provider_id,
|
||||
tenant_id=tenant_id,
|
||||
mask=not include_secrets,
|
||||
)
|
||||
if not include_secrets:
|
||||
tool_data.pop("credentials", None)
|
||||
tool_data.pop("tools", None)
|
||||
tool_data["provider_name"] = provider_id
|
||||
tool_data["source_tenant_id"] = tenant_id
|
||||
exported_tools.append(tool_data)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "exported"))
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.API_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _export_workflow_tools(
|
||||
self,
|
||||
tenant: Tenant,
|
||||
provider_ids: Iterable[str],
|
||||
*,
|
||||
exported_app_ids: set[str],
|
||||
exported_workflow_tools: list[dict[str, Any]],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
provider_ids = self._dedupe(provider_ids)
|
||||
if not provider_ids:
|
||||
return
|
||||
owner = self._get_tenant_owner(tenant.id)
|
||||
if owner is None:
|
||||
for provider_id in provider_ids:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
provider_id,
|
||||
provider_id,
|
||||
"unresolved",
|
||||
f"No owner account found for source tenant: {tenant.name}",
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
for provider_id in provider_ids:
|
||||
try:
|
||||
tool_data = WorkflowToolManageService.get_workflow_tool_by_tool_id(
|
||||
user_id=owner.id,
|
||||
tenant_id=tenant.id,
|
||||
workflow_tool_id=provider_id,
|
||||
)
|
||||
tool_info = jsonable_encoder(tool_data)
|
||||
tool_info["id"] = provider_id
|
||||
tool_info["app_id"] = tool_info.get("workflow_app_id")
|
||||
tool_info["source_tenant_id"] = tenant.id
|
||||
for field_name in ("workflow_tool_id", "workflow_app_id", "tool"):
|
||||
tool_info.pop(field_name, None)
|
||||
exported_workflow_tools.append(tool_info)
|
||||
if tool_info.get("app_id") not in exported_app_ids:
|
||||
workflow_app_id = str(tool_info.get("app_id") or "")
|
||||
workflow_app = db.session.get(App, workflow_app_id) if workflow_app_id else None
|
||||
self._record_dependency_metadata(
|
||||
[
|
||||
DiscoveredDependency(
|
||||
DependencyKind.WORKFLOW_TOOL,
|
||||
workflow_app_id,
|
||||
provider_name=workflow_app.name if workflow_app else tool_info.get("name"),
|
||||
source="workflow_tool_app",
|
||||
)
|
||||
],
|
||||
dependencies,
|
||||
report_items,
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, tool_info.get("name"), "exported")
|
||||
)
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _get_tenant_owner(self, tenant_id: str) -> Account | None:
|
||||
return db.session.scalar(
|
||||
sa.select(Account)
|
||||
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
|
||||
.where(TenantAccountJoin.tenant_id == tenant_id, TenantAccountJoin.role == "owner")
|
||||
.order_by(TenantAccountJoin.created_at.asc())
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def _export_mcp_tools(
|
||||
self,
|
||||
*,
|
||||
tenant_id: str,
|
||||
provider_ids: Iterable[str],
|
||||
include_secrets: bool,
|
||||
exported_mcp_tools: list[dict[str, Any]],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
for provider_id in self._dedupe(provider_ids):
|
||||
if not include_secrets:
|
||||
self._record_dependency_metadata(
|
||||
[DiscoveredDependency(DependencyKind.MCP_TOOL, provider_id, source="mcp_provider")],
|
||||
dependencies,
|
||||
report_items,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
provider = self._get_mcp_provider(tenant_id, provider_id)
|
||||
exported_mcp_tools.append(self._serialize_mcp_provider(provider))
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider.name, "exported"))
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.MCP_TOOL, provider_id, provider_id, "unresolved", str(exc))
|
||||
)
|
||||
|
||||
def _get_mcp_provider(self, tenant_id: str, provider_id: str) -> MCPToolProvider:
|
||||
predicates = [MCPToolProvider.server_identifier == provider_id]
|
||||
if self._is_uuid_string(provider_id):
|
||||
predicates.append(MCPToolProvider.id == provider_id)
|
||||
provider = db.session.scalar(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, sa.or_(*predicates))
|
||||
)
|
||||
if provider is None:
|
||||
raise MigrationDataError(f"MCP provider not found: {provider_id}")
|
||||
return provider
|
||||
|
||||
def _is_uuid_string(self, value: str) -> bool:
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _serialize_mcp_provider(self, provider: MCPToolProvider) -> dict[str, Any]:
|
||||
provider_entity = provider.to_entity()
|
||||
provider_icon = provider_entity.provider_icon
|
||||
if isinstance(provider_icon, dict):
|
||||
icon = provider_icon.get("content")
|
||||
icon_background = provider_icon.get("background")
|
||||
icon_type = "emoji"
|
||||
else:
|
||||
icon = provider_icon
|
||||
icon_background = None
|
||||
icon_type = "url"
|
||||
return {
|
||||
"id": provider.id,
|
||||
"name": provider.name,
|
||||
"server_url": provider_entity.decrypt_server_url(),
|
||||
"server_identifier": provider.server_identifier,
|
||||
"icon": icon,
|
||||
"icon_background": icon_background,
|
||||
"icon_type": icon_type,
|
||||
"configuration": {"timeout": provider.timeout, "sse_read_timeout": provider.sse_read_timeout},
|
||||
"headers": provider_entity.decrypt_headers(),
|
||||
"authentication": self._serialize_mcp_authentication(provider_entity.decrypt_authentication()),
|
||||
"tools": provider.tool_dict,
|
||||
"source_tenant_id": provider.tenant_id,
|
||||
}
|
||||
|
||||
def _serialize_mcp_authentication(self, authentication: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
if not authentication or not authentication.get("client_id"):
|
||||
return None
|
||||
return {
|
||||
"client_id": authentication["client_id"],
|
||||
"client_secret": authentication.get("client_secret"),
|
||||
}
|
||||
|
||||
def _record_dependency_metadata(
|
||||
self,
|
||||
dependencies_to_record: Iterable[DiscoveredDependency],
|
||||
dependencies: list[dict[str, Any]],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
existing = {(item.get("kind"), item.get("provider_id")) for item in dependencies}
|
||||
for dependency in dependencies_to_record:
|
||||
key = (dependency.kind.value, dependency.provider_id)
|
||||
if key in existing:
|
||||
continue
|
||||
existing.add(key)
|
||||
dependencies.append(
|
||||
{
|
||||
"kind": dependency.kind.value,
|
||||
"provider_id": dependency.provider_id,
|
||||
"provider_name": dependency.provider_name,
|
||||
"source": dependency.source,
|
||||
}
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
dependency.provider_id,
|
||||
self._dependency_report_name(dependency),
|
||||
"dependency-only",
|
||||
self._dependency_message(dependency.kind),
|
||||
)
|
||||
)
|
||||
|
||||
def _provider_ids(
|
||||
self,
|
||||
manual_provider_ids: Iterable[str],
|
||||
discovered_dependencies: Iterable[DiscoveredDependency],
|
||||
kind: DependencyKind,
|
||||
) -> list[str]:
|
||||
provider_ids = list(manual_provider_ids)
|
||||
provider_ids.extend(
|
||||
self._provider_export_identifier(dependency)
|
||||
for dependency in discovered_dependencies
|
||||
if dependency.kind == kind
|
||||
)
|
||||
return self._dedupe(provider_ids)
|
||||
|
||||
def _provider_export_identifier(self, dependency: DiscoveredDependency) -> str:
|
||||
if dependency.kind == DependencyKind.API_TOOL and dependency.provider_name:
|
||||
return dependency.provider_name
|
||||
return dependency.provider_id
|
||||
|
||||
def _dependencies_by_kind(
|
||||
self, discovered_dependencies: Iterable[DiscoveredDependency], kind: DependencyKind
|
||||
) -> list[DiscoveredDependency]:
|
||||
return [dependency for dependency in discovered_dependencies if dependency.kind == kind]
|
||||
|
||||
def _dedupe(self, values: Iterable[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for value in values:
|
||||
if value and value not in seen:
|
||||
seen.add(value)
|
||||
result.append(value)
|
||||
return result
|
||||
|
||||
def _dependency_message(self, kind: DependencyKind) -> str:
|
||||
if kind == DependencyKind.MCP_TOOL:
|
||||
return "Configure MCP provider manually in the target tenant unless exporting with secrets enabled."
|
||||
if kind == DependencyKind.BUILTIN_OR_PLUGIN_TOOL:
|
||||
return "Ensure the built-in or plugin tool exists in the target environment."
|
||||
return "Dependency metadata only; ensure the resource exists in the target environment."
|
||||
|
||||
def _dependency_report_name(self, dependency: DiscoveredDependency) -> str:
|
||||
name = dependency.provider_name or dependency.provider_id
|
||||
if dependency.kind == DependencyKind.WORKFLOW_TOOL:
|
||||
return f"workflow {name}"
|
||||
return f"{dependency.kind.value} {name}"
|
||||
@@ -0,0 +1,938 @@
|
||||
"""Apply versioned migration packages to an explicitly resolved target tenant.
|
||||
|
||||
Import target resolution is deliberately performed before any resource import
|
||||
work. The service does not write Click output; callers receive structured
|
||||
report items and can decide how to render them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
from uuid import UUID
|
||||
|
||||
import sqlalchemy as sa
|
||||
import yaml
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.entities.mcp_provider import MCPAuthentication, MCPConfiguration
|
||||
from core.tools.entities.tool_entities import ApiProviderSchemaType, WorkflowToolParameterConfiguration
|
||||
from extensions.ext_database import db
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from models import Account, ApiToken, Tenant, TenantAccountJoin, TenantAccountRole
|
||||
from models.enums import ApiTokenType
|
||||
from models.model import App
|
||||
from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider
|
||||
from services.app_dsl_service import AppDslService
|
||||
from services.data_migration.dependency_discovery_service import DependencyDiscoveryService
|
||||
from services.data_migration.entities import (
|
||||
ConflictStrategy,
|
||||
DependencyKind,
|
||||
IdStrategy,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
ImportTarget,
|
||||
MigrationDataError,
|
||||
MigrationPackage,
|
||||
ReportContext,
|
||||
ResourceIdMapping,
|
||||
ResourceReportItem,
|
||||
ResourceType,
|
||||
)
|
||||
from services.entities.dsl_entities import ImportStatus
|
||||
from services.tools.api_tools_manage_service import ApiToolManageService
|
||||
from services.tools.mcp_tools_manage_service import MCPToolManageService
|
||||
from services.tools.workflow_tools_manage_service import WorkflowToolManageService
|
||||
from services.workflow_service import WorkflowService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportRequest:
|
||||
"""Structured input for package import.
|
||||
|
||||
`cli_target_tenant` and `config_target_tenant` are target tenant names from
|
||||
outer adapters. They intentionally override package metadata, because a
|
||||
migration package may be reused across environments.
|
||||
"""
|
||||
|
||||
package: MigrationPackage
|
||||
cli_target_tenant: str | None = None
|
||||
config_target_tenant: str | None = None
|
||||
operator_email: str | None = None
|
||||
options_override: ImportOptions | None = None
|
||||
|
||||
|
||||
class ImportTargetResolver:
|
||||
"""Resolve the target tenant and operator before import side effects begin."""
|
||||
|
||||
def select_target_tenant_name(self, request: ImportRequest) -> str:
|
||||
if request.cli_target_tenant:
|
||||
return request.cli_target_tenant
|
||||
if request.config_target_tenant:
|
||||
return request.config_target_tenant
|
||||
package_target = request.package.metadata.target_tenant or {}
|
||||
if package_target.get("name"):
|
||||
return package_target["name"]
|
||||
if package_target.get("id"):
|
||||
return package_target["id"]
|
||||
raise MigrationDataError(
|
||||
"Target tenant must be provided by --target-tenant, import config, or package metadata."
|
||||
)
|
||||
|
||||
def resolve(self, request: ImportRequest) -> ImportTarget:
|
||||
target_tenant_name = self.select_target_tenant_name(request)
|
||||
package_target = request.package.metadata.target_tenant or {}
|
||||
if request.cli_target_tenant or request.config_target_tenant:
|
||||
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name)
|
||||
elif package_target.get("id") and self._is_uuid(package_target["id"]):
|
||||
tenant = db.session.get(Tenant, package_target["id"])
|
||||
if tenant is not None and package_target.get("name") and tenant.name != package_target.get("name"):
|
||||
raise MigrationDataError(
|
||||
f"Target tenant id/name mismatch: {package_target['id']} / {package_target['name']}"
|
||||
)
|
||||
else:
|
||||
tenant = self._resolve_tenant_by_id_or_name(target_tenant_name)
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Target tenant not found: {target_tenant_name}")
|
||||
|
||||
account_query = (
|
||||
db.session.query(Account)
|
||||
.join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id)
|
||||
.filter(TenantAccountJoin.tenant_id == tenant.id)
|
||||
)
|
||||
if request.operator_email:
|
||||
account_query = account_query.filter(Account.email == request.operator_email)
|
||||
identity = request.operator_email
|
||||
else:
|
||||
account_query = account_query.filter(TenantAccountJoin.role == TenantAccountRole.OWNER).order_by(
|
||||
TenantAccountJoin.created_at.asc()
|
||||
)
|
||||
identity = "earliest owner"
|
||||
|
||||
account = account_query.first()
|
||||
if account is None:
|
||||
raise MigrationDataError(f"No operator account found for target tenant {target_tenant_name}: {identity}")
|
||||
|
||||
return ImportTarget(
|
||||
tenant_id=tenant.id,
|
||||
tenant_name=tenant.name,
|
||||
operator_id=account.id,
|
||||
operator_email=account.email,
|
||||
)
|
||||
|
||||
def _resolve_tenant_by_id_or_name(self, value: str) -> Tenant | None:
|
||||
if self._is_uuid(value):
|
||||
tenant = db.session.get(Tenant, value)
|
||||
if tenant is not None:
|
||||
return tenant
|
||||
tenants = list(db.session.scalars(sa.select(Tenant).where(Tenant.name == value)).all())
|
||||
if len(tenants) > 1:
|
||||
raise MigrationDataError(f"Target tenant name is ambiguous; use target_tenant.id: {value}")
|
||||
return tenants[0] if tenants else None
|
||||
|
||||
def _is_uuid(self, value: str) -> bool:
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class MigrationImportService:
|
||||
"""Apply package resources using Dify service APIs and structured reporting."""
|
||||
|
||||
target_resolver: ImportTargetResolver
|
||||
|
||||
def __init__(self, *, target_resolver: ImportTargetResolver | None = None) -> None:
|
||||
self.target_resolver = target_resolver or ImportTargetResolver()
|
||||
|
||||
def import_package(self, request: ImportRequest) -> ImportResult:
|
||||
target = self.target_resolver.resolve(request)
|
||||
options = request.options_override or request.package.metadata.import_options
|
||||
report_items = [
|
||||
ResourceReportItem(
|
||||
resource_type=ResourceType.DEPENDENCY,
|
||||
identifier=target.tenant_id,
|
||||
name=target.tenant_name,
|
||||
status="resolved",
|
||||
message=f"operator: {target.operator_email or target.operator_id}",
|
||||
)
|
||||
]
|
||||
id_mapping: dict[str, str] = {}
|
||||
id_mapping_details: list[ResourceIdMapping] = []
|
||||
|
||||
self._import_api_tools(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
self._source_api_provider_ids_by_name(request.package),
|
||||
)
|
||||
self._import_mcp_tools(request.package, target, options, report_items, id_mapping, id_mapping_details)
|
||||
self._preflight_dependency_only_mcp(request.package, target, report_items)
|
||||
workflow_tool_app_ids = self._workflow_tool_source_app_ids(request.package)
|
||||
imported_workflow_ids: set[str] = set()
|
||||
if workflow_tool_app_ids:
|
||||
self._import_workflows(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details=id_mapping_details,
|
||||
imported_workflow_ids=imported_workflow_ids,
|
||||
only_app_ids=workflow_tool_app_ids,
|
||||
)
|
||||
self._import_workflow_tools(request.package, target, options, id_mapping, id_mapping_details, report_items)
|
||||
self._import_workflows(
|
||||
request.package,
|
||||
target,
|
||||
options,
|
||||
report_items,
|
||||
id_mapping,
|
||||
id_mapping_details=id_mapping_details,
|
||||
imported_workflow_ids=imported_workflow_ids,
|
||||
skip_app_ids=imported_workflow_ids,
|
||||
)
|
||||
return ImportResult(
|
||||
report_items=report_items,
|
||||
id_mapping=id_mapping,
|
||||
report_context=ReportContext(
|
||||
target_tenant=target.tenant_name,
|
||||
operator_email=target.operator_email,
|
||||
id_mapping_count=len(id_mapping),
|
||||
id_mappings=dict(id_mapping),
|
||||
id_mapping_details=id_mapping_details,
|
||||
),
|
||||
)
|
||||
|
||||
def _import_workflows(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
imported_workflow_ids: set[str] | None = None,
|
||||
only_app_ids: set[str] | None = None,
|
||||
skip_app_ids: set[str] | None = None,
|
||||
) -> None:
|
||||
account = db.session.get(Account, target.operator_id)
|
||||
tenant = db.session.get(Tenant, target.tenant_id)
|
||||
if account is None:
|
||||
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
|
||||
if tenant is None:
|
||||
raise MigrationDataError(f"Target tenant not found: {target.tenant_id}")
|
||||
account.current_tenant = tenant
|
||||
|
||||
for workflow_data in package.workflows:
|
||||
app_id = self._optional_string(workflow_data.get("id"))
|
||||
if only_app_ids and app_id not in only_app_ids:
|
||||
continue
|
||||
if skip_app_ids and app_id in skip_app_ids:
|
||||
continue
|
||||
dsl_content = self._rewrite_workflow_dsl_provider_ids(
|
||||
self._required_string(workflow_data, "dsl", "workflow"),
|
||||
id_mapping,
|
||||
)
|
||||
existing_app = (
|
||||
self._find_existing_app(app_id, target.tenant_id)
|
||||
if options.id_strategy == IdStrategy.PRESERVE_ID
|
||||
else None
|
||||
)
|
||||
if existing_app is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"App already exists and conflict_strategy=fail: {app_id}")
|
||||
if existing_app is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if app_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW,
|
||||
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
|
||||
{app_id},
|
||||
existing_app.id,
|
||||
)
|
||||
report_items.append(
|
||||
ResourceReportItem(ResourceType.WORKFLOW, str(app_id), workflow_data.get("name"), "skipped")
|
||||
)
|
||||
continue
|
||||
|
||||
imported_app_id = self._import_workflow_app(
|
||||
account=account,
|
||||
workflow_data=workflow_data,
|
||||
dsl_content=dsl_content,
|
||||
app_id=app_id,
|
||||
existing_app=existing_app,
|
||||
options=options,
|
||||
)
|
||||
if app_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW,
|
||||
workflow_data.get("name") if isinstance(workflow_data.get("name"), str) else None,
|
||||
{app_id},
|
||||
imported_app_id,
|
||||
)
|
||||
if imported_workflow_ids is not None:
|
||||
imported_workflow_ids.add(app_id)
|
||||
if options.create_app_api_token_on_import:
|
||||
self._create_or_reuse_app_api_token(imported_app_id, target.tenant_id)
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW,
|
||||
imported_app_id,
|
||||
workflow_data.get("name"),
|
||||
"updated" if existing_app is not None else "created",
|
||||
)
|
||||
)
|
||||
|
||||
def _workflow_tool_source_app_ids(self, package: MigrationPackage) -> set[str]:
|
||||
app_ids: set[str] = set()
|
||||
for workflow_tool_data in package.workflow_tools:
|
||||
app_id = self._optional_string(workflow_tool_data.get("app_id"))
|
||||
if app_id:
|
||||
app_ids.add(app_id)
|
||||
return app_ids
|
||||
|
||||
def _import_workflow_app(
|
||||
self,
|
||||
*,
|
||||
account: Account,
|
||||
workflow_data: dict[str, object],
|
||||
dsl_content: str,
|
||||
app_id: str | None,
|
||||
existing_app: App | None,
|
||||
options: ImportOptions,
|
||||
) -> str:
|
||||
import_service = AppDslService(cast(Session, db.session))
|
||||
if existing_app is not None:
|
||||
import_result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode="yaml-content",
|
||||
yaml_content=dsl_content,
|
||||
app_id=existing_app.id,
|
||||
)
|
||||
else:
|
||||
import_app_id = app_id if self._should_preserve_source_app_id(options) else None
|
||||
import_result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode="yaml-content",
|
||||
yaml_content=dsl_content,
|
||||
import_app_id=import_app_id,
|
||||
)
|
||||
if import_result.status not in {ImportStatus.COMPLETED, ImportStatus.COMPLETED_WITH_WARNINGS}:
|
||||
error = import_result.error or f"unexpected import status {import_result.status}"
|
||||
raise MigrationDataError(f"Workflow import failed: {error}")
|
||||
if import_result.app_id is None:
|
||||
raise MigrationDataError(f"Workflow import did not return an app id: {workflow_data.get('name')}")
|
||||
db.session.commit()
|
||||
return import_result.app_id
|
||||
|
||||
def _rewrite_workflow_dsl_provider_ids(self, dsl_content: str, id_mapping: dict[str, str]) -> str:
|
||||
if not id_mapping:
|
||||
return dsl_content
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
return dsl_content
|
||||
for node in self._workflow_nodes(parsed):
|
||||
data = node.get("data") if isinstance(node, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
self._rewrite_tool_config_provider_id(data, id_mapping)
|
||||
for tool_config in self._agent_tool_configs(data):
|
||||
self._rewrite_tool_config_provider_id(tool_config, id_mapping)
|
||||
return yaml.safe_dump(parsed, sort_keys=False, allow_unicode=True)
|
||||
|
||||
def _rewrite_tool_config_provider_id(self, tool_config: dict[str, Any], id_mapping: dict[str, str]) -> None:
|
||||
provider_id = self._optional_string(tool_config.get("provider_id"))
|
||||
if provider_id and provider_id in id_mapping:
|
||||
tool_config["provider_id"] = id_mapping[provider_id]
|
||||
|
||||
def _source_api_provider_ids_by_name(self, package: MigrationPackage) -> dict[str, set[str]]:
|
||||
provider_ids_by_name: dict[str, set[str]] = {}
|
||||
discovery_service = DependencyDiscoveryService()
|
||||
for workflow_data in package.workflows:
|
||||
dsl_content = self._optional_string(workflow_data.get("dsl"))
|
||||
if not dsl_content:
|
||||
continue
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
for dependency in discovery_service.discover_from_dsl(parsed):
|
||||
if dependency.kind != DependencyKind.API_TOOL or not dependency.provider_name:
|
||||
continue
|
||||
provider_ids_by_name.setdefault(dependency.provider_name, set()).add(dependency.provider_id)
|
||||
return provider_ids_by_name
|
||||
|
||||
def _workflow_nodes(self, dsl: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
nodes: list[dict[str, Any]] = []
|
||||
graph = dsl.get("graph")
|
||||
if isinstance(graph, dict) and isinstance(graph.get("nodes"), list):
|
||||
nodes.extend(node for node in graph["nodes"] if isinstance(node, dict))
|
||||
workflow = dsl.get("workflow")
|
||||
workflow_graph = workflow.get("graph") if isinstance(workflow, dict) else None
|
||||
if isinstance(workflow_graph, dict) and isinstance(workflow_graph.get("nodes"), list):
|
||||
nodes.extend(node for node in workflow_graph["nodes"] if isinstance(node, dict))
|
||||
return nodes
|
||||
|
||||
def _agent_tool_configs(self, data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
configs = data.get("tools")
|
||||
if isinstance(configs, list):
|
||||
return [config for config in configs if isinstance(config, dict)]
|
||||
agent_parameters = data.get("agent_parameters")
|
||||
if not isinstance(agent_parameters, dict):
|
||||
return []
|
||||
tools_parameter = agent_parameters.get("tools")
|
||||
if not isinstance(tools_parameter, dict):
|
||||
return []
|
||||
value = tools_parameter.get("value", [])
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [config for config in value if isinstance(config, dict)]
|
||||
|
||||
def _should_preserve_source_app_id(self, options: ImportOptions) -> bool:
|
||||
return options.id_strategy == IdStrategy.PRESERVE_ID
|
||||
|
||||
def _find_existing_app(self, app_id: str | None, tenant_id: str) -> App | None:
|
||||
if not self._is_uuid_string(app_id):
|
||||
return None
|
||||
return db.session.scalar(sa.select(App).where(App.id == app_id, App.tenant_id == tenant_id))
|
||||
|
||||
def _create_or_reuse_app_api_token(self, app_id: str, tenant_id: str) -> None:
|
||||
existing = db.session.scalar(
|
||||
sa.select(ApiToken).where(
|
||||
ApiToken.type == ApiTokenType.APP,
|
||||
ApiToken.app_id == app_id,
|
||||
ApiToken.tenant_id == tenant_id,
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return
|
||||
api_token = ApiToken()
|
||||
api_token.app_id = app_id
|
||||
api_token.tenant_id = tenant_id
|
||||
api_token.token = ApiToken.generate_api_key("app", 24)
|
||||
api_token.type = ApiTokenType.APP
|
||||
db.session.add(api_token)
|
||||
db.session.commit()
|
||||
|
||||
def _import_api_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
source_provider_ids_by_name: dict[str, set[str]],
|
||||
) -> None:
|
||||
for tool_data in package.tools:
|
||||
provider_name = self._required_string(tool_data, "provider_name", "api_tool")
|
||||
schema = self._required_string(tool_data, "schema", "api_tool")
|
||||
existing = db.session.scalar(
|
||||
sa.select(ApiToolProvider).where(
|
||||
ApiToolProvider.tenant_id == target.tenant_id,
|
||||
ApiToolProvider.name == provider_name,
|
||||
)
|
||||
)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"API tool already exists and conflict_strategy=fail: {provider_name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.API_TOOL,
|
||||
provider_name,
|
||||
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, "skipped"))
|
||||
continue
|
||||
|
||||
schema_info = ApiToolManageService.parser_api_schema(schema=schema)
|
||||
schema_type = cast(ApiProviderSchemaType, schema_info["schema_type"])
|
||||
credentials = (
|
||||
cast(dict[str, Any], tool_data.get("credentials"))
|
||||
if isinstance(tool_data.get("credentials"), dict)
|
||||
else {}
|
||||
)
|
||||
credentials = credentials or {"auth_type": "none"}
|
||||
raw_icon = tool_data.get("icon")
|
||||
icon = (
|
||||
cast(dict[str, Any], raw_icon)
|
||||
if isinstance(raw_icon, dict)
|
||||
else {"content": "tool", "background": "#FEF7C3"}
|
||||
)
|
||||
raw_labels = tool_data.get("labels")
|
||||
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
|
||||
if existing is not None:
|
||||
ApiToolManageService.update_api_tool_provider(
|
||||
user_id=target.operator_id,
|
||||
tenant_id=target.tenant_id,
|
||||
provider_name=provider_name,
|
||||
original_provider=existing.name,
|
||||
_schema_type=schema_type,
|
||||
schema=schema,
|
||||
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
|
||||
credentials=credentials,
|
||||
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
|
||||
labels=labels,
|
||||
icon=icon,
|
||||
)
|
||||
status = "updated"
|
||||
else:
|
||||
ApiToolManageService.create_api_tool_provider(
|
||||
user_id=target.operator_id,
|
||||
tenant_id=target.tenant_id,
|
||||
provider_name=provider_name,
|
||||
schema_type=schema_type,
|
||||
schema=schema,
|
||||
privacy_policy=self._optional_string(tool_data.get("privacy_policy")) or "",
|
||||
credentials=credentials,
|
||||
custom_disclaimer=self._optional_string(tool_data.get("custom_disclaimer")) or "",
|
||||
labels=labels,
|
||||
icon=icon,
|
||||
)
|
||||
status = "created"
|
||||
target_provider = self._find_api_tool_provider(target.tenant_id, provider_name)
|
||||
if target_provider is not None:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.API_TOOL,
|
||||
provider_name,
|
||||
self._api_tool_source_ids(provider_name, tool_data, source_provider_ids_by_name),
|
||||
target_provider.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.API_TOOL, provider_name, provider_name, status))
|
||||
|
||||
def _find_api_tool_provider(self, tenant_id: str, provider_name: str) -> ApiToolProvider | None:
|
||||
return db.session.scalar(
|
||||
sa.select(ApiToolProvider).where(
|
||||
ApiToolProvider.tenant_id == tenant_id,
|
||||
ApiToolProvider.name == provider_name,
|
||||
)
|
||||
)
|
||||
|
||||
def _api_tool_source_ids(
|
||||
self,
|
||||
provider_name: str,
|
||||
tool_data: dict[str, Any],
|
||||
source_provider_ids_by_name: dict[str, set[str]],
|
||||
) -> set[str]:
|
||||
source_ids = set(source_provider_ids_by_name.get(provider_name, set()))
|
||||
source_id = self._optional_string(tool_data.get("id"))
|
||||
if source_id:
|
||||
source_ids.add(source_id)
|
||||
return source_ids
|
||||
|
||||
def _record_id_mappings(
|
||||
self,
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
resource_type: ResourceType,
|
||||
name: str | None,
|
||||
source_ids: Iterable[str],
|
||||
target_id: str,
|
||||
) -> None:
|
||||
for source_id in source_ids:
|
||||
id_mapping[source_id] = target_id
|
||||
id_mapping_details[:] = [item for item in id_mapping_details if item.source_id != source_id]
|
||||
id_mapping_details.append(ResourceIdMapping(resource_type, name, source_id, target_id))
|
||||
|
||||
def _import_workflow_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
report_items: list[ResourceReportItem],
|
||||
) -> None:
|
||||
if not package.workflow_tools:
|
||||
return
|
||||
account = db.session.get(Account, target.operator_id)
|
||||
if account is None:
|
||||
raise MigrationDataError(f"Operator account not found: {target.operator_id}")
|
||||
for workflow_tool_data in package.workflow_tools:
|
||||
app_id = self._optional_string(workflow_tool_data.get("app_id"))
|
||||
resolved_app_id = id_mapping.get(app_id or "", app_id)
|
||||
if not resolved_app_id or self._find_existing_app(resolved_app_id, target.tenant_id) is None:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
|
||||
self._optional_string(workflow_tool_data.get("name")),
|
||||
"unresolved",
|
||||
"Referenced workflow app was not found in the target tenant; workflow tool was skipped.",
|
||||
)
|
||||
)
|
||||
continue
|
||||
try:
|
||||
self._ensure_workflow_app_is_published(target, account, resolved_app_id)
|
||||
except Exception as exc:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
str(workflow_tool_data.get("id", workflow_tool_data.get("name", ""))),
|
||||
self._optional_string(workflow_tool_data.get("name")),
|
||||
"unresolved",
|
||||
f"Referenced workflow app could not be published: {exc}",
|
||||
)
|
||||
)
|
||||
continue
|
||||
workflow_tool_id = self._optional_string(workflow_tool_data.get("id"))
|
||||
tool_name = self._required_string(workflow_tool_data, "name", "workflow_tool")
|
||||
lookup_workflow_tool_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
|
||||
existing = self._find_existing_workflow_tool(
|
||||
target.tenant_id, lookup_workflow_tool_id, tool_name, resolved_app_id
|
||||
)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"Workflow tool already exists and conflict_strategy=fail: {tool_name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if workflow_tool_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
tool_name,
|
||||
{workflow_tool_id},
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, existing.id, tool_name, "skipped"))
|
||||
continue
|
||||
raw_icon = workflow_tool_data.get("icon")
|
||||
icon = (
|
||||
cast(dict[str, Any], raw_icon)
|
||||
if isinstance(raw_icon, dict)
|
||||
else {"content": "🤖", "background": "#FFEAD5"}
|
||||
)
|
||||
raw_parameters = workflow_tool_data.get("parameters")
|
||||
parameters = [
|
||||
parameter
|
||||
if isinstance(parameter, WorkflowToolParameterConfiguration)
|
||||
else WorkflowToolParameterConfiguration(**parameter)
|
||||
for parameter in (raw_parameters if isinstance(raw_parameters, list) else [])
|
||||
if isinstance(parameter, dict | WorkflowToolParameterConfiguration)
|
||||
]
|
||||
raw_labels = workflow_tool_data.get("labels")
|
||||
labels = [str(label) for label in raw_labels] if isinstance(raw_labels, list) else []
|
||||
label = self._optional_string(workflow_tool_data.get("label")) or tool_name
|
||||
description = self._optional_string(workflow_tool_data.get("description")) or ""
|
||||
privacy_policy = self._optional_string(workflow_tool_data.get("privacy_policy")) or ""
|
||||
if existing is not None:
|
||||
WorkflowToolManageService.update_workflow_tool(
|
||||
user_id=account.id,
|
||||
tenant_id=target.tenant_id,
|
||||
workflow_tool_id=existing.id,
|
||||
name=tool_name,
|
||||
label=label,
|
||||
icon=icon,
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
privacy_policy=privacy_policy,
|
||||
labels=labels,
|
||||
)
|
||||
status = "updated"
|
||||
identifier = existing.id
|
||||
else:
|
||||
import_id = workflow_tool_id if options.id_strategy == IdStrategy.PRESERVE_ID else ""
|
||||
WorkflowToolManageService.create_workflow_tool(
|
||||
user_id=account.id,
|
||||
tenant_id=target.tenant_id,
|
||||
workflow_app_id=resolved_app_id,
|
||||
name=tool_name,
|
||||
label=label,
|
||||
icon=icon,
|
||||
description=description,
|
||||
parameters=parameters,
|
||||
privacy_policy=privacy_policy,
|
||||
labels=labels,
|
||||
import_id=import_id or "",
|
||||
)
|
||||
status = "created"
|
||||
target_provider = self._find_existing_workflow_tool(
|
||||
target.tenant_id, import_id or None, tool_name, resolved_app_id
|
||||
)
|
||||
if target_provider is None:
|
||||
raise MigrationDataError(f"Workflow tool was not created: {tool_name}")
|
||||
identifier = target_provider.id
|
||||
if workflow_tool_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.WORKFLOW_TOOL,
|
||||
tool_name,
|
||||
{workflow_tool_id},
|
||||
identifier,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.WORKFLOW_TOOL, identifier, tool_name, status))
|
||||
|
||||
def _ensure_workflow_app_is_published(self, target: ImportTarget, account: Account, app_id: str) -> None:
|
||||
app = self._find_existing_app(app_id, target.tenant_id)
|
||||
if app is None:
|
||||
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
|
||||
if app.workflow_id:
|
||||
return
|
||||
workflow_service = WorkflowService()
|
||||
with sessionmaker(db.engine).begin() as session:
|
||||
app_in_session = session.get(App, app_id)
|
||||
account_in_session = session.get(Account, account.id)
|
||||
if app_in_session is None:
|
||||
raise MigrationDataError(f"Referenced workflow app was not found in target tenant: {app_id}")
|
||||
if account_in_session is None:
|
||||
raise MigrationDataError(f"Operator account not found: {account.id}")
|
||||
workflow = workflow_service.publish_workflow(
|
||||
session=session,
|
||||
app_model=app_in_session,
|
||||
account=account_in_session,
|
||||
marked_name="Migration import",
|
||||
marked_comment="Published automatically for workflow tool import.",
|
||||
)
|
||||
app_in_session.workflow_id = workflow.id
|
||||
app_in_session.updated_by = account.id
|
||||
app_in_session.updated_at = naive_utc_now()
|
||||
|
||||
def _import_mcp_tools(
|
||||
self,
|
||||
package: MigrationPackage,
|
||||
target: ImportTarget,
|
||||
options: ImportOptions,
|
||||
report_items: list[ResourceReportItem],
|
||||
id_mapping: dict[str, str],
|
||||
id_mapping_details: list[ResourceIdMapping],
|
||||
) -> None:
|
||||
for mcp_data in package.mcp_tools:
|
||||
name = self._required_string(mcp_data, "name", "mcp_tool")
|
||||
server_identifier = self._required_string(mcp_data, "server_identifier", "mcp_tool")
|
||||
provider_id = self._optional_string(mcp_data.get("id"))
|
||||
lookup_provider_id = provider_id if options.id_strategy == IdStrategy.PRESERVE_ID else None
|
||||
existing = self._find_existing_mcp_tool(target.tenant_id, lookup_provider_id, server_identifier)
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.FAIL:
|
||||
raise MigrationDataError(f"MCP tool already exists and conflict_strategy=fail: {name}")
|
||||
if existing is not None and options.conflict_strategy == ConflictStrategy.SKIP:
|
||||
if provider_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.MCP_TOOL,
|
||||
name,
|
||||
{provider_id},
|
||||
existing.id,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, existing.id, name, "skipped"))
|
||||
continue
|
||||
|
||||
service = MCPToolManageService(session=cast(Session, db.session))
|
||||
configuration = MCPConfiguration.model_validate(mcp_data.get("configuration") or {})
|
||||
authentication = (
|
||||
MCPAuthentication.model_validate(mcp_data["authentication"]) if mcp_data.get("authentication") else None
|
||||
)
|
||||
if existing is not None:
|
||||
service.update_provider(
|
||||
tenant_id=target.tenant_id,
|
||||
provider_id=existing.id,
|
||||
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
|
||||
name=name,
|
||||
icon=self._optional_string(mcp_data.get("icon")) or "",
|
||||
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
|
||||
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
|
||||
server_identifier=server_identifier,
|
||||
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
|
||||
configuration=configuration,
|
||||
authentication=authentication,
|
||||
)
|
||||
db.session.commit()
|
||||
status = "updated"
|
||||
identifier = existing.id
|
||||
provider = existing
|
||||
else:
|
||||
service.create_provider(
|
||||
tenant_id=target.tenant_id,
|
||||
user_id=target.operator_id,
|
||||
server_url=self._required_string(mcp_data, "server_url", "mcp_tool"),
|
||||
name=name,
|
||||
icon=self._optional_string(mcp_data.get("icon")) or "",
|
||||
icon_type=self._optional_string(mcp_data.get("icon_type")) or "emoji",
|
||||
icon_background=self._optional_string(mcp_data.get("icon_background")) or "",
|
||||
server_identifier=server_identifier,
|
||||
headers=mcp_data.get("headers") if isinstance(mcp_data.get("headers"), dict) else {},
|
||||
configuration=configuration,
|
||||
authentication=authentication,
|
||||
)
|
||||
created_provider = self._find_existing_mcp_tool(target.tenant_id, lookup_provider_id, server_identifier)
|
||||
if created_provider is None:
|
||||
raise MigrationDataError(f"MCP provider was not created: {name}")
|
||||
status = "created"
|
||||
provider = created_provider
|
||||
identifier = provider.id
|
||||
self._restore_mcp_provider_tools(provider, mcp_data)
|
||||
db.session.commit()
|
||||
if provider_id:
|
||||
self._record_id_mappings(
|
||||
id_mapping,
|
||||
id_mapping_details,
|
||||
ResourceType.MCP_TOOL,
|
||||
name,
|
||||
{provider_id},
|
||||
identifier,
|
||||
)
|
||||
report_items.append(ResourceReportItem(ResourceType.MCP_TOOL, identifier, name, status))
|
||||
|
||||
def _restore_mcp_provider_tools(self, provider: MCPToolProvider, mcp_data: dict[str, object]) -> None:
|
||||
tools = mcp_data.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return
|
||||
provider.tools = json.dumps(tools)
|
||||
provider.authed = True
|
||||
|
||||
def _find_existing_mcp_tool(
|
||||
self, tenant_id: str, provider_id: str | None, server_identifier: str
|
||||
) -> MCPToolProvider | None:
|
||||
predicates = [MCPToolProvider.server_identifier == server_identifier]
|
||||
if self._is_uuid_string(provider_id):
|
||||
predicates.append(MCPToolProvider.id == provider_id)
|
||||
return db.session.scalar(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
|
||||
)
|
||||
|
||||
def _is_uuid_string(self, value: str | None) -> bool:
|
||||
if not value:
|
||||
return False
|
||||
try:
|
||||
UUID(value)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _find_existing_workflow_tool(
|
||||
self, tenant_id: str, workflow_tool_id: str | None, tool_name: str, app_id: str
|
||||
) -> WorkflowToolProvider | None:
|
||||
predicates = [WorkflowToolProvider.name == tool_name, WorkflowToolProvider.app_id == app_id]
|
||||
if self._is_uuid_string(workflow_tool_id):
|
||||
predicates.append(WorkflowToolProvider.id == workflow_tool_id)
|
||||
return db.session.scalar(
|
||||
sa.select(WorkflowToolProvider)
|
||||
.where(WorkflowToolProvider.tenant_id == tenant_id, or_(*predicates))
|
||||
.limit(1)
|
||||
)
|
||||
|
||||
def _preflight_dependency_only_mcp(
|
||||
self, package: MigrationPackage, target: ImportTarget, report_items: list[ResourceReportItem]
|
||||
) -> None:
|
||||
for dependency in package.dependencies:
|
||||
if dependency.get("kind") != DependencyKind.MCP_TOOL.value:
|
||||
continue
|
||||
provider_id = str(dependency.get("provider_id", dependency.get("id", "")))
|
||||
provider_name = self._optional_string(dependency.get("provider_name") or dependency.get("name"))
|
||||
existing = self._find_dependency_only_mcp_provider(target.tenant_id, provider_id, provider_name)
|
||||
report_name = f"mcp_tool {provider_name or getattr(existing, 'name', None) or provider_id}"
|
||||
if existing is not None:
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
provider_id,
|
||||
report_name,
|
||||
"available",
|
||||
"MCP provider exists in target tenant.",
|
||||
)
|
||||
)
|
||||
continue
|
||||
reference_summary = self._dependency_only_mcp_reference_summary(package, provider_id, provider_name)
|
||||
message = "missing in target tenant"
|
||||
if reference_summary:
|
||||
message = f"{message}; referenced by {reference_summary}"
|
||||
message = f"{message}; configure it manually before running the workflow."
|
||||
report_items.append(
|
||||
ResourceReportItem(
|
||||
ResourceType.DEPENDENCY,
|
||||
provider_id,
|
||||
report_name,
|
||||
"skipped",
|
||||
message,
|
||||
)
|
||||
)
|
||||
|
||||
def _find_dependency_only_mcp_provider(
|
||||
self, tenant_id: str, provider_id: str, provider_name: str | None
|
||||
) -> MCPToolProvider | None:
|
||||
predicates = [MCPToolProvider.server_identifier == provider_id]
|
||||
if self._is_uuid_string(provider_id):
|
||||
predicates.append(MCPToolProvider.id == provider_id)
|
||||
return db.session.scalar(
|
||||
sa.select(MCPToolProvider).where(MCPToolProvider.tenant_id == tenant_id, or_(*predicates)).limit(1)
|
||||
)
|
||||
|
||||
def _dependency_only_mcp_reference_summary(
|
||||
self, package: MigrationPackage, provider_id: str, provider_name: str | None
|
||||
) -> str:
|
||||
references = self._dependency_only_mcp_references(package, provider_id, provider_name)
|
||||
return "; ".join(references)
|
||||
|
||||
def _dependency_only_mcp_references(
|
||||
self, package: MigrationPackage, provider_id: str, provider_name: str | None
|
||||
) -> list[str]:
|
||||
references: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for workflow_data in package.workflows:
|
||||
workflow_name = self._optional_string(workflow_data.get("name"))
|
||||
workflow_id = self._optional_string(workflow_data.get("id"))
|
||||
workflow_label = workflow_name or workflow_id or "unknown workflow"
|
||||
dsl_content = self._optional_string(workflow_data.get("dsl"))
|
||||
if not dsl_content:
|
||||
continue
|
||||
parsed = yaml.safe_load(dsl_content) if dsl_content else {}
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
for node in self._workflow_nodes(parsed):
|
||||
data = node.get("data") if isinstance(node, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
for tool_config in [data, *self._agent_tool_configs(data)]:
|
||||
if not self._is_mcp_dependency_reference(tool_config, provider_id, provider_name):
|
||||
continue
|
||||
tool_label = self._mcp_tool_reference_label(node, tool_config)
|
||||
reference = f"{workflow_label} / {tool_label}"
|
||||
if reference not in seen:
|
||||
seen.add(reference)
|
||||
references.append(reference)
|
||||
return references
|
||||
|
||||
def _is_mcp_dependency_reference(
|
||||
self, tool_config: dict[str, Any], provider_id: str, provider_name: str | None
|
||||
) -> bool:
|
||||
provider_type = str(tool_config.get("provider_type") or tool_config.get("type") or "").lower()
|
||||
if provider_type != "mcp":
|
||||
return False
|
||||
config_provider_id = self._optional_string(
|
||||
tool_config.get("provider_id") or tool_config.get("provider_name") or tool_config.get("provider")
|
||||
)
|
||||
if config_provider_id == provider_id:
|
||||
return True
|
||||
return bool(provider_name and config_provider_id == provider_name)
|
||||
|
||||
def _mcp_tool_reference_label(self, node: dict[str, Any], tool_config: dict[str, Any]) -> str:
|
||||
for key in ("tool_name", "tool", "name"):
|
||||
value = self._optional_string(tool_config.get(key))
|
||||
if value:
|
||||
return value
|
||||
node_id = self._optional_string(node.get("id"))
|
||||
return node_id or "unknown tool"
|
||||
|
||||
def _required_string(self, value: dict[str, object], field_name: str, resource_name: str) -> str:
|
||||
field_value = value.get(field_name)
|
||||
if not isinstance(field_value, str) or not field_value:
|
||||
raise MigrationDataError(f"Missing required {resource_name} field: {field_name}")
|
||||
return field_value
|
||||
|
||||
def _optional_string(self, value: object) -> str | None:
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return None
|
||||
@@ -0,0 +1,71 @@
|
||||
"""JSON persistence for versioned cross-environment migration packages.
|
||||
|
||||
The package service validates file shape and serializes only structured package
|
||||
entities. It does not perform CLI rendering or database access, keeping it safe
|
||||
to reuse from Click adapters, tests, and future import/export services.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from services.data_migration.entities import (
|
||||
ImportOptions,
|
||||
MigrationDataError,
|
||||
MigrationMetadata,
|
||||
MigrationPackage,
|
||||
SourceTenant,
|
||||
TargetTenantSelector,
|
||||
)
|
||||
|
||||
PACKAGE_VERSION = "1"
|
||||
|
||||
|
||||
class MigrationPackageService:
|
||||
def load_package(self, path: str | Path) -> MigrationPackage:
|
||||
package_path = Path(path)
|
||||
with package_path.open(encoding="utf-8") as file:
|
||||
raw = json.load(file)
|
||||
if not isinstance(raw, dict):
|
||||
raise MigrationDataError("Migration package JSON must be an object.")
|
||||
package = MigrationPackage.from_mapping(raw)
|
||||
if package.metadata.version != PACKAGE_VERSION:
|
||||
raise MigrationDataError(f"Unsupported migration package version: {package.metadata.version}")
|
||||
return package
|
||||
|
||||
def save_package(self, package: MigrationPackage, path: str | Path, *, overwrite: bool) -> None:
|
||||
package_path = Path(path)
|
||||
if package_path.exists() and not overwrite:
|
||||
raise MigrationDataError(f"Output file already exists: {package_path}")
|
||||
package_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with package_path.open("w", encoding="utf-8") as file:
|
||||
json.dump(self.to_mapping(package), file, ensure_ascii=False, indent=2)
|
||||
file.write("\n")
|
||||
|
||||
def build_empty_package(
|
||||
self,
|
||||
*,
|
||||
source_tenant_id: str,
|
||||
source_tenant_name: str,
|
||||
include_secrets: bool,
|
||||
import_options: ImportOptions | None = None,
|
||||
target_tenant: TargetTenantSelector | None = None,
|
||||
) -> MigrationPackage:
|
||||
return MigrationPackage(
|
||||
metadata=MigrationMetadata(
|
||||
version=PACKAGE_VERSION,
|
||||
source_scope="single",
|
||||
source_tenants=[SourceTenant(id=source_tenant_id, name=source_tenant_name)],
|
||||
target_tenant=target_tenant,
|
||||
created_at=datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
include_secrets=include_secrets,
|
||||
import_options=import_options or ImportOptions(),
|
||||
)
|
||||
)
|
||||
|
||||
def to_mapping(self, package: MigrationPackage) -> dict[str, Any]:
|
||||
return asdict(package)
|
||||
@@ -0,0 +1,76 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
|
||||
from services.data_migration.entities import ReportContext, ResourceIdMapping, ResourceReportItem
|
||||
|
||||
|
||||
class MigrationReportService:
|
||||
"""Render structured migration resource results into CLI-friendly summary lines."""
|
||||
|
||||
def render(self, items: list[ResourceReportItem], *, context: ReportContext | None = None) -> list[str]:
|
||||
counts = Counter((item.resource_type.value, item.status) for item in items)
|
||||
lines = self._render_context(context)
|
||||
lines.extend(
|
||||
[f"{resource_type} {status}: {count}" for (resource_type, status), count in sorted(counts.items())]
|
||||
)
|
||||
actionable_items = [
|
||||
item for item in items if item.status in {"dependency-only", "skipped", "unresolved"} and item.message
|
||||
]
|
||||
for item in actionable_items:
|
||||
lines.append(self._render_actionable_detail(item))
|
||||
return lines
|
||||
|
||||
def _render_context(self, context: ReportContext | None) -> list[str]:
|
||||
if context is None:
|
||||
return []
|
||||
lines: list[str] = []
|
||||
if context.output_path:
|
||||
lines.append(f"output: {context.output_path}")
|
||||
if context.source_scope:
|
||||
lines.append(f"source scope: {context.source_scope}")
|
||||
if context.selected_app_count is not None:
|
||||
lines.append(f"selected apps: {context.selected_app_count}")
|
||||
if context.include_secrets is not None:
|
||||
lines.append(f"include secrets: {str(context.include_secrets).lower()}")
|
||||
if context.target_tenant:
|
||||
lines.append(f"target tenant: {context.target_tenant}")
|
||||
if context.operator_email:
|
||||
lines.append(f"operator: {context.operator_email}")
|
||||
if context.app_api_tokens_created or context.app_api_tokens_reused:
|
||||
lines.append(
|
||||
f"app api tokens: {context.app_api_tokens_created} created, {context.app_api_tokens_reused} reused"
|
||||
)
|
||||
if context.id_mappings:
|
||||
lines.append(f"resource references resolved: {len(context.id_mappings)}")
|
||||
if context.id_mapping_details:
|
||||
lines.extend(
|
||||
self._render_id_mapping_detail(item)
|
||||
for item in sorted(
|
||||
context.id_mapping_details,
|
||||
key=lambda item: (item.resource_type.value, item.name or "", item.source_id),
|
||||
)
|
||||
)
|
||||
else:
|
||||
lines.extend(
|
||||
f"- {source_id} -> {target_id}" for source_id, target_id in sorted(context.id_mappings.items())
|
||||
)
|
||||
elif context.id_mapping_count:
|
||||
lines.append(f"resource references resolved: {context.id_mapping_count}")
|
||||
return lines
|
||||
|
||||
def _render_id_mapping_detail(self, item: ResourceIdMapping) -> str:
|
||||
label = item.resource_type.value
|
||||
if item.name:
|
||||
label = f"{label} {item.name}"
|
||||
return f"- {label}: {item.source_id} -> {item.target_id}"
|
||||
|
||||
def _render_actionable_detail(self, item: ResourceReportItem) -> str:
|
||||
if item.resource_type.value == "dependency" and item.name and self._has_dependency_type_prefix(item.name):
|
||||
if item.identifier and item.identifier not in item.name:
|
||||
return f"dependency {item.name}: {item.identifier}: {item.message}"
|
||||
return f"dependency {item.name}: {item.message}"
|
||||
return f"{item.resource_type.value} {item.identifier}: {item.message}"
|
||||
|
||||
def _has_dependency_type_prefix(self, name: str) -> bool:
|
||||
return name.startswith(("workflow ", "api_tool ", "workflow_tool ", "mcp_tool ", "builtin_or_plugin_tool "))
|
||||
@@ -41,6 +41,7 @@ class WorkflowToolManageService:
|
||||
parameters: list[WorkflowToolParameterConfiguration],
|
||||
privacy_policy: str = "",
|
||||
labels: list[str] | None = None,
|
||||
import_id: str = "",
|
||||
):
|
||||
# check if the name is unique
|
||||
existing_workflow_tool_provider: WorkflowToolProvider | None = None
|
||||
@@ -92,7 +93,8 @@ class WorkflowToolManageService:
|
||||
privacy_policy=privacy_policy,
|
||||
version=workflow.version,
|
||||
)
|
||||
|
||||
if import_id:
|
||||
workflow_tool_provider.id = import_id
|
||||
try:
|
||||
WorkflowToolProviderController.from_db(workflow_tool_provider)
|
||||
except Exception as e:
|
||||
|
||||
+298
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import HTTPException
|
||||
|
||||
import services
|
||||
from controllers.console.auth.error import MemberNotInTenantError
|
||||
from controllers.console.workspace import members as members_module
|
||||
from controllers.console.workspace.members import MemberCancelInviteApi, MemberUpdateRoleApi, OwnerTransfer
|
||||
from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole, TenantStatus
|
||||
|
||||
|
||||
def unwrap(func):
|
||||
while hasattr(func, "__wrapped__"):
|
||||
func = func.__wrapped__
|
||||
return func
|
||||
|
||||
|
||||
class WorkspaceMembersIntegrationFactory:
|
||||
@staticmethod
|
||||
def create_tenant(db_session_with_containers) -> Tenant:
|
||||
tenant = Tenant(name=f"Tenant {uuid4()}", plan="basic", status=TenantStatus.NORMAL)
|
||||
db_session_with_containers.add(tenant)
|
||||
db_session_with_containers.commit()
|
||||
return tenant
|
||||
|
||||
@staticmethod
|
||||
def create_account(
|
||||
db_session_with_containers,
|
||||
*,
|
||||
email_prefix: str,
|
||||
tenant: Tenant | None = None,
|
||||
role: TenantAccountRole = TenantAccountRole.NORMAL,
|
||||
current: bool = False,
|
||||
) -> Account:
|
||||
account = Account(
|
||||
name=f"Account {uuid4()}",
|
||||
email=f"{email_prefix}-{uuid4()}@example.com",
|
||||
password="hashed-password",
|
||||
password_salt="salt",
|
||||
interface_language="en-US",
|
||||
timezone="UTC",
|
||||
)
|
||||
db_session_with_containers.add(account)
|
||||
db_session_with_containers.commit()
|
||||
|
||||
if tenant is not None:
|
||||
join = TenantAccountJoin(
|
||||
tenant_id=tenant.id,
|
||||
account_id=account.id,
|
||||
role=role,
|
||||
current=current,
|
||||
)
|
||||
db_session_with_containers.add(join)
|
||||
db_session_with_containers.commit()
|
||||
account.current_tenant = tenant
|
||||
return account
|
||||
|
||||
@staticmethod
|
||||
def create_owner_workspace(db_session_with_containers) -> tuple[Tenant, Account]:
|
||||
tenant = WorkspaceMembersIntegrationFactory.create_tenant(db_session_with_containers)
|
||||
owner = WorkspaceMembersIntegrationFactory.create_account(
|
||||
db_session_with_containers,
|
||||
email_prefix="owner",
|
||||
tenant=tenant,
|
||||
role=TenantAccountRole.OWNER,
|
||||
current=True,
|
||||
)
|
||||
return tenant, owner
|
||||
|
||||
@staticmethod
|
||||
def create_owner_transfer_token(account: Account) -> str:
|
||||
_, token = members_module.AccountService.generate_owner_transfer_token(
|
||||
account.email,
|
||||
account=account,
|
||||
code="123456",
|
||||
additional_data={},
|
||||
)
|
||||
return token
|
||||
|
||||
@staticmethod
|
||||
def get_join(db_session_with_containers, *, tenant: Tenant, account: Account) -> TenantAccountJoin:
|
||||
tenant_id = tenant.id
|
||||
account_id = account.id
|
||||
db_session_with_containers.expire_all()
|
||||
join = (
|
||||
db_session_with_containers.query(TenantAccountJoin)
|
||||
.filter_by(tenant_id=tenant_id, account_id=account_id)
|
||||
.one()
|
||||
)
|
||||
return join
|
||||
|
||||
|
||||
class TestMemberCancelInviteApiWithContainers:
|
||||
def test_cancel_success(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(db_session_with_containers, email_prefix="member")
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/"),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
patch.object(members_module.TenantService, "remove_member_from_tenant") as mock_remove_member,
|
||||
):
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 200
|
||||
assert result["result"] == "success"
|
||||
mock_remove_member.assert_called_once()
|
||||
called_tenant, called_member, called_current_user = mock_remove_member.call_args.args
|
||||
assert called_tenant.id == tenant.id
|
||||
assert called_member.id == member.id
|
||||
assert called_current_user.id == current_user.id
|
||||
|
||||
def test_cancel_not_found(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/"),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
method(api, str(uuid4()))
|
||||
|
||||
def test_cancel_cannot_operate_self(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(db_session_with_containers, email_prefix="member")
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/"),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
patch.object(
|
||||
members_module.TenantService,
|
||||
"remove_member_from_tenant",
|
||||
side_effect=services.errors.account.CannotOperateSelfError("x"),
|
||||
),
|
||||
):
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 400
|
||||
assert result["code"] == "cannot-operate-self"
|
||||
|
||||
def test_cancel_no_permission(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(db_session_with_containers, email_prefix="member")
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/"),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
patch.object(
|
||||
members_module.TenantService,
|
||||
"remove_member_from_tenant",
|
||||
side_effect=services.errors.account.NoPermissionError("x"),
|
||||
),
|
||||
):
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 403
|
||||
assert result["code"] == "forbidden"
|
||||
|
||||
def test_cancel_member_not_in_tenant(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(db_session_with_containers, email_prefix="member")
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/"),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
patch.object(
|
||||
members_module.TenantService,
|
||||
"remove_member_from_tenant",
|
||||
side_effect=services.errors.account.MemberNotInTenantError(),
|
||||
),
|
||||
):
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 404
|
||||
assert result["code"] == "member-not-found"
|
||||
|
||||
|
||||
class TestMemberUpdateRoleApiWithContainers:
|
||||
def test_update_success(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberUpdateRoleApi()
|
||||
method = unwrap(api.put)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(
|
||||
db_session_with_containers,
|
||||
email_prefix="member",
|
||||
tenant=tenant,
|
||||
role=TenantAccountRole.EDITOR,
|
||||
)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/", json={"role": "normal"}),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
):
|
||||
result = method(api, member.id)
|
||||
|
||||
if isinstance(result, tuple):
|
||||
result = result[0]
|
||||
|
||||
assert result["result"] == "success"
|
||||
assert (
|
||||
factory.get_join(db_session_with_containers, tenant=tenant, account=member).role == TenantAccountRole.NORMAL
|
||||
)
|
||||
|
||||
def test_update_member_not_found(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = MemberUpdateRoleApi()
|
||||
method = unwrap(api.put)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/", json={"role": "normal"}),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
method(api, str(uuid4()))
|
||||
|
||||
|
||||
class TestOwnerTransferApiWithContainers:
|
||||
def test_member_not_in_tenant(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = OwnerTransfer()
|
||||
method = unwrap(api.post)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(db_session_with_containers, email_prefix="member")
|
||||
token = factory.create_owner_transfer_token(current_user)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/", json={"token": token}),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
):
|
||||
with pytest.raises(MemberNotInTenantError):
|
||||
method(api, member.id)
|
||||
|
||||
def test_member_not_found(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = OwnerTransfer()
|
||||
method = unwrap(api.post)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
token = factory.create_owner_transfer_token(current_user)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/", json={"token": token}),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
method(api, str(uuid4()))
|
||||
|
||||
def test_transfer_success(self, flask_app_with_containers, db_session_with_containers):
|
||||
api = OwnerTransfer()
|
||||
method = unwrap(api.post)
|
||||
factory = WorkspaceMembersIntegrationFactory
|
||||
tenant, current_user = factory.create_owner_workspace(db_session_with_containers)
|
||||
member = factory.create_account(
|
||||
db_session_with_containers,
|
||||
email_prefix="member",
|
||||
tenant=tenant,
|
||||
role=TenantAccountRole.NORMAL,
|
||||
)
|
||||
token = factory.create_owner_transfer_token(current_user)
|
||||
|
||||
with (
|
||||
flask_app_with_containers.test_request_context("/", json={"token": token}),
|
||||
patch.object(members_module, "current_account_with_tenant", return_value=(current_user, tenant.id)),
|
||||
patch.object(members_module.AccountService, "send_new_owner_transfer_notify_email") as mock_new_owner_email,
|
||||
patch.object(members_module.AccountService, "send_old_owner_transfer_notify_email") as mock_old_owner_email,
|
||||
):
|
||||
result = method(api, member.id)
|
||||
|
||||
assert result["result"] == "success"
|
||||
assert (
|
||||
factory.get_join(db_session_with_containers, tenant=tenant, account=member).role == TenantAccountRole.OWNER
|
||||
)
|
||||
assert (
|
||||
factory.get_join(db_session_with_containers, tenant=tenant, account=current_user).role
|
||||
== TenantAccountRole.ADMIN
|
||||
)
|
||||
mock_new_owner_email.assert_called_once()
|
||||
mock_old_owner_email.assert_called_once()
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
Integration tests for delete_account_task.
|
||||
|
||||
These tests keep billing and email dispatch mocked, but exercise the account
|
||||
lookup through the real Testcontainers PostgreSQL session factory instead of a
|
||||
patched session_factory mock.
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models.account import Account
|
||||
from tasks.delete_account_task import delete_account_task
|
||||
|
||||
|
||||
def _create_account(db_session: Session, *, email: str = "user@example.com") -> Account:
|
||||
account = Account(
|
||||
name=f"account-{uuid4()}",
|
||||
email=email,
|
||||
)
|
||||
db_session.add(account)
|
||||
db_session.commit()
|
||||
return account
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_external_dependencies(mocker):
|
||||
billing_service = mocker.patch("tasks.delete_account_task.BillingService")
|
||||
mail_task = mocker.patch("tasks.delete_account_task.send_deletion_success_task")
|
||||
return billing_service, mail_task
|
||||
|
||||
|
||||
def test_billing_enabled_account_exists_calls_billing_and_sends_email(
|
||||
db_session_with_containers: Session, mock_external_dependencies, mocker
|
||||
) -> None:
|
||||
billing_service, mail_task = mock_external_dependencies
|
||||
account = _create_account(db_session_with_containers, email="a@b.com")
|
||||
mocker.patch("tasks.delete_account_task.dify_config.BILLING_ENABLED", True)
|
||||
|
||||
delete_account_task(account.id)
|
||||
|
||||
billing_service.delete_account.assert_called_once_with(account.id)
|
||||
mail_task.delay.assert_called_once_with(account.email)
|
||||
|
||||
|
||||
def test_billing_disabled_account_exists_sends_email_only(
|
||||
db_session_with_containers: Session, mock_external_dependencies, mocker
|
||||
) -> None:
|
||||
billing_service, mail_task = mock_external_dependencies
|
||||
account = _create_account(db_session_with_containers, email="x@y.com")
|
||||
mocker.patch("tasks.delete_account_task.dify_config.BILLING_ENABLED", False)
|
||||
|
||||
delete_account_task(account.id)
|
||||
|
||||
billing_service.delete_account.assert_not_called()
|
||||
mail_task.delay.assert_called_once_with(account.email)
|
||||
|
||||
|
||||
def test_billing_enabled_account_not_found_calls_billing_no_email(mock_external_dependencies, mocker, caplog) -> None:
|
||||
billing_service, mail_task = mock_external_dependencies
|
||||
account_id = str(uuid4())
|
||||
mocker.patch("tasks.delete_account_task.dify_config.BILLING_ENABLED", True)
|
||||
|
||||
delete_account_task(account_id)
|
||||
|
||||
billing_service.delete_account.assert_called_once_with(account_id)
|
||||
mail_task.delay.assert_not_called()
|
||||
assert any("not found" in record.getMessage().lower() for record in caplog.records)
|
||||
|
||||
|
||||
def test_billing_delete_raises_propagates_and_no_email(
|
||||
db_session_with_containers: Session, mock_external_dependencies, mocker
|
||||
) -> None:
|
||||
billing_service, mail_task = mock_external_dependencies
|
||||
account = _create_account(db_session_with_containers, email="err@example.com")
|
||||
billing_service.delete_account.side_effect = RuntimeError("billing down")
|
||||
mocker.patch("tasks.delete_account_task.dify_config.BILLING_ENABLED", True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="billing down"):
|
||||
delete_account_task(account.id)
|
||||
|
||||
mail_task.delay.assert_not_called()
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
"""Integration test for the cleanup request against the real agenton compositor.
|
||||
|
||||
The bug fixed by A+D was invisible to unit tests that use ``FakeAgentBackendRunClient``
|
||||
because the fake client never runs agenton's ``_validate_session_snapshot``. This
|
||||
test plugs a cleanup request through the real ``Compositor`` (with the same
|
||||
providers the agent backend wires in production) so that the snapshot-vs-
|
||||
composition name-order check would fail loudly if the cleanup builder ever
|
||||
regressed back to the empty-composition shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from agenton.compositor import Compositor, CompositorSessionSnapshot, LayerProvider
|
||||
from agenton.compositor.schemas import LayerSessionSnapshot
|
||||
from agenton.layers.base import LifecycleState
|
||||
from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID
|
||||
from agenton_collections.layers.plain.basic import PromptLayer
|
||||
from agenton_collections.layers.pydantic_ai import PYDANTIC_AI_HISTORY_LAYER_TYPE_ID, PydanticAIHistoryLayer
|
||||
|
||||
from clients.agent_backend import AgentBackendRunRequestBuilder, CleanupLayerSpec
|
||||
|
||||
|
||||
def test_cleanup_request_passes_agenton_snapshot_validation():
|
||||
"""The cleanup request's composition layer names must match the (filtered)
|
||||
snapshot's layer names exactly — agenton's compositor enforces this and
|
||||
the agent backend rejects mismatches as ``run_failed`` asynchronously,
|
||||
which is the trap A/D fixed."""
|
||||
# Persisted (non-plugin) layer specs — these are what cleanup will replay.
|
||||
# We exclude the dify.execution_context layer from this integration check
|
||||
# because its real provider needs a plugin-daemon HTTP client; the cleanup
|
||||
# validation we are exercising is the snapshot-vs-composition name check,
|
||||
# which is purely structural and does not depend on which non-plugin layer
|
||||
# types appear.
|
||||
persisted_specs = [
|
||||
CleanupLayerSpec(
|
||||
name="workflow_node_job_prompt",
|
||||
type=PLAIN_PROMPT_LAYER_TYPE_ID,
|
||||
config={"prefix": "Do the cleanup."},
|
||||
),
|
||||
CleanupLayerSpec(name="history", type=PYDANTIC_AI_HISTORY_LAYER_TYPE_ID),
|
||||
]
|
||||
# Saved snapshot still carries the LLM layer entry — cleanup's
|
||||
# ``_filter_snapshot_to_specs`` must drop it so names match.
|
||||
full_snapshot = CompositorSessionSnapshot(
|
||||
layers=[
|
||||
LayerSessionSnapshot(
|
||||
name="workflow_node_job_prompt",
|
||||
lifecycle_state=LifecycleState.SUSPENDED,
|
||||
runtime_state={},
|
||||
),
|
||||
LayerSessionSnapshot(
|
||||
name="history",
|
||||
lifecycle_state=LifecycleState.SUSPENDED,
|
||||
runtime_state={"messages": []},
|
||||
),
|
||||
LayerSessionSnapshot(
|
||||
name="llm",
|
||||
lifecycle_state=LifecycleState.SUSPENDED,
|
||||
runtime_state={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
cleanup_request = AgentBackendRunRequestBuilder().build_cleanup_request(
|
||||
session_snapshot=full_snapshot,
|
||||
composition_layer_specs=persisted_specs,
|
||||
)
|
||||
|
||||
# Drive the real agenton compositor through ``from_config`` + ``_create_run``
|
||||
# the same way the agent backend's RunScheduler does. ``_create_run`` is the
|
||||
# private path that calls ``_validate_session_snapshot``; we use it directly
|
||||
# to keep the test synchronous (no async ``enter()`` lifecycle needed —
|
||||
# validation is the only thing under test).
|
||||
config = {
|
||||
"schema_version": 1,
|
||||
"layers": [
|
||||
{"name": layer.name, "type": layer.type, "deps": dict(layer.deps), "metadata": dict(layer.metadata)}
|
||||
for layer in cleanup_request.composition.layers
|
||||
],
|
||||
}
|
||||
compositor = Compositor.from_config(
|
||||
config,
|
||||
providers=[
|
||||
LayerProvider.from_layer_type(PromptLayer),
|
||||
LayerProvider.from_layer_type(PydanticAIHistoryLayer),
|
||||
],
|
||||
)
|
||||
|
||||
layer_configs = {layer.name: layer.config for layer in cleanup_request.composition.layers}
|
||||
# This is the call that would raise ``ValueError`` if the cleanup snapshot
|
||||
# and composition disagreed on layer names — the exact failure mode the
|
||||
# original ``layers=[]`` cleanup hit.
|
||||
run = compositor._create_run( # type: ignore[reportPrivateUsage]
|
||||
configs=cast(dict[str, object], layer_configs),
|
||||
session_snapshot=cleanup_request.session_snapshot,
|
||||
)
|
||||
assert list(run.slots.keys()) == ["workflow_node_job_prompt", "history"]
|
||||
|
||||
|
||||
def test_cleanup_request_with_mismatched_specs_would_be_rejected_by_agenton():
|
||||
"""Regression sentinel: if a future refactor stops filtering the snapshot,
|
||||
agenton would reject the request — and that rejection is what the runtime
|
||||
fix is preventing. We confirm the validator does fail when given the
|
||||
pre-fix shape so the previous test's success is not a coincidence."""
|
||||
snapshot_with_extra = CompositorSessionSnapshot(
|
||||
layers=[
|
||||
LayerSessionSnapshot(
|
||||
name="history",
|
||||
lifecycle_state=LifecycleState.SUSPENDED,
|
||||
runtime_state={},
|
||||
),
|
||||
LayerSessionSnapshot(
|
||||
name="llm", # extra layer not in composition
|
||||
lifecycle_state=LifecycleState.SUSPENDED,
|
||||
runtime_state={},
|
||||
),
|
||||
]
|
||||
)
|
||||
compositor = Compositor.from_config(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"layers": [{"name": "history", "type": PYDANTIC_AI_HISTORY_LAYER_TYPE_ID, "deps": {}, "metadata": {}}],
|
||||
},
|
||||
providers=[LayerProvider.from_layer_type(PydanticAIHistoryLayer)],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="layer names must match"):
|
||||
compositor._create_run( # type: ignore[reportPrivateUsage]
|
||||
configs={},
|
||||
session_snapshot=snapshot_with_extra,
|
||||
)
|
||||
@@ -63,3 +63,25 @@ def test_fake_client_cancel_run_returns_cancelled_status():
|
||||
|
||||
assert cancelled.run_id == "fake-run-1"
|
||||
assert cancelled.status == "cancelled"
|
||||
|
||||
|
||||
def test_fake_client_paused_scenario_returns_paused_status_and_event():
|
||||
"""The paused scenario exists for HITL-style flows; both ``wait_run`` and
|
||||
the event stream must report the pause so consumers can branch on it."""
|
||||
client = FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario.PAUSED)
|
||||
|
||||
status = client.wait_run("fake-run-1")
|
||||
events = list(client.stream_events("fake-run-1"))
|
||||
|
||||
assert status.status == "paused"
|
||||
assert status.error is None
|
||||
assert events[-1].type == "run_paused"
|
||||
assert events[-1].data.reason == "human_input_required"
|
||||
|
||||
|
||||
def test_fake_client_success_wait_run_returns_succeeded_status():
|
||||
"""Covers the default SUCCESS branch of ``wait_run`` directly."""
|
||||
status = FakeAgentBackendRunClient().wait_run("fake-run-1")
|
||||
|
||||
assert status.status == "succeeded"
|
||||
assert status.error is None
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
from agenton.compositor.schemas import LayerSessionSnapshot
|
||||
from agenton.layers import ExitIntent
|
||||
from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID
|
||||
from agenton.layers.base import LifecycleState
|
||||
from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID, PromptLayerConfig
|
||||
from agenton_collections.layers.pydantic_ai import PYDANTIC_AI_HISTORY_LAYER_TYPE_ID
|
||||
from dify_agent.layers.dify_plugin import (
|
||||
DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
|
||||
DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
|
||||
DifyPluginLLMLayerConfig,
|
||||
DifyPluginToolConfig,
|
||||
DifyPluginToolsLayerConfig,
|
||||
)
|
||||
from dify_agent.layers.execution_context import DIFY_EXECUTION_CONTEXT_LAYER_TYPE_ID, DifyExecutionContextLayerConfig
|
||||
from dify_agent.layers.output import DIFY_OUTPUT_LAYER_TYPE_ID
|
||||
from dify_agent.protocol import (
|
||||
DIFY_AGENT_HISTORY_LAYER_ID,
|
||||
DIFY_AGENT_MODEL_LAYER_ID,
|
||||
DIFY_AGENT_OUTPUT_LAYER_ID,
|
||||
CreateRunRequest,
|
||||
@@ -26,6 +34,7 @@ from clients.agent_backend import (
|
||||
AgentBackendOutputConfig,
|
||||
AgentBackendRunRequestBuilder,
|
||||
AgentBackendWorkflowNodeRunInput,
|
||||
CleanupLayerSpec,
|
||||
redact_for_agent_backend_log,
|
||||
)
|
||||
|
||||
@@ -71,10 +80,11 @@ def test_request_builder_outputs_dify_agent_create_run_request():
|
||||
WORKFLOW_NODE_JOB_PROMPT_LAYER_ID,
|
||||
WORKFLOW_USER_PROMPT_LAYER_ID,
|
||||
DIFY_EXECUTION_CONTEXT_LAYER_ID,
|
||||
DIFY_AGENT_HISTORY_LAYER_ID,
|
||||
DIFY_AGENT_MODEL_LAYER_ID,
|
||||
DIFY_AGENT_OUTPUT_LAYER_ID,
|
||||
]
|
||||
assert request.on_exit.default is ExitIntent.DELETE
|
||||
assert request.on_exit.default is ExitIntent.SUSPEND
|
||||
assert request.idempotency_key == "workflow-run-1:node-execution-1"
|
||||
assert request.metadata == {"workflow_id": "workflow-1", "node_id": "node-1"}
|
||||
|
||||
@@ -99,9 +109,10 @@ def test_request_builder_sets_model_and_output_layer_contract_ids():
|
||||
layers = {layer.name: layer for layer in request.composition.layers}
|
||||
|
||||
assert layers[DIFY_EXECUTION_CONTEXT_LAYER_ID].type == DIFY_EXECUTION_CONTEXT_LAYER_TYPE_ID
|
||||
assert layers[DIFY_EXECUTION_CONTEXT_LAYER_ID].config.user_id == "user-1"
|
||||
assert cast(DifyExecutionContextLayerConfig, layers[DIFY_EXECUTION_CONTEXT_LAYER_ID].config).user_id == "user-1"
|
||||
assert layers[DIFY_AGENT_HISTORY_LAYER_ID].type == PYDANTIC_AI_HISTORY_LAYER_TYPE_ID
|
||||
assert layers[DIFY_AGENT_MODEL_LAYER_ID].type == DIFY_PLUGIN_LLM_LAYER_TYPE_ID
|
||||
assert layers[DIFY_AGENT_MODEL_LAYER_ID].config.plugin_id == "langgenius/openai"
|
||||
assert cast(DifyPluginLLMLayerConfig, layers[DIFY_AGENT_MODEL_LAYER_ID].config).plugin_id == "langgenius/openai"
|
||||
assert layers[DIFY_AGENT_MODEL_LAYER_ID].deps == {"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID}
|
||||
assert layers[DIFY_AGENT_OUTPUT_LAYER_ID].type == DIFY_OUTPUT_LAYER_TYPE_ID
|
||||
|
||||
@@ -130,16 +141,92 @@ def test_request_builder_adds_dify_plugin_tools_layer_when_configured():
|
||||
|
||||
assert layers[DIFY_PLUGIN_TOOLS_LAYER_ID].type == DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID
|
||||
assert layers[DIFY_PLUGIN_TOOLS_LAYER_ID].deps == {"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID}
|
||||
assert layers[DIFY_PLUGIN_TOOLS_LAYER_ID].config.tools[0].tool_name == "current_time"
|
||||
tools_config = cast(DifyPluginToolsLayerConfig, layers[DIFY_PLUGIN_TOOLS_LAYER_ID].config)
|
||||
assert tools_config.tools[0].tool_name == "current_time"
|
||||
|
||||
|
||||
def test_request_builder_can_suspend_on_exit_for_resume_or_babysit_paths():
|
||||
def test_request_builder_can_delete_on_exit_for_cleanup_paths():
|
||||
run_input = _run_input()
|
||||
run_input.suspend_on_exit = True
|
||||
run_input.suspend_on_exit = False
|
||||
|
||||
request = AgentBackendRunRequestBuilder().build_for_workflow_node(run_input)
|
||||
|
||||
assert request.on_exit.default is ExitIntent.SUSPEND
|
||||
assert request.on_exit.default is ExitIntent.DELETE
|
||||
|
||||
|
||||
def test_request_builder_builds_cleanup_request_replays_persisted_layer_specs():
|
||||
"""The cleanup request must replay the persisted (non-plugin) layer specs
|
||||
and filter the snapshot to match so the agenton compositor's
|
||||
snapshot-vs-composition name-order validator passes."""
|
||||
session_snapshot = CompositorSessionSnapshot(
|
||||
layers=[
|
||||
LayerSessionSnapshot(name="history", lifecycle_state=LifecycleState.SUSPENDED, runtime_state={"k": 1}),
|
||||
LayerSessionSnapshot(name="llm", lifecycle_state=LifecycleState.SUSPENDED, runtime_state={}),
|
||||
]
|
||||
)
|
||||
specs = [CleanupLayerSpec(name="history", type="pydantic_ai.history")]
|
||||
|
||||
request = AgentBackendRunRequestBuilder().build_cleanup_request(
|
||||
session_snapshot=session_snapshot,
|
||||
composition_layer_specs=specs,
|
||||
idempotency_key="run-1:node-1:binding-1:agent-session-cleanup",
|
||||
metadata={"workflow_run_id": "run-1"},
|
||||
)
|
||||
|
||||
assert [layer.name for layer in request.composition.layers] == ["history"]
|
||||
assert request.session_snapshot is not None
|
||||
assert [layer.name for layer in request.session_snapshot.layers] == ["history"]
|
||||
assert request.on_exit.default is ExitIntent.DELETE
|
||||
assert request.idempotency_key == "run-1:node-1:binding-1:agent-session-cleanup"
|
||||
assert request.metadata["agent_backend_lifecycle"] == "session_cleanup"
|
||||
|
||||
|
||||
def test_request_builder_rejects_empty_composition_layer_specs():
|
||||
"""Empty specs would put us back in the original ``layers=[]`` trap that
|
||||
fails on agenton's snapshot-vs-composition validation."""
|
||||
with pytest.raises(ValueError, match="composition_layer_specs"):
|
||||
AgentBackendRunRequestBuilder().build_cleanup_request(
|
||||
session_snapshot=CompositorSessionSnapshot(layers=[]),
|
||||
composition_layer_specs=[],
|
||||
)
|
||||
|
||||
|
||||
def test_extract_cleanup_layer_specs_drops_plugin_layers_keeps_configs():
|
||||
from dify_agent.protocol import RunComposition, RunLayerSpec
|
||||
|
||||
from clients.agent_backend import extract_cleanup_layer_specs
|
||||
|
||||
composition = RunComposition(
|
||||
layers=[
|
||||
RunLayerSpec(
|
||||
name="agent_soul_prompt",
|
||||
type="plain.prompt",
|
||||
config=PromptLayerConfig(prefix="hello"),
|
||||
),
|
||||
RunLayerSpec(
|
||||
name="llm",
|
||||
type="dify.plugin.llm",
|
||||
config=None, # protocol allows None; the redacted config is what matters
|
||||
),
|
||||
RunLayerSpec(
|
||||
name="tools",
|
||||
type="dify.plugin.tools",
|
||||
),
|
||||
RunLayerSpec(
|
||||
name="history",
|
||||
type="pydantic_ai.history",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
specs = extract_cleanup_layer_specs(composition)
|
||||
|
||||
assert [spec.name for spec in specs] == ["agent_soul_prompt", "history"]
|
||||
# Non-plugin configs are dumped as JSON-compatible dicts so the persisted
|
||||
# row can be replayed without holding live pydantic instances.
|
||||
soul_config = specs[0].config
|
||||
assert isinstance(soul_config, dict)
|
||||
assert soul_config.get("prefix") == "hello"
|
||||
|
||||
|
||||
def test_request_builder_rejects_blank_prompts():
|
||||
@@ -159,6 +246,6 @@ def test_request_builder_rejects_blank_prompts():
|
||||
def test_redact_for_agent_backend_log_hides_credentials():
|
||||
request = AgentBackendRunRequestBuilder().build_for_workflow_node(_run_input())
|
||||
|
||||
redacted = redact_for_agent_backend_log(request)
|
||||
redacted = cast(dict[str, Any], redact_for_agent_backend_log(request))
|
||||
|
||||
assert redacted["composition"]["layers"][4]["config"]["credentials"] == "[REDACTED]"
|
||||
assert redacted["composition"]["layers"][5]["config"]["credentials"] == "[REDACTED]"
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import json
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from commands.data_migration import (
|
||||
ID_STRATEGY_CHOICES,
|
||||
export_migration_data,
|
||||
export_migration_data_template,
|
||||
import_migration_data,
|
||||
)
|
||||
|
||||
|
||||
def test_export_command_requires_input_and_output():
|
||||
result = CliRunner().invoke(export_migration_data, [])
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert export_migration_data.name == "export-app-migration"
|
||||
assert "--input" in result.output
|
||||
assert "--output" in result.output
|
||||
|
||||
|
||||
def test_import_command_requires_input_and_target_tenant_or_package_metadata():
|
||||
result = CliRunner().invoke(import_migration_data, [])
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert import_migration_data.name == "import-app-migration"
|
||||
assert "--input" in result.output
|
||||
|
||||
|
||||
def test_import_command_does_not_expose_unimplemented_map_id_strategy():
|
||||
assert ID_STRATEGY_CHOICES == ["preserve-id", "generate-new-id"]
|
||||
|
||||
|
||||
def test_export_template_command_prints_scripted_json_template():
|
||||
result = CliRunner().invoke(export_migration_data_template, [])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert export_migration_data_template.name == "app-migration-template"
|
||||
template = json.loads(result.output)
|
||||
assert template == {
|
||||
"source_tenant": {"mode": "single", "id": "", "name": "admin's Workspace"},
|
||||
"apps": {"modes": ["workflow", "advanced-chat"], "ids": [], "all": True},
|
||||
"include_referenced_tools": True,
|
||||
"additional_tools": {"api_tools": [], "workflow_tools": [], "mcp_tools": []},
|
||||
"include_secrets": False,
|
||||
"import_options": {
|
||||
"create_app_api_token_on_import": False,
|
||||
"id_strategy": "preserve-id",
|
||||
"conflict_strategy": "fail",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_export_template_command_writes_output_file(tmp_path):
|
||||
output_file = tmp_path / "export-template.json"
|
||||
|
||||
result = CliRunner().invoke(export_migration_data_template, ["--output", str(output_file)])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert f"Output written to {output_file}" in result.output
|
||||
assert json.loads(output_file.read_text())["apps"]["all"] is True
|
||||
|
||||
|
||||
def test_export_template_command_requires_overwrite_for_existing_output(tmp_path):
|
||||
output_file = tmp_path / "export-template.json"
|
||||
output_file.write_text("{}")
|
||||
|
||||
result = CliRunner().invoke(export_migration_data_template, ["--output", str(output_file)])
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "already exists" in result.output
|
||||
@@ -0,0 +1,384 @@
|
||||
from commands.data_migration import (
|
||||
CONFLICT_STRATEGY_CHOICES,
|
||||
ID_STRATEGY_CHOICES,
|
||||
_confirm_wizard_summary,
|
||||
_print_auto_tools,
|
||||
_print_final_tool_selection,
|
||||
_print_wizard_step,
|
||||
_prompt_additional_tools,
|
||||
_prompt_output_file,
|
||||
_prompt_tool_category,
|
||||
_resolve_mcp_tool_names,
|
||||
migration_data_wizard,
|
||||
parse_index_selection,
|
||||
)
|
||||
|
||||
|
||||
def test_parse_index_selection_supports_all():
|
||||
assert parse_index_selection("all", ["a", "b", "c"]) == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_wizard_command_uses_app_migration_name():
|
||||
assert migration_data_wizard.name == "app-migration-wizard"
|
||||
|
||||
|
||||
def test_parse_index_selection_supports_comma_indexes():
|
||||
assert parse_index_selection("1, 3", ["a", "b", "c"]) == ["a", "c"]
|
||||
|
||||
|
||||
def test_print_wizard_step_adds_separator(monkeypatch):
|
||||
output_lines = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
|
||||
_print_wizard_step("App Selection")
|
||||
|
||||
assert output_lines == ["", "==== App Selection ===="]
|
||||
|
||||
|
||||
def test_conflict_strategy_choices_exclude_replace():
|
||||
assert CONFLICT_STRATEGY_CHOICES == ["fail", "skip", "update"]
|
||||
|
||||
|
||||
def test_prompt_app_ids_explains_comma_selection_and_default(monkeypatch):
|
||||
from commands.data_migration import _prompt_app_ids
|
||||
|
||||
prompts = []
|
||||
output_lines = []
|
||||
apps = [
|
||||
type("App", (), {"id": "app-1", "name": "embedded", "mode": "workflow"})(),
|
||||
type("App", (), {"id": "app-2", "name": "main", "mode": "advanced-chat"})(),
|
||||
]
|
||||
|
||||
def capture_prompt(text, **kwargs):
|
||||
prompts.append((text, kwargs))
|
||||
return "1,2"
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", capture_prompt)
|
||||
|
||||
assert _prompt_app_ids(apps) == ["app-1", "app-2"]
|
||||
assert prompts == [("Select apps by number, comma-separated numbers, or all", {"default": "all"})]
|
||||
assert "Currently supported app types: workflow and chatflow." in output_lines
|
||||
|
||||
|
||||
def test_prompt_tool_category_marks_auto_discovered_tools(monkeypatch):
|
||||
output_lines = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", lambda *args, **kwargs: "")
|
||||
|
||||
selected = _prompt_tool_category(
|
||||
"Custom API tools",
|
||||
[("weather", "weather", "tool-id"), ("calendar", "calendar", "calendar-id")],
|
||||
auto_tools={"weather": "tool-id"},
|
||||
)
|
||||
|
||||
assert selected == []
|
||||
assert "1. [auto] weather (tool-id)" in output_lines
|
||||
assert "2. [ ] calendar (calendar-id)" in output_lines
|
||||
assert output_lines[:2] == ["", "==== Custom API tools ===="]
|
||||
|
||||
|
||||
def test_prompt_tool_category_explains_comma_selection_and_default(monkeypatch):
|
||||
prompts = []
|
||||
|
||||
def capture_prompt(text, **kwargs):
|
||||
prompts.append((text, kwargs))
|
||||
return ""
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", capture_prompt)
|
||||
|
||||
selected = _prompt_tool_category(
|
||||
"Custom API tools",
|
||||
[("weather", "weather", "tool-id")],
|
||||
auto_tools={},
|
||||
)
|
||||
|
||||
assert selected == []
|
||||
assert prompts == [
|
||||
(
|
||||
"Select custom api tools by number, comma-separated numbers, all, or empty",
|
||||
{"default": "", "show_default": "empty"},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_prompt_output_file_shows_default(monkeypatch):
|
||||
prompts = []
|
||||
|
||||
def capture_prompt(text, **kwargs):
|
||||
prompts.append((text, kwargs))
|
||||
return "migration-data.json"
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", capture_prompt)
|
||||
|
||||
assert _prompt_output_file() == ("migration-data.json", False)
|
||||
assert prompts[0][0] == "Output path"
|
||||
assert prompts[0][1]["show_default"] is True
|
||||
|
||||
|
||||
def test_prompt_tool_category_marks_auto_by_detail_and_supports_multi_select(monkeypatch):
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", lambda *_args, **_kwargs: None)
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", lambda *args, **kwargs: "1,2")
|
||||
|
||||
selected = _prompt_tool_category(
|
||||
"Workflow tools",
|
||||
[("tool-1", "embedded", "app-1"), ("tool-2", "other", "app-2")],
|
||||
auto_tools={"embedded": "app-1"},
|
||||
)
|
||||
|
||||
assert selected == ["tool-1", "tool-2"]
|
||||
|
||||
|
||||
def test_prompt_tool_category_marks_auto_by_value():
|
||||
output_lines = []
|
||||
|
||||
from commands import data_migration
|
||||
|
||||
original_echo = data_migration.click.echo
|
||||
original_prompt = data_migration.click.prompt
|
||||
data_migration.click.echo = output_lines.append
|
||||
data_migration.click.prompt = lambda *args, **kwargs: ""
|
||||
try:
|
||||
_prompt_tool_category(
|
||||
"Workflow tools",
|
||||
[("tool-1", "embedded_workflow_as_tool", "tool-1")],
|
||||
auto_tools={"embedded_workflow_as_tool": "tool-1"},
|
||||
)
|
||||
finally:
|
||||
data_migration.click.echo = original_echo
|
||||
data_migration.click.prompt = original_prompt
|
||||
|
||||
assert "1. [auto] embedded_workflow_as_tool (tool-1)" in output_lines
|
||||
|
||||
|
||||
def test_print_auto_tools_lists_each_category(monkeypatch):
|
||||
output_lines = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
|
||||
_print_auto_tools(
|
||||
{
|
||||
"api_tools": {"weather": "3bac3aa9-dd87-4351-9459-a7099137b028"},
|
||||
"workflow_tools": {"embedded_workflow_as_tool": "e6024578-41b7-4fb5-a81f-9201358e5835"},
|
||||
"mcp_tools": {},
|
||||
}
|
||||
)
|
||||
|
||||
assert "Automatically discovered tools:" in output_lines
|
||||
assert "Custom API tools" in output_lines
|
||||
assert "- weather: 3bac3aa9-dd87-4351-9459-a7099137b028" in output_lines
|
||||
assert "Workflow tools" in output_lines
|
||||
assert "- embedded_workflow_as_tool: e6024578-41b7-4fb5-a81f-9201358e5835" in output_lines
|
||||
assert "MCP tools" in output_lines
|
||||
assert "- none" in output_lines
|
||||
|
||||
|
||||
def test_resolve_mcp_tool_names_does_not_compare_non_uuid_identifier_to_uuid_id(monkeypatch):
|
||||
statements = []
|
||||
|
||||
def capture_scalar(statement):
|
||||
statements.append(str(statement))
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.db.session.scalar", capture_scalar)
|
||||
|
||||
assert _resolve_mcp_tool_names("49a99e46-bc2c-4885-91fa-47615f6192b5", {"my-test-mcp": "my-test-mcp"}) == {
|
||||
"my-test-mcp": "my-test-mcp"
|
||||
}
|
||||
assert "tool_mcp_providers.id =" not in statements[0]
|
||||
assert "tool_mcp_providers.server_identifier =" in statements[0]
|
||||
|
||||
|
||||
def test_prompt_additional_tools_prints_final_selection_when_skipped(monkeypatch):
|
||||
output_lines = []
|
||||
confirm_prompts = []
|
||||
|
||||
def capture_confirm(prompt, **kwargs):
|
||||
confirm_prompts.append((prompt, kwargs))
|
||||
return False
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.confirm", capture_confirm)
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
|
||||
selected = _prompt_additional_tools(
|
||||
"tenant-id",
|
||||
{
|
||||
"api_tools": {"weather": "3bac3aa9-dd87-4351-9459-a7099137b028"},
|
||||
"workflow_tools": {},
|
||||
"mcp_tools": {},
|
||||
},
|
||||
)
|
||||
|
||||
assert selected == {"api_tools": [], "workflow_tools": [], "mcp_tools": []}
|
||||
assert confirm_prompts == [
|
||||
("Export additional tools manually? [y/n, default: n]", {"default": False, "show_default": False})
|
||||
]
|
||||
assert "Final tools to export:" in output_lines
|
||||
assert "- [auto] weather: 3bac3aa9-dd87-4351-9459-a7099137b028" in output_lines
|
||||
|
||||
|
||||
def test_final_tool_selection_deduplicates_manual_tool_already_auto(monkeypatch):
|
||||
output_lines = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
|
||||
_print_final_tool_selection(
|
||||
{
|
||||
"api_tools": {},
|
||||
"workflow_tools": {"embedded_workflow_as_tool": "e6024578-41b7-4fb5-a81f-9201358e5835"},
|
||||
"mcp_tools": {},
|
||||
},
|
||||
{
|
||||
"api_tools": [],
|
||||
"workflow_tools": ["e6024578-41b7-4fb5-a81f-9201358e5835"],
|
||||
"mcp_tools": [],
|
||||
},
|
||||
{"e6024578-41b7-4fb5-a81f-9201358e5835": "embedded_workflow_as_tool: e6024578"},
|
||||
)
|
||||
|
||||
assert "- [auto] embedded_workflow_as_tool: e6024578-41b7-4fb5-a81f-9201358e5835" in output_lines
|
||||
assert not any(line.startswith("- [manual]") for line in output_lines)
|
||||
|
||||
|
||||
def test_prompt_output_file_rejects_yes_no_typo(monkeypatch):
|
||||
import click
|
||||
import pytest
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", lambda *args, **kwargs: "y")
|
||||
|
||||
with pytest.raises(click.ClickException, match="Output path must be a file path"):
|
||||
_prompt_output_file()
|
||||
|
||||
|
||||
def test_confirm_wizard_summary_shows_conflict_strategy(monkeypatch):
|
||||
output_lines = []
|
||||
confirm_prompts = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
monkeypatch.setattr(
|
||||
"commands.data_migration.click.confirm",
|
||||
lambda prompt, **kwargs: confirm_prompts.append((prompt, kwargs)) or True,
|
||||
)
|
||||
|
||||
_confirm_wizard_summary(
|
||||
tenant_name="admin's Workspace",
|
||||
app_names=["main_chatflow"],
|
||||
auto_tools={"api_tools": {}, "workflow_tools": {}, "mcp_tools": {}},
|
||||
additional_tools={"api_tools": [], "workflow_tools": [], "mcp_tools": []},
|
||||
manual_labels={},
|
||||
include_referenced_tools=True,
|
||||
include_secrets=False,
|
||||
create_tokens=True,
|
||||
id_strategy="preserve-id",
|
||||
conflict_strategy="fail",
|
||||
output_file="migration-data.json",
|
||||
)
|
||||
|
||||
assert "id strategy: preserve-id" in output_lines
|
||||
assert "conflict strategy: fail" in output_lines
|
||||
assert confirm_prompts == [("Write migration package? [y/n, default: y]", {"default": True, "show_default": False})]
|
||||
|
||||
|
||||
def test_confirm_wizard_summary_shows_final_deduplicated_tool_selection(monkeypatch):
|
||||
output_lines = []
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
monkeypatch.setattr("commands.data_migration.click.confirm", lambda *args, **kwargs: True)
|
||||
|
||||
_confirm_wizard_summary(
|
||||
tenant_name="admin's Workspace",
|
||||
app_names=["main_chatflow"],
|
||||
auto_tools={
|
||||
"api_tools": {"weather": "weather-id"},
|
||||
"workflow_tools": {"embedded_workflow_as_tool": "workflow-tool-id"},
|
||||
"mcp_tools": {},
|
||||
},
|
||||
additional_tools={
|
||||
"api_tools": ["weather-id", "calendar"],
|
||||
"workflow_tools": [],
|
||||
"mcp_tools": ["mcp-id"],
|
||||
},
|
||||
manual_labels={
|
||||
"calendar": "calendar: calendar-id",
|
||||
"mcp-id": "my-test-mcp: mcp-id",
|
||||
},
|
||||
include_referenced_tools=True,
|
||||
include_secrets=False,
|
||||
create_tokens=False,
|
||||
id_strategy="preserve-id",
|
||||
conflict_strategy="update",
|
||||
output_file="migration-data.json",
|
||||
)
|
||||
|
||||
assert "Final tools to export:" in output_lines
|
||||
assert "Custom API tools" in output_lines
|
||||
assert "- [auto] weather: weather-id" in output_lines
|
||||
assert "- [manual] calendar: calendar-id" in output_lines
|
||||
assert "Workflow tools" in output_lines
|
||||
assert "- [auto] embedded_workflow_as_tool: workflow-tool-id" in output_lines
|
||||
assert "MCP tools" in output_lines
|
||||
assert "- [manual] my-test-mcp: mcp-id" in output_lines
|
||||
assert not any(line.startswith("additional api tools:") for line in output_lines)
|
||||
assert not any(line.startswith("additional workflow tools:") for line in output_lines)
|
||||
assert not any(line.startswith("additional mcp tools:") for line in output_lines)
|
||||
assert "- [manual] weather-id" not in output_lines
|
||||
|
||||
|
||||
def test_import_options_prompts_explain_secrets_reuse_and_conflicts(monkeypatch):
|
||||
from commands.data_migration import _prompt_import_options
|
||||
|
||||
output_lines = []
|
||||
confirm_prompts = []
|
||||
prompt_calls = []
|
||||
|
||||
def capture_confirm(prompt, **kwargs):
|
||||
confirm_prompts.append((prompt, kwargs))
|
||||
return False
|
||||
|
||||
def capture_prompt(prompt, **kwargs):
|
||||
prompt_calls.append((prompt, kwargs))
|
||||
return kwargs["default"]
|
||||
|
||||
monkeypatch.setattr("commands.data_migration.click.echo", output_lines.append)
|
||||
monkeypatch.setattr("commands.data_migration.click.confirm", capture_confirm)
|
||||
monkeypatch.setattr("commands.data_migration.click.prompt", capture_prompt)
|
||||
|
||||
include_secrets, create_tokens, id_strategy, conflict_strategy = _prompt_import_options()
|
||||
|
||||
assert include_secrets is False
|
||||
assert create_tokens is False
|
||||
assert id_strategy == "preserve-id"
|
||||
assert conflict_strategy == "update"
|
||||
assert "Secrets include workflow/app DSL secret values, custom API tool credentials," in output_lines
|
||||
assert "-- Secrets --" in output_lines
|
||||
assert "If you choose no, credentials are omitted or masked," in output_lines
|
||||
assert "-- App API Tokens --" in output_lines
|
||||
assert "When enabled, import will create an app API token if the imported app has none," in output_lines
|
||||
assert "or reuse an existing app API token if one already exists." in output_lines
|
||||
assert "-- ID Strategy --" in output_lines
|
||||
assert "ID strategy controls whether imported app and tool IDs preserve source IDs" in output_lines
|
||||
assert "or use target-generated IDs." in output_lines
|
||||
assert "preserve-id: keep source IDs where the target service supports it." in output_lines
|
||||
assert (
|
||||
"generate-new-id: let the target environment generate new IDs and rewrite references via mapping."
|
||||
in output_lines
|
||||
)
|
||||
assert "-- Conflict Strategy --" in output_lines
|
||||
assert "Conflict strategy controls what import does when a target resource already exists." in output_lines
|
||||
assert "fail: stop at the first conflict; previously committed resources are not rolled back." in output_lines
|
||||
assert "skip: keep the existing target resource and skip importing that resource." in output_lines
|
||||
assert "update: update the existing target resource in place." in output_lines
|
||||
assert confirm_prompts == [
|
||||
("Include secrets in output JSON? [y/n, default: n]", {"default": False, "show_default": False}),
|
||||
("Create or reuse app API tokens during import? [y/n, default: n]", {"default": False, "show_default": False}),
|
||||
]
|
||||
assert prompt_calls[0][0] == "Import ID strategy. Enter one of: preserve-id, generate-new-id"
|
||||
assert prompt_calls[0][1]["default"] == "preserve-id"
|
||||
assert prompt_calls[0][1]["show_default"] is True
|
||||
assert prompt_calls[0][1]["type"].choices == ID_STRATEGY_CHOICES
|
||||
assert prompt_calls[1][0] == "Import conflict strategy. Enter one of: fail, skip, update"
|
||||
assert prompt_calls[1][1]["default"] == "update"
|
||||
assert prompt_calls[1][1]["show_default"] is True
|
||||
assert prompt_calls[1][1]["type"].choices == CONFLICT_STRATEGY_CHOICES
|
||||
@@ -34,7 +34,6 @@ def test_rule_generate_success(app, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
api = generator_module.RuleGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
monkeypatch.setattr(generator_module.LLMGenerator, "generate_rule_config", lambda **_kwargs: {"rules": []})
|
||||
|
||||
with app.test_request_context(
|
||||
@@ -42,7 +41,7 @@ def test_rule_generate_success(app, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
method="POST",
|
||||
json={"instruction": "do it", "model_config": _model_config_payload()},
|
||||
):
|
||||
response = method()
|
||||
response = method("t1")
|
||||
|
||||
assert response == {"rules": []}
|
||||
|
||||
@@ -51,8 +50,6 @@ def test_rule_code_generate_maps_token_error(app, monkeypatch: pytest.MonkeyPatc
|
||||
api = generator_module.RuleCodeGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
def _raise(*_args, **_kwargs):
|
||||
raise ProviderTokenNotInitError("missing token")
|
||||
|
||||
@@ -64,15 +61,13 @@ def test_rule_code_generate_maps_token_error(app, monkeypatch: pytest.MonkeyPatc
|
||||
json={"instruction": "do it", "model_config": _model_config_payload()},
|
||||
):
|
||||
with pytest.raises(ProviderNotInitializeError):
|
||||
method()
|
||||
method("t1")
|
||||
|
||||
|
||||
def test_instruction_generate_app_not_found(app, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
monkeypatch.setattr(generator_module.db, "session", SimpleNamespace(get=lambda *_args, **_kwargs: None))
|
||||
|
||||
with app.test_request_context(
|
||||
@@ -85,7 +80,7 @@ def test_instruction_generate_app_not_found(app, monkeypatch: pytest.MonkeyPatch
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response, status = method()
|
||||
response, status = method("t1")
|
||||
|
||||
assert status == 400
|
||||
assert response["error"] == "app app-1 not found"
|
||||
@@ -95,8 +90,6 @@ def test_instruction_generate_workflow_not_found(app, monkeypatch: pytest.Monkey
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
app_model = SimpleNamespace(id="app-1")
|
||||
monkeypatch.setattr(generator_module.db, "session", SimpleNamespace(get=lambda *_args, **_kwargs: app_model))
|
||||
_install_workflow_service(monkeypatch, workflow=None)
|
||||
@@ -111,7 +104,7 @@ def test_instruction_generate_workflow_not_found(app, monkeypatch: pytest.Monkey
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response, status = method()
|
||||
response, status = method("t1")
|
||||
|
||||
assert status == 400
|
||||
assert response["error"] == "workflow app-1 not found"
|
||||
@@ -121,8 +114,6 @@ def test_instruction_generate_node_missing(app, monkeypatch: pytest.MonkeyPatch)
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
app_model = SimpleNamespace(id="app-1")
|
||||
monkeypatch.setattr(generator_module.db, "session", SimpleNamespace(get=lambda *_args, **_kwargs: app_model))
|
||||
|
||||
@@ -139,7 +130,7 @@ def test_instruction_generate_node_missing(app, monkeypatch: pytest.MonkeyPatch)
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response, status = method()
|
||||
response, status = method("t1")
|
||||
|
||||
assert status == 400
|
||||
assert response["error"] == "node node-1 not found"
|
||||
@@ -149,8 +140,6 @@ def test_instruction_generate_code_node(app, monkeypatch: pytest.MonkeyPatch) ->
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
app_model = SimpleNamespace(id="app-1")
|
||||
monkeypatch.setattr(generator_module.db, "session", SimpleNamespace(get=lambda *_args, **_kwargs: app_model))
|
||||
|
||||
@@ -174,7 +163,7 @@ def test_instruction_generate_code_node(app, monkeypatch: pytest.MonkeyPatch) ->
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response = method()
|
||||
response = method("t1")
|
||||
|
||||
assert response == {"code": "x"}
|
||||
|
||||
@@ -183,7 +172,6 @@ def test_instruction_generate_legacy_modify(app, monkeypatch: pytest.MonkeyPatch
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
monkeypatch.setattr(
|
||||
generator_module.LLMGenerator,
|
||||
"instruction_modify_legacy",
|
||||
@@ -201,7 +189,7 @@ def test_instruction_generate_legacy_modify(app, monkeypatch: pytest.MonkeyPatch
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response = method()
|
||||
response = method("t1")
|
||||
|
||||
assert response == {"instruction": "ok"}
|
||||
|
||||
@@ -210,8 +198,6 @@ def test_instruction_generate_incompatible_params(app, monkeypatch: pytest.Monke
|
||||
api = generator_module.InstructionGenerateApi()
|
||||
method = _unwrap(api.post)
|
||||
|
||||
monkeypatch.setattr(generator_module, "current_account_with_tenant", lambda: (None, "t1"))
|
||||
|
||||
with app.test_request_context(
|
||||
"/console/api/instruction-generate",
|
||||
method="POST",
|
||||
@@ -223,7 +209,7 @@ def test_instruction_generate_incompatible_params(app, monkeypatch: pytest.Monke
|
||||
"model_config": _model_config_payload(),
|
||||
},
|
||||
):
|
||||
response, status = method()
|
||||
response, status = method("t1")
|
||||
|
||||
assert status == 400
|
||||
assert response["error"] == "incompatible parameters"
|
||||
|
||||
@@ -121,7 +121,6 @@ class TestAppMCPServerController:
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch.object(type(console_ns), "payload", new_callable=PropertyMock, return_value=payload),
|
||||
patch("controllers.console.app.mcp_server.current_account_with_tenant", return_value=(None, "tenant-1")),
|
||||
patch("controllers.console.app.mcp_server.db.session.add"),
|
||||
patch("controllers.console.app.mcp_server.db.session.commit"),
|
||||
patch("controllers.console.app.mcp_server.AppMCPServer.generate_server_code", return_value="server-code"),
|
||||
@@ -131,7 +130,7 @@ class TestAppMCPServerController:
|
||||
),
|
||||
):
|
||||
response, status_code = method(
|
||||
api, app_model=SimpleNamespace(id="app-1", name="Demo App", description="App description")
|
||||
api, "tenant-1", app_model=SimpleNamespace(id="app-1", name="Demo App", description="App description")
|
||||
)
|
||||
|
||||
assert response == {"id": "server-1"}
|
||||
|
||||
@@ -1036,6 +1036,48 @@ class TestSegmentListAdvancedCases:
|
||||
assert status == 200
|
||||
assert response["total"] == 1
|
||||
|
||||
def test_segment_list_postgres_keyword_filter_handles_scalar_keywords(self, app: Flask):
|
||||
api = DatasetDocumentSegmentListApi()
|
||||
method = unwrap(api.get)
|
||||
|
||||
dataset = MagicMock()
|
||||
document = MagicMock()
|
||||
pagination = MagicMock(items=[], total=0, pages=0)
|
||||
|
||||
with (
|
||||
app.test_request_context("/?keyword=test"),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "11111111-1111-1111-1111-111111111111"),
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.DatasetService.get_dataset",
|
||||
return_value=dataset,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.DocumentService.get_document",
|
||||
return_value=document,
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.dify_config",
|
||||
SimpleNamespace(SQLALCHEMY_DATABASE_URI_SCHEME="postgresql"),
|
||||
),
|
||||
patch(
|
||||
"controllers.console.datasets.datasets_segments.db.paginate",
|
||||
return_value=pagination,
|
||||
) as paginate_mock,
|
||||
):
|
||||
method(api, "22222222-2222-2222-2222-222222222222", "33333333-3333-3333-3333-333333333333")
|
||||
|
||||
query = paginate_mock.call_args.kwargs["select"]
|
||||
sql = str(query.compile(compile_kwargs={"literal_binds": True}))
|
||||
assert "jsonb_array_elements_text(CASE" in sql
|
||||
assert "ELSE CAST('[]' AS JSONB)" in sql
|
||||
|
||||
def test_segment_list_permission_denied(self, app: Flask):
|
||||
"""Test segment list with permission denied"""
|
||||
api = DatasetDocumentSegmentListApi()
|
||||
|
||||
@@ -3,22 +3,18 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import HTTPException
|
||||
|
||||
import services
|
||||
from controllers.console.auth.error import (
|
||||
CannotTransferOwnerToSelfError,
|
||||
EmailCodeError,
|
||||
InvalidEmailError,
|
||||
InvalidTokenError,
|
||||
MemberNotInTenantError,
|
||||
NotOwnerError,
|
||||
OwnerTransferLimitError,
|
||||
)
|
||||
from controllers.console.error import EmailSendIpLimitError, WorkspaceMembersLimitExceeded
|
||||
from controllers.console.workspace.members import (
|
||||
DatasetOperatorMemberListApi,
|
||||
MemberCancelInviteApi,
|
||||
MemberInviteEmailApi,
|
||||
MemberListApi,
|
||||
MemberUpdateRoleApi,
|
||||
@@ -251,135 +247,7 @@ class TestMemberInviteEmailApi:
|
||||
assert result["invitation_results"][0]["status"] == "failed"
|
||||
|
||||
|
||||
class TestMemberCancelInviteApi:
|
||||
def test_cancel_success(self, app: Flask):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
tenant = MagicMock(id="t1")
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get") as get_mock,
|
||||
patch("controllers.console.workspace.members.TenantService.remove_member_from_tenant"),
|
||||
):
|
||||
get_mock.return_value = member
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 200
|
||||
assert result["result"] == "success"
|
||||
|
||||
def test_cancel_not_found(self, app: Flask):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
tenant = MagicMock(id="t1")
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get") as get_mock,
|
||||
):
|
||||
get_mock.return_value = None
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
method(api, "x")
|
||||
|
||||
def test_cancel_cannot_operate_self(self, app: Flask):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
tenant = MagicMock(id="t1")
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get") as get_mock,
|
||||
patch(
|
||||
"controllers.console.workspace.members.TenantService.remove_member_from_tenant",
|
||||
side_effect=services.errors.account.CannotOperateSelfError("x"),
|
||||
),
|
||||
):
|
||||
get_mock.return_value = member
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 400
|
||||
|
||||
def test_cancel_no_permission(self, app: Flask):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
tenant = MagicMock(id="t1")
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get") as get_mock,
|
||||
patch(
|
||||
"controllers.console.workspace.members.TenantService.remove_member_from_tenant",
|
||||
side_effect=services.errors.account.NoPermissionError("x"),
|
||||
),
|
||||
):
|
||||
get_mock.return_value = member
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 403
|
||||
|
||||
def test_cancel_member_not_in_tenant(self, app: Flask):
|
||||
api = MemberCancelInviteApi()
|
||||
method = unwrap(api.delete)
|
||||
|
||||
tenant = MagicMock(id="t1")
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get") as get_mock,
|
||||
patch(
|
||||
"controllers.console.workspace.members.TenantService.remove_member_from_tenant",
|
||||
side_effect=services.errors.account.MemberNotInTenantError(),
|
||||
),
|
||||
):
|
||||
get_mock.return_value = member
|
||||
result, status = method(api, member.id)
|
||||
|
||||
assert status == 404
|
||||
|
||||
|
||||
class TestMemberUpdateRoleApi:
|
||||
def test_update_success(self, app: Flask):
|
||||
api = MemberUpdateRoleApi()
|
||||
method = unwrap(api.put)
|
||||
|
||||
tenant = MagicMock()
|
||||
user = MagicMock(current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
payload = {"role": "normal"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.db.session.get", return_value=member),
|
||||
patch("controllers.console.workspace.members.TenantService.update_member_role"),
|
||||
):
|
||||
result = method(api, "id")
|
||||
|
||||
if isinstance(result, tuple):
|
||||
result = result[0]
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
def test_update_invalid_role(self, app: Flask):
|
||||
api = MemberUpdateRoleApi()
|
||||
method = unwrap(api.put)
|
||||
@@ -391,23 +259,6 @@ class TestMemberUpdateRoleApi:
|
||||
|
||||
assert status == 400
|
||||
|
||||
def test_update_member_not_found(self, app: Flask):
|
||||
api = MemberUpdateRoleApi()
|
||||
method = unwrap(api.put)
|
||||
|
||||
payload = {"role": "normal"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.members.current_account_with_tenant",
|
||||
return_value=(MagicMock(current_tenant=MagicMock()), "t1"),
|
||||
),
|
||||
patch("controllers.console.workspace.members.db.session.get", return_value=None),
|
||||
):
|
||||
with pytest.raises(HTTPException):
|
||||
method(api, "id")
|
||||
|
||||
|
||||
class TestDatasetOperatorMemberListApi:
|
||||
def test_get_success(self, app: Flask):
|
||||
@@ -637,27 +488,3 @@ class TestOwnerTransferApi:
|
||||
):
|
||||
with pytest.raises(InvalidTokenError):
|
||||
method(api, "2")
|
||||
|
||||
def test_member_not_in_tenant(self, app: Flask):
|
||||
api = OwnerTransfer()
|
||||
method = unwrap(api.post)
|
||||
|
||||
tenant = MagicMock()
|
||||
user = MagicMock(id="1", email="a@test.com", current_tenant=tenant)
|
||||
member = MagicMock()
|
||||
|
||||
payload = {"token": "t"}
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch("controllers.console.workspace.members.current_account_with_tenant", return_value=(user, "t1")),
|
||||
patch("controllers.console.workspace.members.TenantService.is_owner", return_value=True),
|
||||
patch(
|
||||
"controllers.console.workspace.members.AccountService.get_owner_transfer_data",
|
||||
return_value={"email": "a@test.com"},
|
||||
),
|
||||
patch("controllers.console.workspace.members.db.session.get", return_value=member),
|
||||
patch("controllers.console.workspace.members.TenantService.is_member", return_value=False),
|
||||
):
|
||||
with pytest.raises(MemberNotInTenantError):
|
||||
method(api, "2")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
@@ -34,15 +34,11 @@ class TestDefaultModelApi:
|
||||
"/",
|
||||
query_string={"model_type": ModelType.LLM},
|
||||
),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service_mock,
|
||||
):
|
||||
service_mock.return_value.get_default_model_of_model_type.return_value = {"model": "gpt-4"}
|
||||
|
||||
result = method(api)
|
||||
result = method(api, "tenant1")
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -62,13 +58,9 @@ class TestDefaultModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result = method(api)
|
||||
result = method(api, "tenant1")
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
@@ -78,12 +70,11 @@ class TestDefaultModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", query_string={"model_type": ModelType.LLM}),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service,
|
||||
):
|
||||
service.return_value.get_default_model_of_model_type.return_value = None
|
||||
|
||||
result = method(api)
|
||||
result = method(api, "t1")
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -95,15 +86,11 @@ class TestModelProviderModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service_mock,
|
||||
):
|
||||
service_mock.return_value.get_models_by_provider.return_value = []
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -122,14 +109,10 @@ class TestModelProviderModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
patch("controllers.console.workspace.models.ModelLoadBalancingService"),
|
||||
):
|
||||
result, status = method(api, "openai")
|
||||
result, status = method(api, "tenant1", "openai")
|
||||
|
||||
assert status == 200
|
||||
|
||||
@@ -144,13 +127,9 @@ class TestModelProviderModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result, status = method(api, "openai")
|
||||
result, status = method(api, "tenant1", "openai")
|
||||
|
||||
assert status == 204
|
||||
|
||||
@@ -160,12 +139,11 @@ class TestModelProviderModelApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service,
|
||||
):
|
||||
service.return_value.get_models_by_provider.return_value = []
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "t1", "openai")
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -183,10 +161,6 @@ class TestModelProviderModelCredentialApi:
|
||||
"model_type": ModelType.LLM,
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as provider_service,
|
||||
patch("controllers.console.workspace.models.ModelLoadBalancingService") as lb_service,
|
||||
):
|
||||
@@ -198,7 +172,7 @@ class TestModelProviderModelCredentialApi:
|
||||
provider_service.return_value.provider_manager.get_provider_model_available_credentials.return_value = []
|
||||
lb_service.return_value.get_load_balancing_configs.return_value = (False, [])
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert "credentials" in result
|
||||
|
||||
@@ -214,13 +188,9 @@ class TestModelProviderModelCredentialApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result, status = method(api, "openai")
|
||||
result, status = method(api, "tenant1", "openai")
|
||||
|
||||
assert status == 201
|
||||
|
||||
@@ -230,7 +200,6 @@ class TestModelProviderModelCredentialApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", query_string={"model": "gpt", "model_type": ModelType.LLM}),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service,
|
||||
patch("controllers.console.workspace.models.ModelLoadBalancingService") as lb,
|
||||
):
|
||||
@@ -238,7 +207,7 @@ class TestModelProviderModelCredentialApi:
|
||||
service.return_value.provider_manager.get_provider_model_available_credentials.return_value = []
|
||||
lb.return_value.get_load_balancing_configs.return_value = (False, [])
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "t1", "openai")
|
||||
|
||||
assert result["credentials"] == {}
|
||||
|
||||
@@ -254,10 +223,9 @@ class TestModelProviderModelCredentialApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result, status = method(api, "openai")
|
||||
result, status = method(api, "t1", "openai")
|
||||
|
||||
assert status == 204
|
||||
|
||||
@@ -275,13 +243,9 @@ class TestModelProviderModelCredentialSwitchApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
@@ -298,13 +262,9 @@ class TestModelEnableDisableApis:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
@@ -319,13 +279,9 @@ class TestModelEnableDisableApis:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
@@ -343,13 +299,9 @@ class TestModelProviderModelValidateApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService"),
|
||||
):
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert result["result"] == "success"
|
||||
|
||||
@@ -366,15 +318,11 @@ class TestModelProviderModelValidateApi:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", json=payload),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service_mock,
|
||||
):
|
||||
service_mock.return_value.validate_model_credentials.side_effect = CredentialsValidateFailedError("invalid")
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert result["result"] == "error"
|
||||
|
||||
@@ -386,15 +334,11 @@ class TestParameterAndAvailableModels:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", query_string={"model": "gpt-4"}),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service_mock,
|
||||
):
|
||||
service_mock.return_value.get_model_parameter_rules.return_value = []
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "tenant1", "openai")
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -404,15 +348,11 @@ class TestParameterAndAvailableModels:
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch(
|
||||
"controllers.console.workspace.models.current_account_with_tenant",
|
||||
return_value=(MagicMock(), "tenant1"),
|
||||
),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service_mock,
|
||||
):
|
||||
service_mock.return_value.get_models_by_model_type.return_value = []
|
||||
|
||||
result = method(api, ModelType.LLM)
|
||||
result = method(api, "tenant1", ModelType.LLM)
|
||||
|
||||
assert "data" in result
|
||||
|
||||
@@ -422,12 +362,11 @@ class TestParameterAndAvailableModels:
|
||||
|
||||
with (
|
||||
app.test_request_context("/", query_string={"model": "gpt"}),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service,
|
||||
):
|
||||
service.return_value.get_model_parameter_rules.return_value = []
|
||||
|
||||
result = method(api, "openai")
|
||||
result = method(api, "t1", "openai")
|
||||
|
||||
assert result["data"] == []
|
||||
|
||||
@@ -437,11 +376,10 @@ class TestParameterAndAvailableModels:
|
||||
|
||||
with (
|
||||
app.test_request_context("/"),
|
||||
patch("controllers.console.workspace.models.current_account_with_tenant", return_value=(MagicMock(), "t1")),
|
||||
patch("controllers.console.workspace.models.ModelProviderService") as service,
|
||||
):
|
||||
service.return_value.get_models_by_model_type.return_value = []
|
||||
|
||||
result = method(api, ModelType.LLM)
|
||||
result = method(api, "t1", ModelType.LLM)
|
||||
|
||||
assert result["data"] == []
|
||||
|
||||
@@ -1,66 +1,73 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
from controllers.openapi.auth.composition import OAUTH_BEARER_PIPELINE, _resolve_app_authz_strategy
|
||||
from controllers.openapi.auth.pipeline import Pipeline
|
||||
from controllers.openapi.auth.steps import (
|
||||
AppAuthzCheck,
|
||||
AppResolver,
|
||||
BearerCheck,
|
||||
CallerMount,
|
||||
ScopeCheck,
|
||||
SurfaceCheck,
|
||||
WorkspaceMembershipCheck,
|
||||
)
|
||||
from controllers.openapi.auth.strategies import (
|
||||
AccountMounter,
|
||||
AclStrategy,
|
||||
EndUserMounter,
|
||||
MembershipStrategy,
|
||||
)
|
||||
from libs.oauth_bearer import SubjectType
|
||||
from controllers.openapi.auth.composition import account_pipeline, auth_router, external_sso_pipeline
|
||||
from controllers.openapi.auth.flow import When
|
||||
from controllers.openapi.auth.pipeline import AuthPipeline, PipelineRoute, PipelineRouter
|
||||
from libs.oauth_bearer import TokenType
|
||||
|
||||
|
||||
def test_pipeline_is_composed():
|
||||
assert isinstance(OAUTH_BEARER_PIPELINE, Pipeline)
|
||||
def test_account_pipeline_is_auth_pipeline():
|
||||
assert isinstance(account_pipeline, AuthPipeline)
|
||||
|
||||
|
||||
def test_pipeline_step_order():
|
||||
"""BearerCheck → SurfaceCheck → ScopeCheck → AppResolver →
|
||||
WorkspaceMembershipCheck → AppAuthzCheck → CallerMount.
|
||||
SurfaceCheck enforces the dfoa_/dfoe_ surface split + emits
|
||||
`openapi.wrong_surface_denied`. Rate-limit is enforced inside
|
||||
`BearerAuthenticator.authenticate`, not as a separate pipeline step."""
|
||||
steps = OAUTH_BEARER_PIPELINE._steps
|
||||
assert isinstance(steps[0], BearerCheck)
|
||||
assert isinstance(steps[1], SurfaceCheck)
|
||||
assert isinstance(steps[2], ScopeCheck)
|
||||
assert isinstance(steps[3], AppResolver)
|
||||
assert isinstance(steps[4], WorkspaceMembershipCheck)
|
||||
assert isinstance(steps[5], AppAuthzCheck)
|
||||
assert isinstance(steps[6], CallerMount)
|
||||
def test_external_sso_pipeline_is_auth_pipeline():
|
||||
assert isinstance(external_sso_pipeline, AuthPipeline)
|
||||
|
||||
|
||||
def test_pipeline_surface_check_accepts_account_only():
|
||||
"""Current pipeline serves /apps/<id>/run — account surface only."""
|
||||
surface = OAUTH_BEARER_PIPELINE._steps[1]
|
||||
assert isinstance(surface, SurfaceCheck)
|
||||
assert surface._accepted == frozenset({SubjectType.ACCOUNT})
|
||||
def test_auth_router_is_pipeline_router():
|
||||
assert isinstance(auth_router, PipelineRouter)
|
||||
|
||||
|
||||
def test_caller_mount_has_both_mounters():
|
||||
cm = OAUTH_BEARER_PIPELINE._steps[6]
|
||||
kinds = {type(m) for m in cm._mounters}
|
||||
assert AccountMounter in kinds
|
||||
assert EndUserMounter in kinds
|
||||
def test_account_pipeline_prepare_has_four_entries():
|
||||
assert len(account_pipeline._prepare) == 4
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.composition.FeatureService")
|
||||
def test_strategy_resolver_picks_acl_when_enabled(fs):
|
||||
fs.get_system_features.return_value.webapp_auth.enabled = True
|
||||
assert isinstance(_resolve_app_authz_strategy(), AclStrategy)
|
||||
def test_account_auth_list_has_five_entries():
|
||||
assert len(account_pipeline._auth) == 5
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.composition.FeatureService")
|
||||
def test_strategy_resolver_picks_membership_when_disabled(fs):
|
||||
fs.get_system_features.return_value.webapp_auth.enabled = False
|
||||
assert isinstance(_resolve_app_authz_strategy(), MembershipStrategy)
|
||||
def test_external_sso_pipeline_prepare_has_four_entries():
|
||||
assert len(external_sso_pipeline._prepare) == 4
|
||||
|
||||
|
||||
def test_external_sso_auth_list_has_three_entries():
|
||||
assert len(external_sso_pipeline._auth) == 3
|
||||
|
||||
|
||||
def test_account_pipeline_has_unconditional_load_account():
|
||||
non_when = [s for s in account_pipeline._prepare if not isinstance(s, When)]
|
||||
assert len(non_when) == 1
|
||||
|
||||
|
||||
def test_external_sso_pipeline_all_prepare_entries_are_when():
|
||||
assert all(isinstance(s, When) for s in external_sso_pipeline._prepare)
|
||||
|
||||
|
||||
def test_first_auth_entry_is_check_scope_in_both_pipelines():
|
||||
assert not isinstance(account_pipeline._auth[0], When)
|
||||
assert not isinstance(external_sso_pipeline._auth[0], When)
|
||||
|
||||
|
||||
def test_remaining_auth_entries_are_when_for_account():
|
||||
assert all(isinstance(s, When) for s in account_pipeline._auth[1:])
|
||||
|
||||
|
||||
def test_remaining_auth_entries_are_when_for_external_sso():
|
||||
assert all(isinstance(s, When) for s in external_sso_pipeline._auth[1:])
|
||||
|
||||
|
||||
def test_router_routes_contain_both_token_types():
|
||||
assert TokenType.OAUTH_ACCOUNT in auth_router._routes
|
||||
assert TokenType.OAUTH_EXTERNAL_SSO in auth_router._routes
|
||||
|
||||
|
||||
def test_external_sso_route_has_ee_required_edition():
|
||||
route = auth_router._routes[TokenType.OAUTH_EXTERNAL_SSO]
|
||||
assert isinstance(route, PipelineRoute)
|
||||
from controllers.openapi.auth.data import Edition
|
||||
|
||||
assert route.required_edition == frozenset({Edition.EE})
|
||||
|
||||
|
||||
def test_account_route_has_no_required_edition():
|
||||
route = auth_router._routes[TokenType.OAUTH_ACCOUNT]
|
||||
assert isinstance(route, PipelineRoute)
|
||||
assert route.required_edition is None
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from controllers.openapi.auth.conditions import (
|
||||
EDITION_CE,
|
||||
EDITION_EE,
|
||||
EDITION_SAAS,
|
||||
LOADED_APP_IS_PRIVATE,
|
||||
PATH_HAS_APP_ID,
|
||||
TOKEN_IS_OAUTH_ACCOUNT,
|
||||
TOKEN_IS_OAUTH_EXTERNAL_SSO,
|
||||
WEBAPP_AUTH_ENABLED,
|
||||
Cond,
|
||||
config_cond,
|
||||
data_cond,
|
||||
request_cond,
|
||||
)
|
||||
from controllers.openapi.auth.data import AuthData, Edition, RequestContext
|
||||
from libs.oauth_bearer import TokenType
|
||||
from services.enterprise.enterprise_service import WebAppAccessMode
|
||||
|
||||
|
||||
def _ctx(token_type=TokenType.OAUTH_ACCOUNT, path_params=None):
|
||||
return RequestContext(
|
||||
token_type=token_type,
|
||||
path_params=path_params or {},
|
||||
)
|
||||
|
||||
|
||||
def _data(**kwargs):
|
||||
defaults: dict = {"token_type": TokenType.OAUTH_ACCOUNT, "token_hash": "x", "scopes": frozenset()}
|
||||
defaults.update(kwargs)
|
||||
return AuthData(**defaults)
|
||||
|
||||
|
||||
def test_and_both_true():
|
||||
a = Cond(lambda ctx, _: True)
|
||||
b = Cond(lambda ctx, _: True)
|
||||
assert (a & b)(_ctx()) is True
|
||||
|
||||
|
||||
def test_and_one_false():
|
||||
a = Cond(lambda ctx, _: True)
|
||||
b = Cond(lambda ctx, _: False)
|
||||
assert (a & b)(_ctx()) is False
|
||||
|
||||
|
||||
def test_or_one_true():
|
||||
a = Cond(lambda ctx, _: False)
|
||||
b = Cond(lambda ctx, _: True)
|
||||
assert (a | b)(_ctx()) is True
|
||||
|
||||
|
||||
def test_or_both_false():
|
||||
a = Cond(lambda ctx, _: False)
|
||||
b = Cond(lambda ctx, _: False)
|
||||
assert (a | b)(_ctx()) is False
|
||||
|
||||
|
||||
def test_invert():
|
||||
a = Cond(lambda ctx, _: True)
|
||||
assert (~a)(_ctx()) is False
|
||||
|
||||
|
||||
def test_chain_and_or():
|
||||
always_true = Cond(lambda ctx, _: True)
|
||||
always_false = Cond(lambda ctx, _: False)
|
||||
assert ((always_true | always_false) & always_true)(_ctx()) is True
|
||||
|
||||
|
||||
def test_request_cond_ignores_data():
|
||||
c = request_cond(lambda ctx: ctx.token_type == TokenType.OAUTH_ACCOUNT)
|
||||
assert c(_ctx(TokenType.OAUTH_ACCOUNT)) is True
|
||||
assert c(_ctx(TokenType.OAUTH_EXTERNAL_SSO)) is False
|
||||
|
||||
|
||||
def test_data_cond_returns_false_when_data_none():
|
||||
c = data_cond(lambda data: True)
|
||||
assert c(_ctx(), None) is False
|
||||
|
||||
|
||||
def test_data_cond_evaluates_when_data_present():
|
||||
c = data_cond(lambda data: data.token_hash == "secret")
|
||||
assert c(_ctx(), _data(token_hash="secret")) is True
|
||||
assert c(_ctx(), _data(token_hash="other")) is False
|
||||
|
||||
|
||||
def test_config_cond_ignores_ctx_and_data():
|
||||
c = config_cond(lambda: True)
|
||||
assert c(_ctx()) is True
|
||||
c2 = config_cond(lambda: False)
|
||||
assert c2(_ctx(), _data()) is False
|
||||
|
||||
|
||||
def test_token_is_oauth_account():
|
||||
assert TOKEN_IS_OAUTH_ACCOUNT(_ctx(TokenType.OAUTH_ACCOUNT)) is True
|
||||
assert TOKEN_IS_OAUTH_ACCOUNT(_ctx(TokenType.OAUTH_EXTERNAL_SSO)) is False
|
||||
|
||||
|
||||
def test_token_is_oauth_external_sso():
|
||||
assert TOKEN_IS_OAUTH_EXTERNAL_SSO(_ctx(TokenType.OAUTH_EXTERNAL_SSO)) is True
|
||||
|
||||
|
||||
def test_path_has_app_id_true():
|
||||
assert PATH_HAS_APP_ID(_ctx(path_params={"app_id": "abc"})) is True
|
||||
|
||||
|
||||
def test_path_has_app_id_false():
|
||||
assert PATH_HAS_APP_ID(_ctx(path_params={})) is False
|
||||
|
||||
|
||||
def test_edition_ce():
|
||||
with patch("controllers.openapi.auth.conditions.current_edition", return_value=Edition.CE):
|
||||
assert EDITION_CE(_ctx()) is True
|
||||
assert EDITION_EE(_ctx()) is False
|
||||
assert EDITION_SAAS(_ctx()) is False
|
||||
|
||||
|
||||
def test_edition_ee():
|
||||
with patch("controllers.openapi.auth.conditions.current_edition", return_value=Edition.EE):
|
||||
assert EDITION_EE(_ctx()) is True
|
||||
assert EDITION_CE(_ctx()) is False
|
||||
|
||||
|
||||
def test_edition_saas():
|
||||
with patch("controllers.openapi.auth.conditions.current_edition", return_value=Edition.SAAS):
|
||||
assert EDITION_SAAS(_ctx()) is True
|
||||
|
||||
|
||||
def test_webapp_auth_enabled():
|
||||
mock_features = MagicMock()
|
||||
mock_features.webapp_auth.enabled = True
|
||||
with patch("controllers.openapi.auth.conditions.FeatureService.get_system_features", return_value=mock_features):
|
||||
assert WEBAPP_AUTH_ENABLED(_ctx()) is True
|
||||
|
||||
|
||||
def test_loaded_app_is_private():
|
||||
data_private = _data(app_access_mode=WebAppAccessMode.PRIVATE)
|
||||
data_public = _data(app_access_mode=WebAppAccessMode.PUBLIC)
|
||||
data_none = _data(app_access_mode=None)
|
||||
assert LOADED_APP_IS_PRIVATE(_ctx(), data_private) is True
|
||||
assert LOADED_APP_IS_PRIVATE(_ctx(), data_public) is False
|
||||
assert LOADED_APP_IS_PRIVATE(_ctx(), data_none) is False
|
||||
assert LOADED_APP_IS_PRIVATE(_ctx(), None) is False
|
||||
@@ -1,21 +0,0 @@
|
||||
from controllers.openapi.auth.context import Context
|
||||
|
||||
|
||||
def test_context_starts_unpopulated():
|
||||
ctx = Context(required_scope="apps:run")
|
||||
assert ctx.bearer_token is None
|
||||
assert ctx.path_params == {}
|
||||
assert ctx.subject_type is None
|
||||
assert ctx.subject_email is None
|
||||
assert ctx.account_id is None
|
||||
assert ctx.scopes == frozenset()
|
||||
assert ctx.app is None
|
||||
assert ctx.tenant is None
|
||||
assert ctx.caller is None
|
||||
assert ctx.caller_kind is None
|
||||
|
||||
|
||||
def test_context_fields_are_mutable():
|
||||
ctx = Context(required_scope="apps:run")
|
||||
ctx.scopes = frozenset({"full"})
|
||||
assert "full" in ctx.scopes
|
||||
@@ -0,0 +1,117 @@
|
||||
import uuid
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from controllers.openapi.auth.data import (
|
||||
AuthData,
|
||||
Edition,
|
||||
ExternalIdentity,
|
||||
RequestContext,
|
||||
current_edition,
|
||||
)
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
|
||||
|
||||
def test_current_edition_saas():
|
||||
with patch("controllers.openapi.auth.data.dify_config") as cfg:
|
||||
cfg.EDITION = "CLOUD"
|
||||
cfg.ENTERPRISE_ENABLED = True
|
||||
assert current_edition() == Edition.SAAS
|
||||
|
||||
|
||||
def test_current_edition_ee():
|
||||
with patch("controllers.openapi.auth.data.dify_config") as cfg:
|
||||
cfg.EDITION = "SELF_HOSTED"
|
||||
cfg.ENTERPRISE_ENABLED = True
|
||||
assert current_edition() == Edition.EE
|
||||
|
||||
|
||||
def test_current_edition_ce():
|
||||
with patch("controllers.openapi.auth.data.dify_config") as cfg:
|
||||
cfg.EDITION = "SELF_HOSTED"
|
||||
cfg.ENTERPRISE_ENABLED = False
|
||||
assert current_edition() == Edition.CE
|
||||
|
||||
|
||||
def test_external_identity_frozen():
|
||||
ei = ExternalIdentity(email="a@b.com", issuer="idp")
|
||||
with pytest.raises(ValidationError):
|
||||
ei.email = "other@b.com" # type: ignore[misc]
|
||||
|
||||
|
||||
def test_external_identity_issuer_optional():
|
||||
ei = ExternalIdentity(email="a@b.com")
|
||||
assert ei.issuer is None
|
||||
|
||||
|
||||
def test_request_context_frozen():
|
||||
ctx = RequestContext(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
path_params={"app_id": "123"},
|
||||
)
|
||||
with pytest.raises(ValidationError):
|
||||
ctx.token_type = TokenType.OAUTH_EXTERNAL_SSO # type: ignore[misc]
|
||||
|
||||
|
||||
def test_request_context_scope_optional():
|
||||
ctx = RequestContext(token_type=TokenType.OAUTH_ACCOUNT, path_params={})
|
||||
assert ctx.scope is None
|
||||
|
||||
|
||||
def test_auth_data_is_mutable():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
token_hash="abc",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
)
|
||||
data.token_type = TokenType.OAUTH_EXTERNAL_SSO
|
||||
assert data.token_type == TokenType.OAUTH_EXTERNAL_SSO
|
||||
|
||||
|
||||
def test_auth_data_path_params_defaults_empty():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
token_hash="abc",
|
||||
scopes=frozenset(),
|
||||
)
|
||||
assert data.path_params == {}
|
||||
|
||||
|
||||
def test_auth_data_account_id_optional():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
token_hash="abc",
|
||||
scopes=frozenset({Scope.APPS_RUN}),
|
||||
external_identity=ExternalIdentity(email="u@sso.com"),
|
||||
)
|
||||
assert data.account_id is None
|
||||
|
||||
|
||||
def test_auth_data_external_identity_none_for_account():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
account_id=uuid.uuid4(),
|
||||
token_hash="abc",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
)
|
||||
assert data.external_identity is None
|
||||
|
||||
|
||||
def test_auth_data_tenants_default_empty():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
token_hash="abc",
|
||||
scopes=frozenset(),
|
||||
)
|
||||
assert data.tenants == {}
|
||||
|
||||
|
||||
def test_auth_data_token_id_optional():
|
||||
data = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
token_hash="abc",
|
||||
scopes=frozenset(),
|
||||
)
|
||||
assert data.token_id is None
|
||||
@@ -0,0 +1,42 @@
|
||||
import inspect
|
||||
|
||||
from controllers.openapi.auth.conditions import Cond
|
||||
from controllers.openapi.auth.data import AuthData, RequestContext
|
||||
from controllers.openapi.auth.flow import When
|
||||
from libs.oauth_bearer import TokenType
|
||||
|
||||
|
||||
def _ctx():
|
||||
return RequestContext(token_type=TokenType.OAUTH_ACCOUNT, path_params={})
|
||||
|
||||
|
||||
def _data():
|
||||
return AuthData(token_type=TokenType.OAUTH_ACCOUNT, token_hash="x", scopes=frozenset())
|
||||
|
||||
|
||||
def test_applies_returns_true_when_condition_true():
|
||||
w = When(Cond(lambda ctx, _: True), then=lambda b: None)
|
||||
assert w.applies(_ctx()) is True
|
||||
|
||||
|
||||
def test_applies_returns_false_when_condition_false():
|
||||
w = When(Cond(lambda ctx, _: False), then=lambda b: None)
|
||||
assert w.applies(_ctx()) is False
|
||||
|
||||
|
||||
def test_applies_with_data():
|
||||
w = When(Cond(lambda ctx, data: data is not None), then=lambda b: None)
|
||||
assert w.applies(_ctx(), _data()) is True
|
||||
assert w.applies(_ctx(), None) is False
|
||||
|
||||
|
||||
def test_call_invokes_step():
|
||||
calls = []
|
||||
w = When(Cond(lambda ctx, _: True), then=lambda arg: calls.append(arg))
|
||||
w("payload")
|
||||
assert calls == ["payload"]
|
||||
|
||||
|
||||
def test_then_is_keyword_only():
|
||||
sig = inspect.signature(When.__init__)
|
||||
assert sig.parameters["then"].kind.name == "KEYWORD_ONLY"
|
||||
@@ -1,59 +1,269 @@
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.pipeline import Pipeline
|
||||
from controllers.openapi.auth.data import AuthData, Edition
|
||||
from controllers.openapi.auth.pipeline import AuthPipeline, PipelineRoute, PipelineRouter
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
|
||||
|
||||
def test_run_invokes_each_step_in_order():
|
||||
calls = []
|
||||
|
||||
class S:
|
||||
def __init__(self, tag):
|
||||
self.tag = tag
|
||||
|
||||
def __call__(self, ctx):
|
||||
calls.append(self.tag)
|
||||
|
||||
Pipeline(S("a"), S("b"), S("c")).run(Context(required_scope="x"))
|
||||
assert calls == ["a", "b", "c"]
|
||||
def _make_identity(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
account_id=None,
|
||||
scopes=None,
|
||||
token_hash="testhash",
|
||||
subject_email=None,
|
||||
subject_issuer=None,
|
||||
verified_tenants=None,
|
||||
token_id=None,
|
||||
):
|
||||
identity = MagicMock()
|
||||
identity.token_type = token_type
|
||||
identity.account_id = account_id or uuid.uuid4()
|
||||
identity.scopes = scopes or frozenset({Scope.FULL})
|
||||
identity.token_hash = token_hash
|
||||
identity.subject_email = subject_email
|
||||
identity.subject_issuer = subject_issuer
|
||||
identity.verified_tenants = verified_tenants or {}
|
||||
identity.token_id = token_id or uuid.uuid4()
|
||||
return identity
|
||||
|
||||
|
||||
def test_run_short_circuits_on_raise():
|
||||
calls = []
|
||||
|
||||
class Boom:
|
||||
def __call__(self, ctx):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
class Tail:
|
||||
def __call__(self, ctx):
|
||||
calls.append("ran")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
Pipeline(Boom(), Tail()).run(Context(required_scope="x"))
|
||||
assert calls == []
|
||||
@pytest.fixture
|
||||
def app():
|
||||
return Flask(__name__)
|
||||
|
||||
|
||||
def test_guard_decorator_runs_pipeline_and_unpacks_handler_kwargs():
|
||||
seen = {}
|
||||
def _make_router(token_type=TokenType.OAUTH_ACCOUNT, prepare=None, auth=None):
|
||||
pipeline = AuthPipeline(prepare=prepare or [], auth=auth or [])
|
||||
return PipelineRouter({token_type: PipelineRoute(pipeline)})
|
||||
|
||||
class FakeStep:
|
||||
def __call__(self, ctx):
|
||||
ctx.app = "APP"
|
||||
ctx.caller = "CALLER"
|
||||
ctx.caller_kind = "account"
|
||||
|
||||
pipeline = Pipeline(FakeStep())
|
||||
def _fake_identity():
|
||||
return _make_identity()
|
||||
|
||||
@pipeline.guard(scope="apps:run")
|
||||
def handler(app_model, caller, caller_kind):
|
||||
seen["app_model"] = app_model
|
||||
seen["caller"] = caller
|
||||
seen["caller_kind"] = caller_kind
|
||||
return "ok"
|
||||
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/x", method="POST"):
|
||||
assert handler() == "ok"
|
||||
assert seen == {"app_model": "APP", "caller": "CALLER", "caller_kind": "account"}
|
||||
# --- PipelineRouter.guard ---
|
||||
|
||||
|
||||
def test_guard_passes_auth_data_to_view(app):
|
||||
router = _make_router()
|
||||
received = {}
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.set_auth_ctx", return_value=MagicMock()),
|
||||
patch("controllers.openapi.auth.pipeline.reset_auth_ctx"),
|
||||
):
|
||||
mock_auth.return_value.authenticate.return_value = _fake_identity()
|
||||
|
||||
@router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def view(*, auth_data):
|
||||
received["data"] = auth_data
|
||||
|
||||
view()
|
||||
|
||||
assert isinstance(received["data"], AuthData)
|
||||
|
||||
|
||||
def test_guard_edition_gate_returns_404(app):
|
||||
router = _make_router()
|
||||
|
||||
with app.test_request_context("/test"):
|
||||
with patch("controllers.openapi.auth.pipeline.current_edition", return_value=Edition.CE):
|
||||
|
||||
@router.guard(scope=Scope.FULL, edition=frozenset({Edition.EE}))
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
with pytest.raises(NotFound):
|
||||
view()
|
||||
|
||||
|
||||
def test_guard_token_type_gate_returns_403(app):
|
||||
router = _make_router()
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.emit_wrong_surface"),
|
||||
patch("controllers.openapi.auth.pipeline.current_edition", return_value=Edition.CE),
|
||||
):
|
||||
identity = _fake_identity()
|
||||
identity.token_type = TokenType.OAUTH_EXTERNAL_SSO
|
||||
mock_auth.return_value.authenticate.return_value = identity
|
||||
|
||||
@router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
with pytest.raises(Forbidden):
|
||||
view()
|
||||
|
||||
|
||||
def test_guard_unregistered_token_type_returns_403(app):
|
||||
router = _make_router(token_type=TokenType.OAUTH_ACCOUNT)
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.current_edition", return_value=Edition.CE),
|
||||
):
|
||||
identity = _fake_identity()
|
||||
identity.token_type = TokenType.OAUTH_EXTERNAL_SSO
|
||||
mock_auth.return_value.authenticate.return_value = identity
|
||||
|
||||
@router.guard(scope=Scope.FULL)
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
with pytest.raises(Forbidden):
|
||||
view()
|
||||
|
||||
|
||||
def test_guard_no_bearer_returns_401(app):
|
||||
router = _make_router()
|
||||
|
||||
with app.test_request_context("/test"):
|
||||
with patch("controllers.openapi.auth.pipeline.extract_bearer", return_value=None):
|
||||
|
||||
@router.guard(scope=Scope.FULL)
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
with pytest.raises(Unauthorized):
|
||||
view()
|
||||
|
||||
|
||||
def test_guard_runs_prepare_steps_in_order(app):
|
||||
order = []
|
||||
|
||||
def p1(b):
|
||||
order.append("p1")
|
||||
|
||||
def p2(b):
|
||||
order.append("p2")
|
||||
|
||||
router = _make_router(prepare=[p1, p2])
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.set_auth_ctx", return_value=MagicMock()),
|
||||
patch("controllers.openapi.auth.pipeline.reset_auth_ctx"),
|
||||
):
|
||||
mock_auth.return_value.authenticate.return_value = _fake_identity()
|
||||
|
||||
@router.guard(scope=Scope.FULL)
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
view()
|
||||
|
||||
assert order == ["p1", "p2"]
|
||||
|
||||
|
||||
def test_guard_resets_auth_ctx_on_exception(app):
|
||||
router = _make_router()
|
||||
reset_called = []
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.set_auth_ctx", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.reset_auth_ctx", side_effect=lambda t: reset_called.append(t)),
|
||||
):
|
||||
mock_auth.return_value.authenticate.return_value = _fake_identity()
|
||||
|
||||
@router.guard(scope=Scope.FULL)
|
||||
def view(*, auth_data):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
view()
|
||||
|
||||
assert reset_called == ["tok"]
|
||||
|
||||
|
||||
def test_router_rejects_token_type_on_wrong_edition(app):
|
||||
pipeline = AuthPipeline(prepare=[], auth=[])
|
||||
route = PipelineRoute(pipeline, required_edition=frozenset({Edition.EE}))
|
||||
router = PipelineRouter({TokenType.OAUTH_EXTERNAL_SSO: route})
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.current_edition", return_value=Edition.CE),
|
||||
):
|
||||
identity = _make_identity(token_type=TokenType.OAUTH_EXTERNAL_SSO)
|
||||
mock_auth.return_value.authenticate.return_value = identity
|
||||
|
||||
@router.guard(scope=Scope.APPS_RUN)
|
||||
def view(*, auth_data):
|
||||
pass
|
||||
|
||||
with pytest.raises(Forbidden):
|
||||
view()
|
||||
|
||||
|
||||
def test_guard_populates_external_identity_from_subject_email(app):
|
||||
from controllers.openapi.auth.data import ExternalIdentity
|
||||
|
||||
router = _make_router(token_type=TokenType.OAUTH_EXTERNAL_SSO)
|
||||
received = {}
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.set_auth_ctx", return_value=MagicMock()),
|
||||
patch("controllers.openapi.auth.pipeline.reset_auth_ctx"),
|
||||
):
|
||||
identity = _make_identity(
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
subject_email="user@sso.com",
|
||||
subject_issuer="https://idp.example.com",
|
||||
)
|
||||
mock_auth.return_value.authenticate.return_value = identity
|
||||
|
||||
@router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_EXTERNAL_SSO}))
|
||||
def view(*, auth_data):
|
||||
received["data"] = auth_data
|
||||
|
||||
view()
|
||||
|
||||
assert isinstance(received["data"].external_identity, ExternalIdentity)
|
||||
assert received["data"].external_identity.email == "user@sso.com"
|
||||
assert received["data"].external_identity.issuer == "https://idp.example.com"
|
||||
|
||||
|
||||
def test_guard_no_external_identity_when_subject_email_absent(app):
|
||||
router = _make_router()
|
||||
received = {}
|
||||
|
||||
with app.test_request_context("/test", headers={"Authorization": "Bearer tok"}):
|
||||
with (
|
||||
patch("controllers.openapi.auth.pipeline.extract_bearer", return_value="tok"),
|
||||
patch("controllers.openapi.auth.pipeline.get_authenticator") as mock_auth,
|
||||
patch("controllers.openapi.auth.pipeline.set_auth_ctx", return_value=MagicMock()),
|
||||
patch("controllers.openapi.auth.pipeline.reset_auth_ctx"),
|
||||
):
|
||||
mock_auth.return_value.authenticate.return_value = _make_identity(subject_email=None)
|
||||
|
||||
@router.guard(scope=Scope.FULL, allowed_token_types=frozenset({TokenType.OAUTH_ACCOUNT}))
|
||||
def view(*, auth_data):
|
||||
received["data"] = auth_data
|
||||
|
||||
view()
|
||||
|
||||
assert received["data"].external_identity is None
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.data import AuthData, ExternalIdentity
|
||||
from controllers.openapi.auth.prepare import (
|
||||
load_account,
|
||||
load_app,
|
||||
load_app_access_mode,
|
||||
load_tenant,
|
||||
resolve_external_user,
|
||||
)
|
||||
from libs.oauth_bearer import TokenType
|
||||
|
||||
|
||||
def _make_auth_data(**kwargs) -> AuthData:
|
||||
mock_fields = {k: kwargs.pop(k) for k in ("app", "tenant", "caller") if k in kwargs}
|
||||
data = AuthData(
|
||||
token_type=kwargs.pop("token_type", TokenType.OAUTH_ACCOUNT),
|
||||
token_hash=kwargs.pop("token_hash", "testhash"),
|
||||
scopes=kwargs.pop("scopes", frozenset()),
|
||||
**kwargs,
|
||||
)
|
||||
for k, v in mock_fields.items():
|
||||
setattr(data, k, v)
|
||||
return data
|
||||
|
||||
|
||||
def test_load_app_writes_app_to_data():
|
||||
app = MagicMock()
|
||||
app.status = "normal"
|
||||
app.enable_api = True
|
||||
data = _make_auth_data(path_params={"app_id": "abc"})
|
||||
with patch("controllers.openapi.auth.prepare.AppService.get_app_by_id", return_value=app):
|
||||
load_app(data)
|
||||
assert data.app is app
|
||||
|
||||
|
||||
def test_load_app_raises_not_found_when_missing():
|
||||
data = _make_auth_data(path_params={"app_id": "missing"})
|
||||
with patch("controllers.openapi.auth.prepare.AppService.get_app_by_id", return_value=None):
|
||||
with pytest.raises(NotFound):
|
||||
load_app(data)
|
||||
|
||||
|
||||
def test_load_app_raises_not_found_when_not_normal():
|
||||
app = MagicMock()
|
||||
app.status = "archived"
|
||||
data = _make_auth_data(path_params={"app_id": "abc"})
|
||||
with patch("controllers.openapi.auth.prepare.AppService.get_app_by_id", return_value=app):
|
||||
with pytest.raises(NotFound):
|
||||
load_app(data)
|
||||
|
||||
|
||||
def test_load_app_raises_forbidden_when_api_disabled():
|
||||
app = MagicMock()
|
||||
app.status = "normal"
|
||||
app.enable_api = False
|
||||
data = _make_auth_data(path_params={"app_id": "abc"})
|
||||
with patch("controllers.openapi.auth.prepare.AppService.get_app_by_id", return_value=app):
|
||||
with pytest.raises(Forbidden):
|
||||
load_app(data)
|
||||
|
||||
|
||||
def test_load_tenant_writes_tenant():
|
||||
app = MagicMock()
|
||||
app.tenant_id = uuid.uuid4()
|
||||
tenant = MagicMock()
|
||||
tenant.status = "normal"
|
||||
data = _make_auth_data(app=app)
|
||||
with patch("controllers.openapi.auth.prepare.TenantService.get_tenant_by_id", return_value=tenant):
|
||||
load_tenant(data)
|
||||
assert data.tenant is tenant
|
||||
|
||||
|
||||
def test_load_tenant_raises_forbidden_when_archived():
|
||||
from models.account import TenantStatus
|
||||
|
||||
app = MagicMock()
|
||||
app.tenant_id = uuid.uuid4()
|
||||
tenant = MagicMock()
|
||||
tenant.status = TenantStatus.ARCHIVE
|
||||
data = _make_auth_data(app=app)
|
||||
with patch("controllers.openapi.auth.prepare.TenantService.get_tenant_by_id", return_value=tenant):
|
||||
with pytest.raises(Forbidden):
|
||||
load_tenant(data)
|
||||
|
||||
|
||||
def test_load_tenant_raises_forbidden_when_missing():
|
||||
app = MagicMock()
|
||||
app.tenant_id = uuid.uuid4()
|
||||
data = _make_auth_data(app=app)
|
||||
with patch("controllers.openapi.auth.prepare.TenantService.get_tenant_by_id", return_value=None):
|
||||
with pytest.raises(Forbidden):
|
||||
load_tenant(data)
|
||||
|
||||
|
||||
def test_load_tenant_raises_500_when_app_not_loaded():
|
||||
from werkzeug.exceptions import InternalServerError
|
||||
|
||||
data = _make_auth_data()
|
||||
with pytest.raises(InternalServerError):
|
||||
load_tenant(data)
|
||||
|
||||
|
||||
def test_load_account_writes_caller():
|
||||
account = MagicMock()
|
||||
account_id = uuid.uuid4()
|
||||
data = _make_auth_data(account_id=account_id)
|
||||
with patch("controllers.openapi.auth.prepare.AccountService.get_account_by_id", return_value=account):
|
||||
load_account(data)
|
||||
assert data.caller is account
|
||||
assert data.caller_kind == "account"
|
||||
|
||||
|
||||
def test_load_account_sets_current_tenant_when_tenant_present():
|
||||
account = MagicMock()
|
||||
tenant = MagicMock()
|
||||
data = _make_auth_data(account_id=uuid.uuid4(), tenant=tenant)
|
||||
with patch("controllers.openapi.auth.prepare.AccountService.get_account_by_id", return_value=account):
|
||||
load_account(data)
|
||||
assert account.current_tenant is tenant
|
||||
|
||||
|
||||
def test_load_account_raises_unauthorized_when_not_found():
|
||||
data = _make_auth_data(account_id=uuid.uuid4())
|
||||
with patch("controllers.openapi.auth.prepare.AccountService.get_account_by_id", return_value=None):
|
||||
with pytest.raises(Unauthorized):
|
||||
load_account(data)
|
||||
|
||||
|
||||
def test_resolve_external_user_writes_caller():
|
||||
tenant = MagicMock()
|
||||
app = MagicMock()
|
||||
end_user = MagicMock()
|
||||
ext = ExternalIdentity(email="user@sso.com")
|
||||
data = _make_auth_data(tenant=tenant, app=app, external_identity=ext)
|
||||
with patch("controllers.openapi.auth.prepare.EndUserService.get_or_create_end_user_by_type", return_value=end_user):
|
||||
resolve_external_user(data)
|
||||
assert data.caller is end_user
|
||||
assert data.caller_kind == "end_user"
|
||||
|
||||
|
||||
def test_resolve_external_user_raises_unauthorized_when_context_missing():
|
||||
data = _make_auth_data(tenant=None, app=MagicMock(), external_identity=ExternalIdentity(email="u@s.com"))
|
||||
with pytest.raises(Unauthorized):
|
||||
resolve_external_user(data)
|
||||
|
||||
|
||||
def test_load_app_access_mode_writes_mode():
|
||||
from services.enterprise.enterprise_service import WebAppAccessMode
|
||||
|
||||
app = MagicMock()
|
||||
app.id = "app-1"
|
||||
settings = MagicMock()
|
||||
settings.access_mode = "public"
|
||||
data = _make_auth_data(app=app)
|
||||
with patch(
|
||||
"controllers.openapi.auth.prepare.EnterpriseService.WebAppAuth.get_app_access_mode_by_id",
|
||||
return_value=settings,
|
||||
):
|
||||
load_app_access_mode(data)
|
||||
assert data.app_access_mode == WebAppAccessMode.PUBLIC
|
||||
|
||||
|
||||
def test_load_app_access_mode_writes_none_when_value_error():
|
||||
app = MagicMock()
|
||||
app.id = "app-1"
|
||||
data = _make_auth_data(app=app)
|
||||
with patch(
|
||||
"controllers.openapi.auth.prepare.EnterpriseService.WebAppAuth.get_app_access_mode_by_id",
|
||||
side_effect=ValueError("No data found."),
|
||||
):
|
||||
load_app_access_mode(data)
|
||||
assert data.app_access_mode is None
|
||||
|
||||
|
||||
def test_load_app_access_mode_no_op_when_app_missing():
|
||||
data = _make_auth_data()
|
||||
load_app_access_mode(data)
|
||||
assert data.app_access_mode is None
|
||||
@@ -26,7 +26,7 @@ from flask import Flask
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
|
||||
from controllers.openapi.auth.role_gate import require_workspace_role
|
||||
from libs.oauth_bearer import AuthContext, Scope, SubjectType, reset_auth_ctx, set_auth_ctx
|
||||
from libs.oauth_bearer import AuthContext, Scope, SubjectType, TokenType, reset_auth_ctx, set_auth_ctx
|
||||
from models.account import TenantAccountRole
|
||||
|
||||
# Tokens from `_seed`'s `set_auth_ctx` calls, drained after each test so a
|
||||
@@ -55,7 +55,7 @@ def _account_ctx(account_id: uuid.UUID | None = None) -> AuthContext:
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
token_id=uuid.uuid4(),
|
||||
source="oauth_account",
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
expires_at=datetime.now(UTC),
|
||||
token_hash="h1",
|
||||
verified_tenants={},
|
||||
@@ -71,7 +71,7 @@ def _sso_ctx() -> AuthContext:
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({Scope.APPS_RUN}),
|
||||
token_id=uuid.uuid4(),
|
||||
source="oauth_external_sso",
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
expires_at=datetime.now(UTC),
|
||||
token_hash="h2",
|
||||
verified_tenants={},
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import BadRequest, Forbidden, NotFound
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import AppResolver
|
||||
from models import TenantStatus
|
||||
|
||||
|
||||
def _ctx(path_params: dict[str, str] | None) -> Context:
|
||||
return Context(required_scope="apps:run", path_params=path_params or {})
|
||||
|
||||
|
||||
def _app(*, status="normal", enable_api=True):
|
||||
return SimpleNamespace(id="app1", tenant_id="t1", status=status, enable_api=enable_api)
|
||||
|
||||
|
||||
def _tenant(*, status=TenantStatus.NORMAL):
|
||||
return SimpleNamespace(id="t1", status=status)
|
||||
|
||||
|
||||
def test_resolver_rejects_missing_path_param():
|
||||
with pytest.raises(BadRequest):
|
||||
AppResolver()(_ctx({}))
|
||||
|
||||
|
||||
def test_resolver_rejects_empty_path_params():
|
||||
# `Pipeline.guard` always seeds an empty dict when Flask reports no
|
||||
# view args, so a missing `app_id` key surfaces here as BadRequest.
|
||||
with pytest.raises(BadRequest):
|
||||
AppResolver()(_ctx(None))
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.db")
|
||||
def test_resolver_404_when_app_missing(db):
|
||||
db.session.get.side_effect = [None]
|
||||
with pytest.raises(NotFound):
|
||||
AppResolver()(_ctx({"app_id": "x"}))
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.db")
|
||||
def test_resolver_403_when_disabled(db):
|
||||
db.session.get.side_effect = [_app(enable_api=False)]
|
||||
with pytest.raises(Forbidden) as exc:
|
||||
AppResolver()(_ctx({"app_id": "x"}))
|
||||
assert "service_api_disabled" in str(exc.value.description)
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.db")
|
||||
def test_resolver_403_when_tenant_archived(db):
|
||||
db.session.get.side_effect = [_app(), _tenant(status=TenantStatus.ARCHIVE)]
|
||||
with pytest.raises(Forbidden):
|
||||
AppResolver()(_ctx({"app_id": "x"}))
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.db")
|
||||
def test_resolver_populates_app_and_tenant(db):
|
||||
db.session.get.side_effect = [_app(), _tenant()]
|
||||
ctx = _ctx({"app_id": "x"})
|
||||
AppResolver()(ctx)
|
||||
assert ctx.app.id == "app1"
|
||||
assert ctx.tenant.id == "t1"
|
||||
@@ -1,76 +0,0 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import AppAuthzCheck
|
||||
from controllers.openapi.auth.strategies import AclStrategy, MembershipStrategy
|
||||
from libs.oauth_bearer import SubjectType
|
||||
|
||||
|
||||
def _ctx(*, subject_type, account_id="acc1"):
|
||||
c = Context(required_scope="apps:run")
|
||||
c.subject_type = subject_type
|
||||
c.subject_email = "alice@example.com"
|
||||
c.account_id = account_id
|
||||
c.app = SimpleNamespace(id="app1")
|
||||
c.tenant = SimpleNamespace(id="t1")
|
||||
return c
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.strategies.EnterpriseService")
|
||||
def test_acl_strategy_private_calls_inner_api(ent):
|
||||
ent.WebAppAuth.get_app_access_mode_by_id.return_value = SimpleNamespace(access_mode="private")
|
||||
ent.WebAppAuth.is_user_allowed_to_access_webapp.return_value = True
|
||||
assert AclStrategy().authorize(_ctx(subject_type=SubjectType.ACCOUNT)) is True
|
||||
ent.WebAppAuth.is_user_allowed_to_access_webapp.assert_called_once_with(
|
||||
user_id="acc1",
|
||||
app_id="app1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("access_mode", "subject_type", "expected"),
|
||||
[
|
||||
("public", SubjectType.ACCOUNT, True),
|
||||
("public", SubjectType.EXTERNAL_SSO, True),
|
||||
("sso_verified", SubjectType.ACCOUNT, True),
|
||||
("sso_verified", SubjectType.EXTERNAL_SSO, True),
|
||||
("private_all", SubjectType.ACCOUNT, True),
|
||||
("private_all", SubjectType.EXTERNAL_SSO, False),
|
||||
("private", SubjectType.EXTERNAL_SSO, False),
|
||||
],
|
||||
)
|
||||
@patch("controllers.openapi.auth.strategies.EnterpriseService")
|
||||
def test_acl_strategy_subject_mode_matrix(ent, access_mode, subject_type, expected):
|
||||
"""Step 1 matrix: subject vs access-mode compatibility. No inner API call expected."""
|
||||
ent.WebAppAuth.get_app_access_mode_by_id.return_value = SimpleNamespace(access_mode=access_mode)
|
||||
account_id = "acc1" if subject_type == SubjectType.ACCOUNT else None
|
||||
assert AclStrategy().authorize(_ctx(subject_type=subject_type, account_id=account_id)) is expected
|
||||
ent.WebAppAuth.is_user_allowed_to_access_webapp.assert_not_called()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.strategies.TenantService.account_belongs_to_tenant")
|
||||
@patch("controllers.openapi.auth.strategies.db")
|
||||
def test_membership_strategy_uses_join_lookup(db_mock, member):
|
||||
member.return_value = True
|
||||
assert MembershipStrategy().authorize(_ctx(subject_type=SubjectType.ACCOUNT)) is True
|
||||
member.assert_called_once_with(db_mock.session, "acc1", "t1")
|
||||
|
||||
|
||||
def test_membership_strategy_rejects_external_sso():
|
||||
assert MembershipStrategy().authorize(_ctx(subject_type=SubjectType.EXTERNAL_SSO, account_id=None)) is False
|
||||
|
||||
|
||||
def test_app_authz_check_raises_when_strategy_denies():
|
||||
deny = SimpleNamespace(authorize=lambda c: False)
|
||||
with pytest.raises(Forbidden) as exc:
|
||||
AppAuthzCheck(lambda: deny)(_ctx(subject_type=SubjectType.ACCOUNT))
|
||||
assert "subject_no_app_access" in str(exc.value.description)
|
||||
|
||||
|
||||
def test_app_authz_check_passes_when_strategy_allows():
|
||||
allow = SimpleNamespace(authorize=lambda c: True)
|
||||
AppAuthzCheck(lambda: allow)(_ctx(subject_type=SubjectType.ACCOUNT))
|
||||
@@ -1,83 +0,0 @@
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import Unauthorized
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import BearerCheck
|
||||
from libs.oauth_bearer import (
|
||||
AuthContext,
|
||||
InvalidBearerError,
|
||||
Scope,
|
||||
SubjectType,
|
||||
reset_auth_ctx,
|
||||
try_get_auth_ctx,
|
||||
)
|
||||
|
||||
|
||||
def _ctx(bearer_token: str | None) -> Context:
|
||||
return Context(required_scope="apps:run", bearer_token=bearer_token)
|
||||
|
||||
|
||||
def test_bearer_check_rejects_missing_header():
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(), pytest.raises(Unauthorized):
|
||||
BearerCheck()(_ctx(None))
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.get_authenticator")
|
||||
def test_bearer_check_rejects_unknown_prefix(get_auth):
|
||||
get_auth.return_value.authenticate.side_effect = InvalidBearerError("invalid_bearer")
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context(), pytest.raises(Unauthorized):
|
||||
BearerCheck()(_ctx("xxx_abc"))
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.get_authenticator")
|
||||
def test_bearer_check_populates_context_and_publishes_auth_ctx(get_auth):
|
||||
tok_id = uuid.uuid4()
|
||||
authn = AuthContext(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
subject_email="a@x.com",
|
||||
subject_issuer=None,
|
||||
account_id=None,
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
token_id=tok_id,
|
||||
source="oauth-account",
|
||||
expires_at=datetime.now(UTC),
|
||||
token_hash="hash-1",
|
||||
verified_tenants={},
|
||||
)
|
||||
get_auth.return_value.authenticate.return_value = authn
|
||||
|
||||
app = Flask(__name__)
|
||||
ctx = _ctx("dfoa_abc")
|
||||
with app.test_request_context():
|
||||
BearerCheck()(ctx)
|
||||
try:
|
||||
assert ctx.subject_type == SubjectType.ACCOUNT
|
||||
assert ctx.subject_email == "a@x.com"
|
||||
assert ctx.scopes == frozenset({Scope.FULL})
|
||||
assert ctx.source == "oauth-account"
|
||||
assert ctx.token_id == tok_id
|
||||
assert ctx.token_hash == "hash-1"
|
||||
# BearerCheck must also publish the same identity on the
|
||||
# openapi auth ContextVar so the surface gate + downstream
|
||||
# handlers don't see two different identity sources between
|
||||
# the decorator + pipeline paths. The reset token is parked
|
||||
# on `ctx.auth_ctx_reset_token` for `Pipeline.guard` to
|
||||
# consume in its `finally`.
|
||||
published = try_get_auth_ctx()
|
||||
assert published is authn
|
||||
assert published.client_id == "difyctl"
|
||||
assert ctx.auth_ctx_reset_token is not None
|
||||
finally:
|
||||
# In production `Pipeline.guard` resets the ContextVar; in
|
||||
# this isolated step-level test we reset it ourselves so the
|
||||
# value doesn't leak into the next test on the same worker.
|
||||
assert ctx.auth_ctx_reset_token is not None
|
||||
reset_auth_ctx(ctx.auth_ctx_reset_token)
|
||||
@@ -1,157 +0,0 @@
|
||||
"""Unit tests for WorkspaceMembershipCheck (Layer 0)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import WorkspaceMembershipCheck
|
||||
from libs.oauth_bearer import SubjectType
|
||||
|
||||
|
||||
def _ctx(*, subject_type, account_id, tenant_id, cached_verified_tenants=None, token_hash=None) -> Context:
|
||||
c = Context(required_scope="apps:read")
|
||||
c.subject_type = subject_type
|
||||
c.account_id = account_id
|
||||
c.tenant = SimpleNamespace(id=tenant_id) if tenant_id else None
|
||||
c.cached_verified_tenants = cached_verified_tenants
|
||||
c.token_hash = token_hash
|
||||
return c
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def step():
|
||||
return WorkspaceMembershipCheck()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_skips_when_enterprise_enabled(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = True
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id=str(uuid.uuid4()),
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
cached_verified_tenants={},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
step(ctx) # no raise
|
||||
mock_db.session.execute.assert_not_called()
|
||||
mock_record.assert_not_called()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_skips_for_external_sso(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.EXTERNAL_SSO,
|
||||
account_id=None,
|
||||
tenant_id=str(uuid.uuid4()),
|
||||
cached_verified_tenants={},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
step(ctx) # no raise
|
||||
mock_db.session.execute.assert_not_called()
|
||||
mock_record.assert_not_called()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_uses_cached_ok(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id="a1",
|
||||
tenant_id="t1",
|
||||
cached_verified_tenants={"t1": True},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
step(ctx)
|
||||
mock_db.session.execute.assert_not_called()
|
||||
mock_record.assert_not_called()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_uses_cached_denied(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id="a1",
|
||||
tenant_id="t1",
|
||||
cached_verified_tenants={"t1": False},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
with pytest.raises(Forbidden, match="workspace_membership_revoked"):
|
||||
step(ctx)
|
||||
mock_db.session.execute.assert_not_called()
|
||||
mock_record.assert_not_called()
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_denies_when_no_membership(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
mock_db.session.execute.return_value.scalar_one_or_none.return_value = None
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id="a1",
|
||||
tenant_id="t1",
|
||||
cached_verified_tenants={},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
with pytest.raises(Forbidden, match="workspace_membership_revoked"):
|
||||
step(ctx)
|
||||
mock_record.assert_called_once_with("hash-1", "t1", False)
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_denies_when_account_inactive(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
mock_db.session.execute.side_effect = [
|
||||
MagicMock(scalar_one_or_none=MagicMock(return_value="join-id")),
|
||||
MagicMock(scalar_one_or_none=MagicMock(return_value="banned")),
|
||||
]
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id="a1",
|
||||
tenant_id="t1",
|
||||
cached_verified_tenants={},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
with pytest.raises(Forbidden, match="workspace_membership_revoked"):
|
||||
step(ctx)
|
||||
mock_record.assert_called_once_with("hash-1", "t1", False)
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.steps.dify_config")
|
||||
@patch("libs.oauth_bearer.record_layer0_verdict")
|
||||
@patch("libs.oauth_bearer.db")
|
||||
def test_allows_active_member(mock_db, mock_record, mock_cfg, step):
|
||||
mock_cfg.ENTERPRISE_ENABLED = False
|
||||
mock_db.session.execute.side_effect = [
|
||||
MagicMock(scalar_one_or_none=MagicMock(return_value="join-id")),
|
||||
MagicMock(scalar_one_or_none=MagicMock(return_value="active")),
|
||||
]
|
||||
ctx = _ctx(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
account_id="a1",
|
||||
tenant_id="t1",
|
||||
cached_verified_tenants={},
|
||||
token_hash="hash-1",
|
||||
)
|
||||
step(ctx) # no raise
|
||||
mock_record.assert_called_once_with("hash-1", "t1", True)
|
||||
@@ -1,77 +0,0 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Unauthorized
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import CallerMount
|
||||
from controllers.openapi.auth.strategies import AccountMounter, EndUserMounter
|
||||
from core.app.entities.app_invoke_entities import InvokeFrom
|
||||
from libs.oauth_bearer import SubjectType
|
||||
|
||||
|
||||
def _ctx(*, subject_type, account_id=None, subject_email=None):
|
||||
c = Context(required_scope="apps:run")
|
||||
c.subject_type = subject_type
|
||||
c.account_id = account_id
|
||||
c.subject_email = subject_email
|
||||
c.app = SimpleNamespace(id="app1")
|
||||
c.tenant = SimpleNamespace(id="t1")
|
||||
return c
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.strategies._login_as")
|
||||
@patch("controllers.openapi.auth.strategies.db")
|
||||
def test_account_mounter(db, login):
|
||||
account = SimpleNamespace()
|
||||
db.session.get.return_value = account
|
||||
ctx = _ctx(subject_type=SubjectType.ACCOUNT, account_id="acc1")
|
||||
AccountMounter().mount(ctx)
|
||||
assert ctx.caller is account
|
||||
assert ctx.caller.current_tenant is ctx.tenant
|
||||
assert ctx.caller_kind == "account"
|
||||
login.assert_called_once_with(account)
|
||||
|
||||
|
||||
@patch("controllers.openapi.auth.strategies._login_as")
|
||||
@patch("controllers.openapi.auth.strategies.EndUserService")
|
||||
def test_end_user_mounter(svc, login):
|
||||
eu = SimpleNamespace()
|
||||
svc.get_or_create_end_user_by_type.return_value = eu
|
||||
ctx = _ctx(subject_type=SubjectType.EXTERNAL_SSO, subject_email="a@x.com")
|
||||
EndUserMounter().mount(ctx)
|
||||
svc.get_or_create_end_user_by_type.assert_called_once_with(
|
||||
InvokeFrom.OPENAPI,
|
||||
tenant_id="t1",
|
||||
app_id="app1",
|
||||
user_id="a@x.com",
|
||||
)
|
||||
assert ctx.caller is eu
|
||||
assert ctx.caller_kind == "end_user"
|
||||
|
||||
|
||||
def test_caller_mount_dispatches_by_subject_type():
|
||||
seen = {}
|
||||
|
||||
class Fake:
|
||||
def __init__(self, st, tag):
|
||||
self._st, self._tag = st, tag
|
||||
|
||||
def applies_to(self, st):
|
||||
return st == self._st
|
||||
|
||||
def mount(self, ctx):
|
||||
seen["who"] = self._tag
|
||||
|
||||
cm = CallerMount(
|
||||
Fake(SubjectType.ACCOUNT, "acct"),
|
||||
Fake(SubjectType.EXTERNAL_SSO, "sso"),
|
||||
)
|
||||
cm(_ctx(subject_type=SubjectType.EXTERNAL_SSO))
|
||||
assert seen == {"who": "sso"}
|
||||
|
||||
|
||||
def test_caller_mount_raises_when_none_applies():
|
||||
with pytest.raises(Unauthorized):
|
||||
CallerMount()(_ctx(subject_type=SubjectType.ACCOUNT))
|
||||
@@ -1,25 +0,0 @@
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import ScopeCheck
|
||||
|
||||
|
||||
def _ctx(scopes, required):
|
||||
c = Context(required_scope=required)
|
||||
c.scopes = frozenset(scopes)
|
||||
return c
|
||||
|
||||
|
||||
def test_scope_check_passes_on_full():
|
||||
ScopeCheck()(_ctx({"full"}, "apps:run"))
|
||||
|
||||
|
||||
def test_scope_check_passes_on_explicit_match():
|
||||
ScopeCheck()(_ctx({"apps:run"}, "apps:run"))
|
||||
|
||||
|
||||
def test_scope_check_rejects_when_missing():
|
||||
with pytest.raises(Forbidden) as exc:
|
||||
ScopeCheck()(_ctx({"apps:read"}, "apps:run"))
|
||||
assert "insufficient_scope" in str(exc.value.description)
|
||||
@@ -1,239 +0,0 @@
|
||||
"""Surface gate tests.
|
||||
|
||||
The gate has two attachment forms — decorator (`accept_subjects`) and
|
||||
pipeline step (`SurfaceCheck`) — and both must:
|
||||
- 403 on mismatched subject type with a canonical-path hint
|
||||
- emit `openapi.wrong_surface_denied` once with the right payload
|
||||
- pass-through on match
|
||||
- raise RuntimeError (not 403) if the auth ContextVar is unset — that's
|
||||
a wiring bug, not a user-driven failure
|
||||
|
||||
Identity is published via `libs.oauth_bearer.set_auth_ctx` / read with
|
||||
`try_get_auth_ctx`. Tests wrap the publish in a `_publish_auth_ctx`
|
||||
context manager so the ContextVar resets even when an assertion fails;
|
||||
that keeps state from leaking into the next test on the same worker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from datetime import UTC, datetime
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from controllers.openapi.auth.context import Context
|
||||
from controllers.openapi.auth.steps import SurfaceCheck
|
||||
from controllers.openapi.auth.surface_gate import _coerce_subject_type, accept_subjects, check_surface
|
||||
from libs.oauth_bearer import AuthContext, Scope, SubjectType, reset_auth_ctx, set_auth_ctx
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _publish_auth_ctx(ctx: AuthContext) -> Iterator[None]:
|
||||
token = set_auth_ctx(ctx)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
reset_auth_ctx(token)
|
||||
|
||||
|
||||
def _account_ctx() -> AuthContext:
|
||||
return AuthContext(
|
||||
subject_type=SubjectType.ACCOUNT,
|
||||
subject_email="user@example.com",
|
||||
subject_issuer="dify:account",
|
||||
account_id=uuid.uuid4(),
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
token_id=uuid.uuid4(),
|
||||
source="oauth_account",
|
||||
expires_at=datetime.now(UTC),
|
||||
token_hash="h1",
|
||||
verified_tenants={},
|
||||
)
|
||||
|
||||
|
||||
def _sso_ctx() -> AuthContext:
|
||||
return AuthContext(
|
||||
subject_type=SubjectType.EXTERNAL_SSO,
|
||||
subject_email="sso@partner.com",
|
||||
subject_issuer="https://idp.partner.com",
|
||||
account_id=None,
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({Scope.APPS_RUN, Scope.APPS_READ_PERMITTED_EXTERNAL}),
|
||||
token_id=uuid.uuid4(),
|
||||
source="oauth_external_sso",
|
||||
expires_at=datetime.now(UTC),
|
||||
token_hash="h2",
|
||||
verified_tenants={},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_surface — shared core
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_surface_passes_when_subject_in_accepted():
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/apps"), _publish_auth_ctx(_account_ctx()):
|
||||
check_surface(frozenset({SubjectType.ACCOUNT})) # no raise
|
||||
|
||||
|
||||
def test_check_surface_rejects_on_wrong_subject_and_emits_audit():
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/permitted-external-apps"), _publish_auth_ctx(_account_ctx()):
|
||||
with patch("controllers.openapi.auth.surface_gate.emit_wrong_surface") as emit:
|
||||
with pytest.raises(Forbidden) as exc:
|
||||
check_surface(frozenset({SubjectType.EXTERNAL_SSO}))
|
||||
assert "wrong_surface" in exc.value.description
|
||||
# canonical-path hint should point at the caller's surface,
|
||||
# not the surface they were rejected from
|
||||
assert "/openapi/v1/apps" in exc.value.description
|
||||
emit.assert_called_once()
|
||||
kwargs = emit.call_args.kwargs
|
||||
assert kwargs["subject_type"] == SubjectType.ACCOUNT.value
|
||||
assert kwargs["attempted_path"] == "/openapi/v1/permitted-external-apps"
|
||||
assert kwargs["client_id"] == "difyctl"
|
||||
assert kwargs["token_id"] is not None
|
||||
|
||||
|
||||
def test_check_surface_rejects_sso_on_account_surface():
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/apps"), _publish_auth_ctx(_sso_ctx()):
|
||||
with patch("controllers.openapi.auth.surface_gate.emit_wrong_surface") as emit:
|
||||
with pytest.raises(Forbidden):
|
||||
check_surface(frozenset({SubjectType.ACCOUNT}))
|
||||
kwargs = emit.call_args.kwargs
|
||||
assert kwargs["subject_type"] == SubjectType.EXTERNAL_SSO.value
|
||||
|
||||
|
||||
def test_check_surface_runtime_error_when_auth_ctx_missing():
|
||||
"""Missing auth ContextVar means the bearer layer didn't run — wiring
|
||||
bug, not a user-driven failure. Surface as RuntimeError (loud) so a
|
||||
future refactor doesn't accidentally let a route skip authentication
|
||||
and return a 403 that looks identical to a legitimate wrong-surface
|
||||
deny.
|
||||
"""
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/apps"):
|
||||
with pytest.raises(RuntimeError):
|
||||
check_surface(frozenset({SubjectType.ACCOUNT}))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# @accept_subjects — decorator form
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_app() -> Flask:
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/account-only")
|
||||
@accept_subjects(SubjectType.ACCOUNT)
|
||||
def _account_only():
|
||||
return "ok"
|
||||
|
||||
@app.route("/external-only")
|
||||
@accept_subjects(SubjectType.EXTERNAL_SSO)
|
||||
def _external_only():
|
||||
return "ok"
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def test_accept_subjects_decorator_passes_on_match():
|
||||
app = _make_app()
|
||||
with app.test_request_context("/account-only"), _publish_auth_ctx(_account_ctx()):
|
||||
# Re-route through the decorated function by reaching for view_function
|
||||
view = app.view_functions["_account_only"]
|
||||
assert view() == "ok"
|
||||
|
||||
|
||||
def test_accept_subjects_decorator_403_on_miss():
|
||||
app = _make_app()
|
||||
with app.test_request_context("/external-only"), _publish_auth_ctx(_account_ctx()):
|
||||
view = app.view_functions["_external_only"]
|
||||
with patch("controllers.openapi.auth.surface_gate.emit_wrong_surface"):
|
||||
with pytest.raises(Forbidden):
|
||||
view()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SurfaceCheck — pipeline step form
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pipeline_ctx() -> Context:
|
||||
# SurfaceCheck reads ``request.path`` from Flask's global request — set up
|
||||
# via ``app.test_request_context`` in the calling tests — not from Context.
|
||||
return Context(required_scope=Scope.APPS_RUN)
|
||||
|
||||
|
||||
def test_surface_check_passes_on_match():
|
||||
step = SurfaceCheck(accepted=frozenset({SubjectType.ACCOUNT}))
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/apps/x/run"), _publish_auth_ctx(_account_ctx()):
|
||||
step(_pipeline_ctx()) # no raise
|
||||
|
||||
|
||||
def test_surface_check_rejects_on_miss_and_emits_audit():
|
||||
step = SurfaceCheck(accepted=frozenset({SubjectType.EXTERNAL_SSO}))
|
||||
app = Flask(__name__)
|
||||
with app.test_request_context("/openapi/v1/apps/x/run"), _publish_auth_ctx(_account_ctx()):
|
||||
with patch("controllers.openapi.auth.surface_gate.emit_wrong_surface") as emit:
|
||||
with pytest.raises(Forbidden):
|
||||
step(_pipeline_ctx())
|
||||
emit.assert_called_once()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _coerce_subject_type — normalises whatever sat on ctx.subject_type
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The gate reads `ctx.subject_type` via `getattr(..., None)`, so the value
|
||||
# could be a real enum (happy path), a raw string (e.g. rehydrated from a
|
||||
# dict-shaped context), `None` (attribute missing), or something unexpected
|
||||
# from a buggy upstream. The coercer must collapse all of that to
|
||||
# `SubjectType | None` so `check_surface` can do a clean set-membership
|
||||
# check and emit a clean audit payload.
|
||||
|
||||
|
||||
def test_coerce_subject_type_returns_none_for_none():
|
||||
assert _coerce_subject_type(None) is None
|
||||
|
||||
|
||||
def test_coerce_subject_type_returns_enum_instance_unchanged():
|
||||
# Identity matters: we don't want to round-trip through the string
|
||||
# constructor for an already-valid enum.
|
||||
assert _coerce_subject_type(SubjectType.ACCOUNT) is SubjectType.ACCOUNT
|
||||
assert _coerce_subject_type(SubjectType.EXTERNAL_SSO) is SubjectType.EXTERNAL_SSO
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("account", SubjectType.ACCOUNT),
|
||||
("external_sso", SubjectType.EXTERNAL_SSO),
|
||||
],
|
||||
)
|
||||
def test_coerce_subject_type_parses_known_strings(raw: str, expected: SubjectType):
|
||||
assert _coerce_subject_type(raw) is expected
|
||||
|
||||
|
||||
def test_coerce_subject_type_raises_on_unknown_string():
|
||||
# Unknown strings reach `SubjectType(raw)` which raises ValueError.
|
||||
# We surface that loudly rather than silently returning None, because
|
||||
# a string that *looks* like a subject type but isn't is almost
|
||||
# certainly an upstream bug worth catching.
|
||||
with pytest.raises(ValueError):
|
||||
_coerce_subject_type("not_a_subject")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [123, 1.5, b"account", object(), ["account"], {"account"}])
|
||||
def test_coerce_subject_type_returns_none_for_non_string_non_enum(raw: object):
|
||||
assert _coerce_subject_type(raw) is None
|
||||
@@ -0,0 +1,142 @@
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from werkzeug.exceptions import Forbidden, Unauthorized
|
||||
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from controllers.openapi.auth.verify import (
|
||||
check_acl,
|
||||
check_app_access,
|
||||
check_membership,
|
||||
check_private_app_permission,
|
||||
check_scope,
|
||||
)
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
from models.account import Tenant
|
||||
from models.model import App
|
||||
from services.enterprise.enterprise_service import WebAppAccessMode
|
||||
|
||||
|
||||
def _data(**kwargs) -> AuthData:
|
||||
defaults: dict = {"token_type": TokenType.OAUTH_ACCOUNT, "token_hash": "hash", "scopes": frozenset({Scope.FULL})}
|
||||
defaults.update(kwargs)
|
||||
return AuthData(**defaults)
|
||||
|
||||
|
||||
def test_check_scope_passes_when_required_is_none():
|
||||
check_scope(_data(required_scope=None))
|
||||
|
||||
|
||||
def test_check_scope_passes_when_full_in_scopes():
|
||||
check_scope(_data(required_scope=Scope.APPS_RUN, scopes=frozenset({Scope.FULL})))
|
||||
|
||||
|
||||
def test_check_scope_passes_when_exact_scope_present():
|
||||
check_scope(_data(required_scope=Scope.APPS_RUN, scopes=frozenset({Scope.APPS_RUN})))
|
||||
|
||||
|
||||
def test_check_scope_raises_forbidden_when_scope_missing():
|
||||
with pytest.raises(Forbidden, match="insufficient_scope"):
|
||||
check_scope(_data(required_scope=Scope.APPS_RUN, scopes=frozenset({Scope.APPS_READ})))
|
||||
|
||||
|
||||
def test_check_membership_raises_unauthorized_when_tenant_none():
|
||||
with pytest.raises(Unauthorized):
|
||||
check_membership(_data(tenant=None))
|
||||
|
||||
|
||||
def test_check_membership_calls_check_workspace_membership():
|
||||
tenant = MagicMock(spec=Tenant)
|
||||
tenant.id = "tenant-1"
|
||||
data = _data(
|
||||
account_id=uuid.uuid4(),
|
||||
token_hash="myhash",
|
||||
tenants={"tenant-1": True},
|
||||
tenant=tenant,
|
||||
)
|
||||
with patch("controllers.openapi.auth.verify.check_workspace_membership") as mock_cwm:
|
||||
check_membership(data)
|
||||
mock_cwm.assert_called_once_with(
|
||||
account_id=data.account_id,
|
||||
tenant_id="tenant-1",
|
||||
token_hash="myhash",
|
||||
membership_cache=data.tenants,
|
||||
)
|
||||
|
||||
|
||||
def test_check_app_access_passes_when_tenant_none():
|
||||
check_app_access(_data(tenant=None))
|
||||
|
||||
|
||||
def test_check_app_access_passes_when_member():
|
||||
tenant = MagicMock(spec=Tenant)
|
||||
tenant.id = "t1"
|
||||
data = _data(account_id=uuid.uuid4(), tenant=tenant)
|
||||
with patch("controllers.openapi.auth.verify.TenantService.account_belongs_to_tenant", return_value=True):
|
||||
check_app_access(data)
|
||||
|
||||
|
||||
def test_check_app_access_raises_when_not_member():
|
||||
tenant = MagicMock(spec=Tenant)
|
||||
tenant.id = "t1"
|
||||
data = _data(account_id=uuid.uuid4(), tenant=tenant)
|
||||
with patch("controllers.openapi.auth.verify.TenantService.account_belongs_to_tenant", return_value=False):
|
||||
with pytest.raises(Forbidden, match="subject_no_app_access"):
|
||||
check_app_access(data)
|
||||
|
||||
|
||||
def test_check_acl_raises_when_app_or_mode_missing():
|
||||
with pytest.raises(Forbidden):
|
||||
check_acl(_data(app=None, app_access_mode=None))
|
||||
|
||||
|
||||
def test_check_acl_account_allowed_for_public():
|
||||
app = MagicMock(spec=App)
|
||||
data = _data(token_type=TokenType.OAUTH_ACCOUNT, app=app, app_access_mode=WebAppAccessMode.PUBLIC)
|
||||
check_acl(data)
|
||||
|
||||
|
||||
def test_check_acl_external_sso_blocked_for_private():
|
||||
app = MagicMock(spec=App)
|
||||
data = _data(
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
app=app,
|
||||
app_access_mode=WebAppAccessMode.PRIVATE,
|
||||
)
|
||||
with pytest.raises(Forbidden, match="subject_not_allowed_for_access_mode"):
|
||||
check_acl(data)
|
||||
|
||||
|
||||
def test_check_acl_external_sso_allowed_for_sso_verified():
|
||||
app = MagicMock(spec=App)
|
||||
data = _data(
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
app=app,
|
||||
app_access_mode=WebAppAccessMode.SSO_VERIFIED,
|
||||
)
|
||||
check_acl(data)
|
||||
|
||||
|
||||
def test_check_private_app_permission_raises_when_app_none():
|
||||
with pytest.raises(Forbidden):
|
||||
check_private_app_permission(_data(app=None))
|
||||
|
||||
|
||||
def test_check_private_app_permission_raises_when_user_not_allowed():
|
||||
app = MagicMock(spec=App)
|
||||
app.id = "app-1"
|
||||
data = _data(account_id=uuid.uuid4(), app=app)
|
||||
target = "controllers.openapi.auth.verify.EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp"
|
||||
with patch(target, return_value=False):
|
||||
with pytest.raises(Forbidden, match="user_not_allowed_for_private_app"):
|
||||
check_private_app_permission(data)
|
||||
|
||||
|
||||
def test_check_private_app_permission_passes_when_allowed():
|
||||
app = MagicMock(spec=App)
|
||||
app.id = "app-1"
|
||||
data = _data(account_id=uuid.uuid4(), app=app)
|
||||
target = "controllers.openapi.auth.verify.EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp"
|
||||
with patch(target, return_value=True):
|
||||
check_private_app_permission(data)
|
||||
@@ -1,20 +1,36 @@
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
from controllers.openapi import bp as openapi_bp
|
||||
from controllers.openapi.auth.pipeline import Pipeline
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from controllers.openapi.auth.pipeline import PipelineRouter
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
|
||||
|
||||
def _stub_execute(self, args, kwargs, view, *, scope=None, allowed_token_types=None, edition=None):
|
||||
"""Bypass all auth logic; inject minimal AuthData and call the view directly."""
|
||||
kwargs["auth_data"] = AuthData(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
account_id=uuid.uuid4(),
|
||||
token_hash="test",
|
||||
token_id=uuid.uuid4(),
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
required_scope=scope,
|
||||
)
|
||||
return view(*args, **kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bypass_pipeline(monkeypatch):
|
||||
"""Stub Pipeline.run so endpoint decoration does not invoke real auth.
|
||||
"""Stub PipelineRouter._execute so endpoints skip real auth at request time.
|
||||
|
||||
Module-level @OAUTH_BEARER_PIPELINE.guard(...) captures the real
|
||||
pipeline at import time; mocking the module attribute does not undo
|
||||
that. Patching Pipeline.run on the class is the bypass that actually
|
||||
works.
|
||||
Module-level @auth_router.guard(...) captures the real router at import
|
||||
time — patching guard itself does nothing. Patching _execute on the class
|
||||
is the seam that fires at request time.
|
||||
"""
|
||||
monkeypatch.setattr(Pipeline, "run", lambda self, ctx: None)
|
||||
monkeypatch.setattr(PipelineRouter, "_execute", _stub_execute)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@@ -86,7 +86,7 @@ def test_subject_match_for_account_filters_by_account_id():
|
||||
"""Account subject scopes queries via account_id."""
|
||||
import uuid as _uuid
|
||||
|
||||
from libs.oauth_bearer import AuthContext, SubjectType
|
||||
from libs.oauth_bearer import AuthContext, SubjectType, TokenType
|
||||
from services.oauth_device_flow import subject_match_clauses
|
||||
|
||||
aid = _uuid.uuid4()
|
||||
@@ -98,7 +98,7 @@ def test_subject_match_for_account_filters_by_account_id():
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({"full"}),
|
||||
token_id=_uuid.uuid4(),
|
||||
source="oauth_account",
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
expires_at=None,
|
||||
token_hash="h1",
|
||||
verified_tenants={},
|
||||
@@ -116,7 +116,7 @@ def test_subject_match_for_external_sso_filters_by_email_and_issuer():
|
||||
"""
|
||||
import uuid as _uuid
|
||||
|
||||
from libs.oauth_bearer import AuthContext, SubjectType
|
||||
from libs.oauth_bearer import AuthContext, SubjectType, TokenType
|
||||
from services.oauth_device_flow import subject_match_clauses
|
||||
|
||||
ctx = AuthContext(
|
||||
@@ -127,7 +127,7 @@ def test_subject_match_for_external_sso_filters_by_email_and_issuer():
|
||||
client_id="difyctl",
|
||||
scopes=frozenset({"apps:run"}),
|
||||
token_id=_uuid.uuid4(),
|
||||
source="oauth_external_sso",
|
||||
token_type=TokenType.OAUTH_EXTERNAL_SSO,
|
||||
expires_at=None,
|
||||
token_hash="h1",
|
||||
verified_tenants={},
|
||||
|
||||
@@ -57,7 +57,11 @@ def test_stop_task_endpoint_registered(openapi_app):
|
||||
|
||||
|
||||
def test_stop_task_calls_queue_manager_and_graph_engine(app, bypass_pipeline, monkeypatch):
|
||||
import uuid
|
||||
|
||||
from controllers.openapi.app_run import AppRunTaskStopApi
|
||||
from controllers.openapi.auth.data import AuthData
|
||||
from libs.oauth_bearer import Scope, TokenType
|
||||
|
||||
queue_mock = Mock()
|
||||
graph_mock = Mock()
|
||||
@@ -69,15 +73,23 @@ def test_stop_task_calls_queue_manager_and_graph_engine(app, bypass_pipeline, mo
|
||||
monkeypatch.setattr(run_module, "GraphEngineManager", graph_mock)
|
||||
monkeypatch.setattr(run_module, "redis_client", object())
|
||||
|
||||
auth_data = AuthData.model_construct(
|
||||
token_type=TokenType.OAUTH_ACCOUNT,
|
||||
account_id=uuid.uuid4(),
|
||||
token_hash="test",
|
||||
scopes=frozenset({Scope.FULL}),
|
||||
app=SimpleNamespace(id="app-1", tenant_id="t-1"),
|
||||
caller=SimpleNamespace(id="acct-1"),
|
||||
caller_kind="account",
|
||||
)
|
||||
|
||||
api = AppRunTaskStopApi()
|
||||
with app.test_request_context("/openapi/v1/apps/app-1/tasks/task-1/stop", method="POST"):
|
||||
result = api.post.__wrapped__(
|
||||
api,
|
||||
app_id="app-1",
|
||||
task_id="task-1",
|
||||
app_model=SimpleNamespace(id="app-1", tenant_id="t-1"),
|
||||
caller=SimpleNamespace(id="acct-1"),
|
||||
caller_kind="account",
|
||||
auth_data=auth_data,
|
||||
)
|
||||
|
||||
queue_mock.set_stop_flag_no_user_check.assert_called_once_with("task-1")
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user