Compare commits
211
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
29c7469206 | ||
|
|
19858061e6 | ||
|
|
0cc2d69177 | ||
|
|
f6bdd9c673 | ||
|
|
795febe737 | ||
|
|
fa4e4ac34a | ||
|
|
9476581992 | ||
|
|
06ad009c48 | ||
|
|
bb41da013b | ||
|
|
997993196b | ||
|
|
dda75e4192 | ||
|
|
410e827263 | ||
|
|
c457182f4d | ||
|
|
d2596222d7 | ||
|
|
df17f15bcb | ||
|
|
0a777fe7ff | ||
|
|
258a9fbfd4 | ||
|
|
6e21df1bb1 | ||
|
|
64b7cfce24 | ||
|
|
447a5b1076 | ||
|
|
eaa41ff318 | ||
|
|
b56ac4af4d | ||
|
|
2a0661a769 | ||
|
|
00fe96325d | ||
|
|
a3c18c561e | ||
|
|
1835e8ef31 | ||
|
|
2b8e40af32 | ||
|
|
625ed2a0e7 | ||
|
|
45711eb6dd | ||
|
|
fd703739b5 | ||
|
|
82c8741c73 | ||
|
|
78023bf50d | ||
|
|
0ad985515b | ||
|
|
da2177c7b2 | ||
|
|
8d293023cc | ||
|
|
6736f4005b | ||
|
|
bd8f0104d6 | ||
|
|
018fe7a9d3 | ||
|
|
d5788cc019 | ||
|
|
4da4fa72cd | ||
|
|
8853ed99a5 | ||
|
|
34a5ca191b | ||
|
|
1271cfcf3f | ||
|
|
2a0b1a8a63 | ||
|
|
4a2e92f43f | ||
|
|
84889b88ec | ||
|
|
6f034d2c93 | ||
|
|
f91ea49414 | ||
|
|
62db37c403 | ||
|
|
de07730548 | ||
|
|
9b2aa8b216 | ||
|
|
8ee3a7eabc | ||
|
|
0f9fffb7a2 | ||
|
|
187501f53e | ||
|
|
03fad2e041 | ||
|
|
acb5ee29e1 | ||
|
|
c5aadfe557 | ||
|
|
c76ff4c38c | ||
|
|
4b355b7039 | ||
|
|
a3654d6a7d | ||
|
|
855464e7b4 | ||
|
|
05a25943e7 | ||
|
|
c5d471dbdd | ||
|
|
ba449890a9 | ||
|
|
072953a83a | ||
|
|
7b95e2a75f | ||
|
|
01efc6eecb | ||
|
|
a4c7261bf9 | ||
|
|
0aa04f610e | ||
|
|
66afa3bf4a | ||
|
|
c0b991c5f9 | ||
|
|
d2aea76dec | ||
|
|
bc21972878 | ||
|
|
389a7608bf | ||
|
|
3abbab4798 | ||
|
|
84f3c410c4 | ||
|
|
04f9026724 | ||
|
|
3f0f57c594 | ||
|
|
79a5b31bc6 | ||
|
|
f35655a5bb | ||
|
|
97489d5a44 | ||
|
|
e9bd8741f4 | ||
|
|
33acaa558e | ||
|
|
b38caf3cdb | ||
|
|
cb2b36f1aa | ||
|
|
2d15743b96 | ||
|
|
d5e5227c6e | ||
|
|
9b432d0d29 | ||
|
|
5216ee1d20 | ||
|
|
ba8ce362d7 | ||
|
|
b95d313128 | ||
|
|
45ac70132c | ||
|
|
6e772259bb | ||
|
|
af465fb527 | ||
|
|
62bbc0dbeb | ||
|
|
ed1289281d | ||
|
|
e37f32e4a0 | ||
|
|
150613cc05 | ||
|
|
a5194ff17b | ||
|
|
cdaf5d4562 | ||
|
|
145b0e622e | ||
|
|
4bab62d458 | ||
|
|
b99f8941bb | ||
|
|
32d2594877 | ||
|
|
7b188287da | ||
|
|
f6db23444d | ||
|
|
a5c989d82b | ||
|
|
2622b1ce0a | ||
|
|
1e672530c4 | ||
|
|
852169d99c | ||
|
|
d7860571aa | ||
|
|
3683831738 | ||
|
|
f5024f5440 | ||
|
|
4f8477427e | ||
|
|
8d406aec31 | ||
|
|
f8754286ec | ||
|
|
7da6b2ce36 | ||
|
|
aa718add24 | ||
|
|
fcc1d39f5e | ||
|
|
861831f176 | ||
|
|
d333aa31ea | ||
|
|
97e6a14c13 | ||
|
|
6a338654fb | ||
|
|
8f075c82bc | ||
|
|
28d17da3b1 | ||
|
|
891b2dc537 | ||
|
|
75f7069541 | ||
|
|
b7193d1cba | ||
|
|
ef0115d340 | ||
|
|
3afb9b3230 | ||
|
|
0862641533 | ||
|
|
cbd7872520 | ||
|
|
23882a704e | ||
|
|
1bd5254ea7 | ||
|
|
e13069dbe3 | ||
|
|
edd3d4a218 | ||
|
|
79751f7622 | ||
|
|
cb64446fa3 | ||
|
|
e77ee82526 | ||
|
|
b3f163fb0f | ||
|
|
873ef2b592 | ||
|
|
43d725a1ce | ||
|
|
c8ccfba960 | ||
|
|
eee8d6cf7b | ||
|
|
11016563b1 | ||
|
|
b7edc5ef0c | ||
|
|
f93a5b95c6 | ||
|
|
6022939bf0 | ||
|
|
30df1433cc | ||
|
|
aa7b65c602 | ||
|
|
ba24745c07 | ||
|
|
38ea0de907 | ||
|
|
bbc31a3403 | ||
|
|
605c0fe02b | ||
|
|
15674a9db6 | ||
|
|
9a2d039e93 | ||
|
|
58d1dd8873 | ||
|
|
5ea884f799 | ||
|
|
2ec34b2cfb | ||
|
|
fa65f45bff | ||
|
|
63ca2b94b5 | ||
|
|
a7ef41a8f5 | ||
|
|
5c6372d2f7 | ||
|
|
5fd06fafe0 | ||
|
|
af4b65b295 | ||
|
|
6e5fc1081b | ||
|
|
0e84ae7338 | ||
|
|
5a792945f5 | ||
|
|
2f3c785f27 | ||
|
|
38045078dc | ||
|
|
83f40b85f4 | ||
|
|
aff8a49bbf | ||
|
|
b0329cd53c | ||
|
|
2a008423d8 | ||
|
|
f962c9e47a | ||
|
|
af7e59de7c | ||
|
|
61b07ab17d | ||
|
|
a7aff83d52 | ||
|
|
9de7e0fe44 | ||
|
|
50341357b3 | ||
|
|
96e34e7b24 | ||
|
|
1aea3460af | ||
|
|
48e536ba39 | ||
|
|
33fe0dfd60 | ||
|
|
a8ffc93a5a | ||
|
|
14ac8dfbf6 | ||
|
|
b820ccf086 | ||
|
|
415c0db22e | ||
|
|
6ad1a66a50 | ||
|
|
2f2d7c26d4 | ||
|
|
18f4640ad1 | ||
|
|
f4ad6ce978 | ||
|
|
e9c0e9de9e | ||
|
|
8a33161080 | ||
|
|
752af8a270 | ||
|
|
678ce2ab37 | ||
|
|
b737833e2a | ||
|
|
872b6906f2 | ||
|
|
d876f6dba5 | ||
|
|
cd98193234 | ||
|
|
bfffcb7d0f | ||
|
|
d66de8a47b | ||
|
|
1a2ba2237d | ||
|
|
8ff92c9ef8 | ||
|
|
511dbb9974 | ||
|
|
4381ec8fee | ||
|
|
cb41fb3e76 | ||
|
|
7dfd84472f | ||
|
|
62bdbc8628 | ||
|
|
85cc183501 | ||
|
|
5af8f6af4d |
@@ -32,12 +32,11 @@ Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance.
|
||||
- `e2e/` uses Cucumber for scenarios and Playwright as the browser layer.
|
||||
- `DifyWorld` is the per-scenario context object. Type `this` as `DifyWorld` and use `async function`, not arrow functions.
|
||||
- Keep glue organized by capability under `e2e/features/step-definitions/`; use `common/` only for broadly reusable steps.
|
||||
- Browser session behavior comes from `features/support/hooks.ts`:
|
||||
- default: authenticated session with shared storage state
|
||||
- `@unauthenticated`: clean browser context
|
||||
- `@authenticated`: readability/selective-run tag only unless implementation changes
|
||||
- `@fresh`: only for `e2e:full*` flows
|
||||
- Treat `e2e/AGENTS.md`, `features/support/hooks.ts`, and the Cucumber configuration as the owners of current session and tag semantics. Verify them when behavior depends on session state instead of copying a tag inventory into this skill.
|
||||
- Do not import Playwright Test runner patterns that bypass the current Cucumber + `DifyWorld` architecture unless the task is explicitly about changing that architecture.
|
||||
- Perform the behavior under test through Playwright. APIs are allowed for setup, seed preparation, persistence polling, and cleanup, but ordinary Console JSON and representable multipart operations must use the scenario- or process-owned generated oRPC client with request and response validation enabled. Keep the setup/cleanup API identity independent from an unauthenticated or logged-out behavior browser.
|
||||
- Consume generated operations directly. Do not add one-to-one API wrappers, handwritten endpoint URLs, response DTO casts, duplicate schemas, global mutable clients, or TanStack Query caching in Cucumber. Keep helpers only for real fixture construction, multi-operation orchestration, invariants, polling, derived test views, or protocol adapters.
|
||||
- Keep SSE, binary, redirect-only, external-service, and readiness exceptions centralized under their protocol owner. A contract mismatch must fail and be fixed at the backend schema owner followed by regeneration; never weaken validation to make E2E pass.
|
||||
|
||||
## Workflow
|
||||
|
||||
@@ -66,7 +65,7 @@ Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance.
|
||||
- If a product element has real user-facing semantics but no accessible name, prefer fixing that accessible contract over adding a test id.
|
||||
5. Validate narrowly.
|
||||
- Run the narrowest tagged scenario or flow that exercises the change.
|
||||
- Run `vpr lint --fix --quiet` from the repository root and `pnpm -C e2e type-check`.
|
||||
- Run the package-required static checks documented in `e2e/AGENTS.md`.
|
||||
- Broaden verification only when the change affects hooks, tags, setup, or shared step semantics.
|
||||
|
||||
## Review Checklist
|
||||
@@ -77,6 +76,8 @@ Keep this skill focused on Cucumber, Playwright, and package-level E2E guidance.
|
||||
- Are locators user-facing and assertions web-first?
|
||||
- Does the change introduce hidden coupling across scenarios, tags, or instance state?
|
||||
- Does it document or implement behavior that differs from the real hooks or configuration?
|
||||
- Does setup/cleanup use the generated client directly, with any remaining helper owning more than a one-to-one endpoint forward?
|
||||
- Is every raw HTTP call a documented protocol or infrastructure exception rather than an ordinary Console operation?
|
||||
|
||||
Lead findings with correctness, flake risk, and architecture drift.
|
||||
|
||||
|
||||
@@ -9,12 +9,11 @@ Use this skill for Vitest work under `web/` and `packages/dify-ui/`. Do not use
|
||||
|
||||
## Required Source
|
||||
|
||||
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill defines the execution workflow and must not add requirements that conflict with or duplicate that guide.
|
||||
Before writing, changing, or reviewing frontend tests, read `web/docs/test.md` completely. It is the single source of truth. This skill provides an execution checklist and must not redefine or extend that policy.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the source, its behavior owner, nearby specs, and relevant public dependencies.
|
||||
1. Identify whether the contract belongs in `web/`, Dify UI Browser Mode, or a styled Storybook test.
|
||||
1. Apply the canonical guide to decide whether a test is needed and choose its boundary.
|
||||
1. For a behavior change or bug fix, write or identify the failing scenario first when practical.
|
||||
1. Implement one coherent scenario at a time and run the focused spec before expanding scope.
|
||||
@@ -33,4 +32,4 @@ vp test run path/to/spec-or-directory
|
||||
vp test run --project unit src/path/to/spec
|
||||
```
|
||||
|
||||
For styled Dify UI behavior, run `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
|
||||
Run Dify UI Storybook tests with `vp test --project storybook --run`. Run broader checks only after the focused behavior passes.
|
||||
|
||||
+17
-44
@@ -8,7 +8,6 @@
|
||||
|
||||
# Lint bulk suppression baselines.
|
||||
/oxlint-suppressions.json
|
||||
/eslint-suppressions.json
|
||||
|
||||
# CODEOWNERS file
|
||||
/.github/CODEOWNERS @laipz8200 @crazywoola
|
||||
@@ -33,31 +32,9 @@
|
||||
# Backend (default owner, more specific rules below will override)
|
||||
/api/ @QuantumGhost
|
||||
|
||||
# Backend - MCP
|
||||
/api/core/mcp/ @Nov1c444
|
||||
/api/core/entities/mcp_provider.py @Nov1c444
|
||||
/api/services/tools/mcp_tools_manage_service.py @Nov1c444
|
||||
/api/controllers/mcp/ @Nov1c444
|
||||
/api/controllers/console/app/mcp_server.py @Nov1c444
|
||||
|
||||
# Backend - Tests
|
||||
/api/tests/ @laipz8200 @QuantumGhost
|
||||
|
||||
/api/tests/**/*mcp* @Nov1c444
|
||||
|
||||
# Backend - Workflow - Engine (Core graph execution engine)
|
||||
/api/core/workflow/graph_engine/ @laipz8200 @QuantumGhost
|
||||
/api/core/workflow/runtime/ @laipz8200 @QuantumGhost
|
||||
/api/core/workflow/graph/ @laipz8200 @QuantumGhost
|
||||
/api/core/workflow/graph_events/ @laipz8200 @QuantumGhost
|
||||
/api/core/workflow/node_events/ @laipz8200 @QuantumGhost
|
||||
|
||||
# Backend - Workflow - Nodes (Agent, Iteration, Loop, LLM)
|
||||
/api/core/workflow/nodes/agent/ @Nov1c444
|
||||
/api/core/workflow/nodes/iteration/ @Nov1c444
|
||||
/api/core/workflow/nodes/loop/ @Nov1c444
|
||||
/api/core/workflow/nodes/llm/ @Nov1c444
|
||||
|
||||
# Backend - RAG (Retrieval Augmented Generation)
|
||||
/api/core/rag/ @JohnJyong
|
||||
/api/services/rag_pipeline/ @JohnJyong
|
||||
@@ -111,7 +88,6 @@
|
||||
/api/core/app/layers/trigger_post_layer.py @CourTeous33
|
||||
/api/services/trigger/ @CourTeous33
|
||||
/api/models/trigger.py @CourTeous33
|
||||
/api/fields/workflow_trigger_fields.py @CourTeous33
|
||||
/api/repositories/workflow_trigger_log_repository.py @CourTeous33
|
||||
/api/repositories/sqlalchemy_workflow_trigger_log_repository.py @CourTeous33
|
||||
/api/libs/schedule_utils.py @CourTeous33
|
||||
@@ -136,11 +112,11 @@
|
||||
/api/controllers/console/billing/ @hj24 @zyssyz123
|
||||
|
||||
# Backend - Enterprise
|
||||
/api/configs/enterprise/ @GarfieldDai @GareArc
|
||||
/api/services/enterprise/ @GarfieldDai @GareArc
|
||||
/api/services/feature_service.py @GarfieldDai @GareArc
|
||||
/api/controllers/console/feature.py @GarfieldDai @GareArc
|
||||
/api/controllers/web/feature.py @GarfieldDai @GareArc
|
||||
/api/configs/enterprise/ @GareArc
|
||||
/api/services/enterprise/ @GareArc
|
||||
/api/services/feature_service.py @GareArc
|
||||
/api/controllers/console/feature.py @GareArc
|
||||
/api/controllers/web/feature.py @GareArc
|
||||
|
||||
# Backend - Database Migrations
|
||||
/api/migrations/ @snakevash @laipz8200 @MRZHUH
|
||||
@@ -153,7 +129,6 @@
|
||||
|
||||
# Frontend - Platform and Features
|
||||
/web/config/ @lyzno1
|
||||
/web/contract/ @lyzno1
|
||||
/web/env.ts @lyzno1
|
||||
/web/features/ @lyzno1
|
||||
/web/hooks/ @lyzno1
|
||||
@@ -212,7 +187,6 @@
|
||||
/web/app/components/rag-pipeline/store/ @iamjoel @zxhlyh
|
||||
|
||||
# Frontend - RAG - Documents List
|
||||
/web/app/components/datasets/documents/list.tsx @iamjoel @WTW0313
|
||||
/web/app/components/datasets/documents/create-from-pipeline/ @iamjoel @WTW0313
|
||||
|
||||
# Frontend - RAG - Segments List
|
||||
@@ -231,22 +205,22 @@
|
||||
/web/app/components/plugins/marketplace/ @iamjoel @Yessenia-d
|
||||
|
||||
# Frontend - Login and Registration
|
||||
/web/app/signin/ @douxc @iamjoel
|
||||
/web/app/signup/ @douxc @iamjoel
|
||||
/web/app/reset-password/ @douxc @iamjoel
|
||||
/web/app/install/ @douxc @iamjoel
|
||||
/web/app/init/ @douxc @iamjoel
|
||||
/web/app/forgot-password/ @douxc @iamjoel
|
||||
/web/app/account/ @douxc @iamjoel
|
||||
/web/app/signin/ @iamjoel
|
||||
/web/app/signup/ @iamjoel
|
||||
/web/app/reset-password/ @iamjoel
|
||||
/web/app/install/ @iamjoel
|
||||
/web/app/init/ @iamjoel
|
||||
/web/app/forgot-password/ @iamjoel
|
||||
/web/app/account/ @iamjoel
|
||||
|
||||
# Frontend - Service Authentication
|
||||
/web/service/base.ts @douxc @iamjoel
|
||||
/web/service/base.ts @iamjoel
|
||||
|
||||
# Frontend - WebApp Authentication and Access Control
|
||||
/web/app/(shareLayout)/components/ @douxc @iamjoel
|
||||
/web/app/(shareLayout)/webapp-signin/ @douxc @iamjoel
|
||||
/web/app/(shareLayout)/webapp-reset-password/ @douxc @iamjoel
|
||||
/web/app/components/app/app-access-control/ @douxc @iamjoel
|
||||
/web/app/(shareLayout)/components/ @iamjoel
|
||||
/web/app/(shareLayout)/webapp-signin/ @iamjoel
|
||||
/web/app/(shareLayout)/webapp-reset-password/ @iamjoel
|
||||
/web/app/components/app/app-access-control/ @iamjoel
|
||||
|
||||
# Frontend - Explore Page
|
||||
/web/app/components/explore/ @CodingOnStar @iamjoel
|
||||
@@ -265,7 +239,6 @@
|
||||
/web/app/components/base/**/*.spec.tsx @hyoban @CodingOnStar
|
||||
|
||||
# Frontend - Utils and Hooks
|
||||
/web/utils/classnames.ts @iamjoel @zxhlyh
|
||||
/web/utils/time.ts @iamjoel @zxhlyh
|
||||
/web/utils/format.ts @iamjoel @zxhlyh
|
||||
/web/utils/clipboard.ts @iamjoel @zxhlyh
|
||||
|
||||
@@ -47,9 +47,6 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Run dify config tests
|
||||
run: uv run --project api pytest api/tests/unit_tests/configs/test_env_consistency.py
|
||||
|
||||
- name: Run Unit Tests
|
||||
run: |
|
||||
uv run --project api pytest \
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
steps:
|
||||
- id: skip_check
|
||||
continue-on-error: true
|
||||
uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
|
||||
uses: fkirc/skip-duplicate-actions@b974a9395958c231af965b70070979a577efa578 # v5.3.2
|
||||
with:
|
||||
cancel_others: 'true'
|
||||
concurrent_skipping: same_content_newer
|
||||
@@ -72,6 +72,7 @@ jobs:
|
||||
- 'docker/volumes/sandbox/conf/**'
|
||||
cli:
|
||||
- 'cli/**'
|
||||
- 'packages/contracts/**'
|
||||
- 'packages/tsconfig/**'
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
@@ -80,7 +81,6 @@ jobs:
|
||||
- '.npmrc'
|
||||
- '.nvmrc'
|
||||
- '.github/workflows/cli-tests.yml'
|
||||
- '.github/workflows/cli-docker-build.yml'
|
||||
- '.github/actions/setup-web/**'
|
||||
web:
|
||||
- 'web/**'
|
||||
@@ -105,6 +105,7 @@ jobs:
|
||||
- 'docker/docker-compose.middleware.yaml'
|
||||
- 'docker/envs/middleware.env.example'
|
||||
- '.github/workflows/web-e2e.yml'
|
||||
- '.github/workflows/main-ci.yml'
|
||||
- '.github/actions/setup-web/**'
|
||||
vdb:
|
||||
- 'api/core/rag/datasource/**'
|
||||
@@ -334,6 +335,8 @@ jobs:
|
||||
- check-changes
|
||||
if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed == 'true'
|
||||
uses: ./.github/workflows/web-e2e.yml
|
||||
with:
|
||||
run-external-runtime: false
|
||||
secrets: inherit
|
||||
|
||||
web-e2e-skip:
|
||||
|
||||
@@ -26,19 +26,30 @@ jobs:
|
||||
external_e2e:
|
||||
- 'e2e/features/agent-v2/**'
|
||||
- 'e2e/features/step-definitions/agent-v2/**'
|
||||
- 'e2e/features/step-definitions/common/**'
|
||||
- 'e2e/features/support/**'
|
||||
- 'e2e/fixtures/auth.ts'
|
||||
- 'e2e/fixtures/test-materials/**'
|
||||
- 'e2e/scripts/**'
|
||||
- 'e2e/support/**'
|
||||
- 'e2e/cucumber.config.ts'
|
||||
- 'e2e/package.json'
|
||||
- 'e2e/test-env.ts'
|
||||
- 'e2e/tsconfig.json'
|
||||
- 'e2e/tsx-register.js'
|
||||
- 'package.json'
|
||||
- 'pnpm-lock.yaml'
|
||||
- '.nvmrc'
|
||||
- '.github/workflows/post-merge.yml'
|
||||
- '.github/workflows/web-e2e.yml'
|
||||
- '.github/actions/setup-web/**'
|
||||
- 'docker/docker-compose.middleware.yaml'
|
||||
- 'docker/envs/middleware.env.example'
|
||||
- 'dify-agent/**'
|
||||
- 'dify-agent-runtime/**'
|
||||
- 'api/pyproject.toml'
|
||||
- 'api/uv.lock'
|
||||
- 'api/tests/integration_tests/.env.example'
|
||||
- 'api/clients/agent_backend/**'
|
||||
- 'api/core/app/apps/agent_app/**'
|
||||
- 'api/core/workflow/nodes/agent_v2/**'
|
||||
@@ -48,8 +59,13 @@ jobs:
|
||||
- 'api/services/plugin/**'
|
||||
- 'api/core/tools/**'
|
||||
- 'api/services/tools/**'
|
||||
- 'packages/contracts/package.json'
|
||||
- 'packages/contracts/generated/api/console/agent/**'
|
||||
- 'packages/contracts/generated/api/console/apps/**'
|
||||
- 'packages/contracts/generated/api/console/datasets/**'
|
||||
- 'packages/contracts/generated/api/console/orpc.gen.ts'
|
||||
- 'packages/contracts/generated/api/console/workspaces/**'
|
||||
- 'packages/contracts/generated/api/service/**'
|
||||
- 'web/features/agent-v2/**'
|
||||
- 'web/app/(commonLayout)/agents/**'
|
||||
- 'web/app/(commonLayout)/@detailSidebar/agents/**'
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
@@ -51,13 +51,13 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
|
||||
- name: Run golangci-lint
|
||||
uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v6.5.0
|
||||
uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v6.5.0
|
||||
with:
|
||||
working-directory: dify-agent-runtime
|
||||
version: latest
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: dify-agent-runtime/go.mod
|
||||
cache-dependency-path: dify-agent-runtime/go.sum
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Use Node.js
|
||||
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: ''
|
||||
|
||||
@@ -158,7 +158,7 @@ jobs:
|
||||
|
||||
- name: Run Claude Code for Translation Sync
|
||||
if: steps.context.outputs.CHANGED_FILES != ''
|
||||
uses: anthropics/claude-code-action@e90deca47693f9457b72f2b53c17d7c445a87342 # v1.0.171
|
||||
uses: anthropics/claude-code-action@af0559ee4f514d1ef21826982bed13f7edc3c35e # v1.0.178
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -4,9 +4,9 @@ on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
run-external-runtime:
|
||||
required: false
|
||||
description: Run only the prepared and external runtime suite instead of the core suites.
|
||||
required: true
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -19,6 +19,7 @@ jobs:
|
||||
test:
|
||||
name: Web Full-Stack E2E
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
timeout-minutes: 120
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -45,6 +46,7 @@ jobs:
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Run E2E support unit tests
|
||||
if: ${{ !inputs.run-external-runtime }}
|
||||
working-directory: ./e2e
|
||||
run: vp run test:unit
|
||||
|
||||
@@ -53,6 +55,7 @@ jobs:
|
||||
run: vp run e2e:install
|
||||
|
||||
- name: Run isolated source-api and built-web Cucumber E2E tests
|
||||
if: ${{ !inputs.run-external-runtime }}
|
||||
working-directory: ./e2e
|
||||
env:
|
||||
E2E_ADMIN_EMAIL: e2e-admin@example.com
|
||||
@@ -62,7 +65,53 @@ jobs:
|
||||
E2E_INIT_PASSWORD: E2eInit12345
|
||||
run: vp run e2e:full
|
||||
|
||||
- name: Run external runtime E2E tests
|
||||
- name: Preserve Chromium E2E report and logs
|
||||
if: ${{ !cancelled() && !inputs.run-external-runtime }}
|
||||
run: |
|
||||
if [[ -d e2e/cucumber-report ]]; then
|
||||
mv e2e/cucumber-report e2e/cucumber-report-non-external
|
||||
fi
|
||||
if [[ -d e2e/.logs ]]; then
|
||||
mv e2e/.logs e2e/.logs-non-external
|
||||
fi
|
||||
|
||||
- name: Run WebKit keyboard and browser smoke tests
|
||||
if: ${{ !inputs.run-external-runtime }}
|
||||
working-directory: ./e2e
|
||||
env:
|
||||
E2E_ADMIN_EMAIL: e2e-admin@example.com
|
||||
E2E_ADMIN_NAME: E2E Admin
|
||||
E2E_ADMIN_PASSWORD: E2eAdmin12345
|
||||
E2E_BROWSER: webkit
|
||||
E2E_INIT_PASSWORD: E2eInit12345
|
||||
run: |
|
||||
teardown_webkit_smoke() {
|
||||
local run_status=$?
|
||||
trap - EXIT
|
||||
if ! vp run e2e:middleware:down; then
|
||||
echo "::error title=E2E teardown failed::WebKit smoke middleware did not shut down cleanly."
|
||||
if [[ "$run_status" -eq 0 ]]; then
|
||||
run_status=1
|
||||
fi
|
||||
fi
|
||||
exit "$run_status"
|
||||
}
|
||||
|
||||
trap teardown_webkit_smoke EXIT
|
||||
vp run e2e:middleware:up
|
||||
vp run e2e -- --tags '@browser-smoke'
|
||||
|
||||
- name: Preserve WebKit E2E report and logs
|
||||
if: ${{ !cancelled() && !inputs.run-external-runtime }}
|
||||
run: |
|
||||
if [[ -d e2e/cucumber-report ]]; then
|
||||
mv e2e/cucumber-report e2e/cucumber-report-webkit
|
||||
fi
|
||||
if [[ -d e2e/.logs ]]; then
|
||||
mv e2e/.logs e2e/.logs-webkit
|
||||
fi
|
||||
|
||||
- name: Run prepared and external runtime E2E tests
|
||||
if: ${{ inputs.run-external-runtime }}
|
||||
working-directory: ./e2e
|
||||
env:
|
||||
@@ -72,13 +121,10 @@ jobs:
|
||||
E2E_AGENT_DECISION_MODEL_NAME: ${{ vars.E2E_AGENT_DECISION_MODEL_NAME || 'gpt-5.5' }}
|
||||
E2E_AGENT_DECISION_MODEL_PROVIDER: ${{ vars.E2E_AGENT_DECISION_MODEL_PROVIDER || 'openai' }}
|
||||
E2E_AGENT_DECISION_MODEL_TYPE: ${{ vars.E2E_AGENT_DECISION_MODEL_TYPE || 'llm' }}
|
||||
E2E_EXTERNAL_RUNTIME_SEED_SPECS: ${{ vars.E2E_EXTERNAL_RUNTIME_SEED_SPECS }}
|
||||
E2E_EXTERNAL_RUNTIME_TAGS: ${{ vars.E2E_EXTERNAL_RUNTIME_TAGS }}
|
||||
E2E_FORCE_WEB_BUILD: "1"
|
||||
E2E_INIT_PASSWORD: E2eInit12345
|
||||
E2E_MARKETPLACE_API_URL: ${{ vars.E2E_MARKETPLACE_API_URL }}
|
||||
E2E_MARKETPLACE_PLUGIN_IDS: ${{ vars.E2E_MARKETPLACE_PLUGIN_IDS }}
|
||||
E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS: ${{ vars.E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS }}
|
||||
E2E_MODEL_PROVIDER_CREDENTIALS_JSON: ${{ secrets.E2E_MODEL_PROVIDER_CREDENTIALS_JSON }}
|
||||
E2E_SPEECH_TO_TEXT_MODEL_NAME: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_NAME || 'gpt-4o-mini-transcribe' }}
|
||||
E2E_SPEECH_TO_TEXT_MODEL_PROVIDER: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_PROVIDER || 'openai' }}
|
||||
@@ -92,20 +138,22 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -d cucumber-report ]]; then
|
||||
rm -rf cucumber-report-non-external
|
||||
mv cucumber-report cucumber-report-non-external
|
||||
fi
|
||||
teardown_external_runtime() {
|
||||
local run_status=$?
|
||||
trap - EXIT
|
||||
if ! vp run e2e:middleware:down; then
|
||||
echo "::error title=E2E teardown failed::External runtime middleware did not shut down cleanly."
|
||||
if [[ "$run_status" -eq 0 ]]; then
|
||||
run_status=1
|
||||
fi
|
||||
fi
|
||||
exit "$run_status"
|
||||
}
|
||||
|
||||
if [[ -d .logs ]]; then
|
||||
rm -rf .logs-non-external
|
||||
mv .logs .logs-non-external
|
||||
fi
|
||||
|
||||
trap 'vp run e2e:middleware:down' EXIT
|
||||
trap teardown_external_runtime EXIT
|
||||
vp run e2e:middleware:up
|
||||
vp run e2e:external:prepare
|
||||
vp run e2e:external
|
||||
vp run e2e:post-merge:prepare
|
||||
vp run e2e:post-merge
|
||||
|
||||
- name: Upload Cucumber report
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -115,6 +163,7 @@ jobs:
|
||||
path: |
|
||||
e2e/cucumber-report
|
||||
e2e/cucumber-report-non-external
|
||||
e2e/cucumber-report-webkit
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload E2E logs
|
||||
@@ -125,5 +174,15 @@ jobs:
|
||||
path: |
|
||||
e2e/.logs/*.log
|
||||
e2e/.logs-non-external/*.log
|
||||
e2e/.logs-webkit/*.log
|
||||
include-hidden-files: true
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload E2E seed report
|
||||
if: ${{ !cancelled() && inputs.run-external-runtime }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-seed-report
|
||||
path: e2e/seed-report
|
||||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
@@ -17,8 +17,6 @@ jobs:
|
||||
test:
|
||||
name: Web Tests (${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
|
||||
runs-on: depot-ubuntu-24.04-4
|
||||
env:
|
||||
VITEST_COVERAGE_SCOPE: app-components
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
||||
Vendored
+4
-18
@@ -5,7 +5,9 @@
|
||||
"name": "Python: API (gevent)",
|
||||
"type": "debugpy",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/api/app.py",
|
||||
"module": "gevent.monkey",
|
||||
"args": ["--module", "app"],
|
||||
"gevent": true,
|
||||
"jinja": true,
|
||||
"justMyCode": true,
|
||||
"cwd": "${workspaceFolder}/api",
|
||||
@@ -33,22 +35,6 @@
|
||||
"justMyCode": false,
|
||||
"cwd": "${workspaceFolder}/api",
|
||||
"python": "${workspaceFolder}/api/.venv/bin/python"
|
||||
},
|
||||
{
|
||||
"name": "Next.js: debug full stack",
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"program": "${workspaceFolder}/web/node_modules/next/dist/bin/next",
|
||||
"runtimeArgs": ["--inspect"],
|
||||
"skipFiles": ["<node_internals>/**"],
|
||||
"serverReadyAction": {
|
||||
"action": "debugWithChrome",
|
||||
"killOnServerStop": true,
|
||||
"pattern": "- Local:.+(https?://.+)",
|
||||
"uriFormat": "%s",
|
||||
"webRoot": "${workspaceFolder}/web"
|
||||
},
|
||||
"cwd": "${workspaceFolder}/web"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -107,6 +107,7 @@ test:
|
||||
echo "Target: $(TARGET_TESTS)"; \
|
||||
uv run --project api --dev pytest $(TARGET_TESTS); \
|
||||
else \
|
||||
set -e; \
|
||||
echo "Running backend unit tests"; \
|
||||
uv run --project api --dev pytest -p no:benchmark --timeout "$${PYTEST_TIMEOUT:-20}" -n auto \
|
||||
api/tests/unit_tests \
|
||||
@@ -124,6 +125,7 @@ test-all:
|
||||
echo "Target: $(TARGET_TESTS)"; \
|
||||
uv run --project api --dev pytest $(TARGET_TESTS); \
|
||||
else \
|
||||
set -e; \
|
||||
echo "Running backend unit tests"; \
|
||||
uv run --project api --dev pytest -p no:benchmark --timeout "$${PYTEST_TIMEOUT:-20}" -n auto \
|
||||
api/tests/unit_tests \
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<a href="https://discord.gg/FngNHpbcY7" target="_blank">
|
||||
<img src="https://img.shields.io/discord/1082486657678311454?logo=discord&labelColor=%20%235462eb&logoColor=%20%23f5f5f5&color=%20%235462eb"
|
||||
alt="chat on Discord"></a>
|
||||
<a href="https://reddit.com/r/difyai" target="_blank">
|
||||
<a href="https://reddit.com/r/difyai" target="_blank">
|
||||
<img src="https://img.shields.io/reddit/subreddit-subscribers/difyai?style=plastic&logo=reddit&label=r%2Fdifyai&labelColor=white"
|
||||
alt="join Reddit"></a>
|
||||
<a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
|
||||
@@ -71,7 +71,7 @@ Dify is an open-source LLM app development platform. Its intuitive interface com
|
||||
|
||||
<br/>
|
||||
|
||||
The easiest way to start the Dify server is through [Docker Compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) are installed on your machine:
|
||||
The easiest way to start the Dify server is through [Docker Compose](docker/docker-compose.yaml). Before running Dify with the following commands, make sure that [Docker](https://docs.docker.com/get-docker/) and Docker Compose v2.24.0 or later are installed on your machine:
|
||||
|
||||
```bash
|
||||
cd dify
|
||||
|
||||
@@ -561,6 +561,8 @@ WORKFLOW_MAX_EXECUTION_STEPS=500
|
||||
WORKFLOW_MAX_EXECUTION_TIME=1200
|
||||
WORKFLOW_CALL_MAX_DEPTH=5
|
||||
MAX_VARIABLE_SIZE=204800
|
||||
# Maximum concurrent node-builder LLM calls per workflow generation request
|
||||
WORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS=6
|
||||
|
||||
# GraphEngine Worker Pool Configuration
|
||||
# Minimum number of workers per GraphEngine instance (default: 1)
|
||||
@@ -665,10 +667,27 @@ PLUGIN_REMOTE_INSTALL_HOST=localhost
|
||||
PLUGIN_MAX_PACKAGE_SIZE=15728640
|
||||
PLUGIN_MODEL_SCHEMA_CACHE_TTL=3600
|
||||
PLUGIN_MODEL_PROVIDERS_CACHE_TTL=86400
|
||||
# Comma-separated marketplace plugin IDs whose latest versions are installed for newly registered users.
|
||||
# Example: langgenius/openai,langgenius/gemini
|
||||
NEW_USER_DEFAULT_PLUGIN_IDS=
|
||||
# Comma-separated model_type:provider:model entries assigned after default plugins finish installing.
|
||||
# Example: llm:langgenius/openai/openai:gpt-4o-mini,text-embedding:langgenius/openai/openai:text-embedding-3-small
|
||||
NEW_USER_DEFAULT_MODELS=
|
||||
INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
|
||||
|
||||
# Dify Agent backend
|
||||
AGENT_BACKEND_BASE_URL=http://localhost:5050
|
||||
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
|
||||
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
|
||||
AGENT_BACKEND_RUN_TIMEOUT_SECONDS=1200
|
||||
|
||||
# KnowledgeFS (Dataset 2.0)
|
||||
KNOWLEDGE_FS_ENABLED=false
|
||||
KNOWLEDGE_FS_BASE_URL=
|
||||
# Shared with KnowledgeFS; use at least 32 random characters.
|
||||
KNOWLEDGE_FS_JWT_SECRET=
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS=300
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS=10
|
||||
|
||||
# Marketplace configuration
|
||||
MARKETPLACE_ENABLED=true
|
||||
|
||||
+19
@@ -1,5 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
# ``python -m app`` (docker DEBUG=true, or IDE debugging) serves through the
|
||||
# gevent pywsgi server at the bottom of this file, so the stdlib must be
|
||||
# monkey-patched BEFORE any other import pulls in sockets or locks. Without
|
||||
# this, every request runs as a greenlet on one OS thread while blocking
|
||||
# calls (LLM invokes, ``Future.result`` waits, DB I/O) pin that thread — the
|
||||
# whole process freezes until the call returns. Gunicorn and Celery apply
|
||||
# their own patching (see gunicorn.conf.py / celery_entrypoint.py), and
|
||||
# ``flask run`` uses real Werkzeug threads, so both skip this branch.
|
||||
if __name__ == "__main__":
|
||||
from gevent import monkey
|
||||
|
||||
monkey.patch_all()
|
||||
|
||||
import psycogreen.gevent as psycogreen_gevent
|
||||
from grpc.experimental import gevent as grpc_gevent
|
||||
|
||||
grpc_gevent.init_gevent()
|
||||
psycogreen_gevent.patch_psycopg()
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
@@ -8,7 +8,7 @@ creating another wire contract.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Protocol
|
||||
|
||||
from dify_agent.client import (
|
||||
@@ -45,7 +45,13 @@ class AgentBackendRunClient(Protocol):
|
||||
def cancel_run(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
|
||||
"""Request explicit cancellation for one Agent backend run."""
|
||||
|
||||
def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
|
||||
def stream_events(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
after: str | None = None,
|
||||
should_stop: Callable[[], bool] | None = None,
|
||||
) -> Iterator[RunEvent]:
|
||||
"""Yield public ``dify-agent`` run events in stream order."""
|
||||
|
||||
def wait_run(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
|
||||
@@ -61,7 +67,15 @@ class _DifyAgentSyncClient(Protocol):
|
||||
def cancel_run_sync(self, run_id: str, request: CancelRunRequest | None = None) -> CancelRunResponse:
|
||||
"""Cancel one run synchronously."""
|
||||
|
||||
def stream_events_sync(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
|
||||
def stream_events_sync(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
after: str | None = None,
|
||||
max_reconnects: int | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
should_stop: Callable[[], bool] | None = None,
|
||||
) -> Iterator[RunEvent]:
|
||||
"""Stream run events synchronously."""
|
||||
|
||||
def wait_run_sync(self, run_id: str, *, timeout_seconds: float | None = None) -> RunStatusResponse:
|
||||
@@ -73,8 +87,16 @@ class DifyAgentBackendRunClient:
|
||||
|
||||
client: _DifyAgentSyncClient
|
||||
|
||||
def __init__(self, client: _DifyAgentSyncClient) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
client: _DifyAgentSyncClient,
|
||||
*,
|
||||
stream_max_reconnects: int = 3,
|
||||
stream_timeout_seconds: float = 1200,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self._stream_max_reconnects = stream_max_reconnects
|
||||
self._stream_timeout_seconds = stream_timeout_seconds
|
||||
|
||||
def create_run(self, request: CreateRunRequest) -> CreateRunResponse:
|
||||
"""Create one run through ``POST /runs`` and normalize client exceptions."""
|
||||
@@ -90,10 +112,22 @@ class DifyAgentBackendRunClient:
|
||||
except Exception as exc:
|
||||
raise _normalize_dify_agent_error(exc) from exc
|
||||
|
||||
def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
|
||||
def stream_events(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
after: str | None = None,
|
||||
should_stop: Callable[[], bool] | None = None,
|
||||
) -> Iterator[RunEvent]:
|
||||
"""Stream run events from ``/events/sse`` with the wrapped client's reconnect policy."""
|
||||
try:
|
||||
yield from self.client.stream_events_sync(run_id, after=after)
|
||||
yield from self.client.stream_events_sync(
|
||||
run_id,
|
||||
after=after,
|
||||
max_reconnects=self._stream_max_reconnects,
|
||||
timeout_seconds=self._stream_timeout_seconds,
|
||||
should_stop=should_stop,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise _normalize_dify_agent_error(exc) from exc
|
||||
|
||||
|
||||
@@ -13,10 +13,17 @@ def create_agent_backend_run_client(
|
||||
base_url: str | None = None,
|
||||
use_fake: bool = False,
|
||||
fake_scenario: str | FakeAgentBackendScenario = FakeAgentBackendScenario.SUCCESS,
|
||||
stream_read_timeout_seconds: float = 30,
|
||||
stream_max_reconnects: int = 3,
|
||||
stream_run_timeout_seconds: float = 1200,
|
||||
) -> AgentBackendRunClient:
|
||||
"""Create the API-side run client without hiding the ``dify-agent`` protocol."""
|
||||
if use_fake:
|
||||
return FakeAgentBackendRunClient(scenario=FakeAgentBackendScenario(fake_scenario))
|
||||
if base_url is None:
|
||||
raise ValueError("base_url is required when creating a real Agent backend client")
|
||||
return DifyAgentBackendRunClient(Client(base_url=base_url))
|
||||
return DifyAgentBackendRunClient(
|
||||
Client(base_url=base_url, stream_timeout=stream_read_timeout_seconds),
|
||||
stream_max_reconnects=stream_max_reconnects,
|
||||
stream_timeout_seconds=stream_run_timeout_seconds,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ separate ``agent-backend.v1`` event stream.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
|
||||
@@ -69,9 +69,17 @@ class FakeAgentBackendRunClient:
|
||||
del request
|
||||
return CancelRunResponse(run_id=run_id, status="cancelled")
|
||||
|
||||
def stream_events(self, run_id: str, *, after: str | None = None) -> Iterator[RunEvent]:
|
||||
def stream_events(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
after: str | None = None,
|
||||
should_stop: Callable[[], bool] | None = None,
|
||||
) -> Iterator[RunEvent]:
|
||||
"""Yield the deterministic public ``RunEvent`` sequence for ``run_id``."""
|
||||
for event in self._events(run_id):
|
||||
if should_stop is not None and should_stop():
|
||||
return
|
||||
if after is not None and event.id is not None and event.id <= after:
|
||||
continue
|
||||
yield event
|
||||
|
||||
+209
-63
@@ -1,6 +1,8 @@
|
||||
import datetime
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import TypedDict
|
||||
|
||||
@@ -21,6 +23,7 @@ from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEX_PREFIXES = tuple("0123456789abcdef")
|
||||
_TARGET_MONTH_PATTERN = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
|
||||
|
||||
|
||||
class WorkflowRunArchivePlanRow(TypedDict):
|
||||
@@ -66,6 +69,7 @@ def _parse_tenant_prefixes(prefixes: str | None) -> list[str]:
|
||||
|
||||
|
||||
def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[str] | None:
|
||||
"""Keep an omitted scope unset while rejecting an explicitly empty scope."""
|
||||
if raw_ids is None:
|
||||
return None
|
||||
parsed = sorted({raw_id.strip() for raw_id in raw_ids.split(",") if raw_id.strip()})
|
||||
@@ -74,6 +78,27 @@ def _parse_comma_separated_ids(raw_ids: str | None, *, param_name: str) -> list[
|
||||
return parsed
|
||||
|
||||
|
||||
def _parse_archive_target_month(target_month: str) -> tuple[int, int]:
|
||||
"""Validate the V2 catalog month selector and return its numeric components."""
|
||||
if not _TARGET_MONTH_PATTERN.fullmatch(target_month):
|
||||
raise click.BadParameter("target-month must use YYYY-MM format", param_hint="--target-month")
|
||||
year_text, month_text = target_month.split("-", maxsplit=1)
|
||||
return int(year_text), int(month_text)
|
||||
|
||||
|
||||
def _parse_archive_catalog_cursor(after_catalog_id: str | None) -> str | None:
|
||||
"""Normalize the exclusive V2 catalog keyset cursor when one is provided."""
|
||||
if after_catalog_id is None:
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(after_catalog_id))
|
||||
except ValueError as exc:
|
||||
raise click.BadParameter(
|
||||
"after-catalog-id must be a UUID returned by the same V2 operation and scope",
|
||||
param_hint="--after-catalog-id",
|
||||
) from exc
|
||||
|
||||
|
||||
def _get_archive_candidate_tenant_ids_by_prefix(
|
||||
session: Session,
|
||||
prefix: str,
|
||||
@@ -810,9 +835,11 @@ def backfill_workflow_run_archive_bundles(
|
||||
click.echo(click.style(f" ... and {len(summary.errors) - 10} more failures", fg="red"))
|
||||
|
||||
|
||||
def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
def _echo_bundle_archive_operation_summary(summary, *, dry_run: bool) -> None:
|
||||
status = "completed successfully" if summary.bundles_failed == 0 else "completed with failures"
|
||||
fg = "green" if summary.bundles_failed == 0 else "red"
|
||||
cursor_label = "preview_next_catalog_id" if dry_run else "next_catalog_id"
|
||||
cursor_value = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
click.echo(
|
||||
click.style(
|
||||
f"{summary.operation} {status}. "
|
||||
@@ -821,10 +848,12 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
f"archive_bytes={summary.archive_bytes} duration={summary.elapsed_time:.2f}s "
|
||||
f"validation_time={summary.validation_time:.2f}s "
|
||||
f"runs_per_second={summary.runs_per_second:.2f} rows_per_second={summary.rows_per_second:.2f} "
|
||||
f"bytes_per_second={summary.bytes_per_second:.2f}",
|
||||
f"bytes_per_second={summary.bytes_per_second:.2f} {cursor_label}={cursor_value or 'none'}",
|
||||
fg=fg,
|
||||
)
|
||||
)
|
||||
if dry_run:
|
||||
click.echo(click.style("Dry-run cursor is preview-only; do not persist it for a destructive run.", fg="yellow"))
|
||||
click.echo(click.style("table,row_count", fg="white"))
|
||||
for table_name in [
|
||||
"workflow_runs",
|
||||
@@ -842,7 +871,8 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
click.style(
|
||||
f" bundle={result.bundle_id} tenant={result.tenant_id} runs={result.run_count} "
|
||||
f"rows={result.row_count} archive_bytes={result.archive_bytes} "
|
||||
f"time={result.elapsed_time:.2f}s validation={result.validation_time:.2f}s",
|
||||
f"catalog_id={result.catalog_id} time={result.elapsed_time:.2f}s "
|
||||
f"validation={result.validation_time:.2f}s",
|
||||
fg="white",
|
||||
)
|
||||
)
|
||||
@@ -850,7 +880,7 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
click.echo(
|
||||
click.style(
|
||||
f" failed bundle={result.bundle_id} tenant={result.tenant_id} "
|
||||
f"object_prefix={result.object_prefix} error={result.error}",
|
||||
f"catalog_id={result.catalog_id} object_prefix={result.object_prefix} error={result.error}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
@@ -867,25 +897,24 @@ def _echo_bundle_archive_operation_summary(summary) -> None:
|
||||
)
|
||||
@click.option("--run-id", required=False, help="Workflow run ID to restore.")
|
||||
@click.option(
|
||||
"--start-from",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--target-month",
|
||||
metavar="YYYY-MM",
|
||||
default=None,
|
||||
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
|
||||
help="V2 catalog month to restore; required unless --run-id is used.",
|
||||
)
|
||||
@click.option(
|
||||
"--end-before",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--after-catalog-id",
|
||||
default=None,
|
||||
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
|
||||
help="Exclusive V2 cursor from the same restore month and tenant scope.",
|
||||
)
|
||||
@click.option("--workers", default=1, show_default=True, type=int, help="V1 --run-id compatibility only.")
|
||||
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to restore.")
|
||||
@click.option("--limit", type=click.IntRange(min=1), default=100, show_default=True, help="Maximum V2 catalog rows.")
|
||||
@click.option("--dry-run", is_flag=True, help="Preview without restoring.")
|
||||
def restore_workflow_runs(
|
||||
tenant_ids: str | None,
|
||||
run_id: str | None,
|
||||
start_from: datetime.datetime | None,
|
||||
end_before: datetime.datetime | None,
|
||||
target_month: str | None,
|
||||
after_catalog_id: str | None,
|
||||
workers: int,
|
||||
limit: int,
|
||||
dry_run: bool,
|
||||
@@ -905,23 +934,20 @@ def restore_workflow_runs(
|
||||
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
|
||||
from services.retention.workflow_run.restore_archived_workflow_run import WorkflowRunRestore
|
||||
|
||||
parsed_tenant_ids = None
|
||||
if tenant_ids:
|
||||
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
|
||||
if not parsed_tenant_ids:
|
||||
raise click.BadParameter("tenant-ids must not be empty")
|
||||
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
|
||||
|
||||
if (start_from is None) ^ (end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before must be provided together.")
|
||||
if run_id is None and (start_from is None or end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before are required for batch restore.")
|
||||
if workers < 1:
|
||||
raise click.BadParameter("workers must be at least 1")
|
||||
if run_id is not None and (target_month is not None or after_catalog_id is not None):
|
||||
raise click.UsageError("--target-month and --after-catalog-id are only valid for V2 batch restore.")
|
||||
if run_id is None and target_month is None:
|
||||
raise click.UsageError("--target-month is required for V2 batch restore.")
|
||||
|
||||
start_time = datetime.datetime.now(datetime.UTC)
|
||||
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Starting restore of workflow run {run_id} at {start_time.isoformat()}.",
|
||||
f"Starting restore of {target_desc} at {start_time.isoformat()}.",
|
||||
fg="white",
|
||||
)
|
||||
)
|
||||
@@ -955,17 +981,20 @@ def restore_workflow_runs(
|
||||
click.echo(
|
||||
click.style("--workers is ignored for V2 bundle restore; bundles are processed serially.", fg="yellow")
|
||||
)
|
||||
assert start_from is not None
|
||||
assert end_before is not None
|
||||
assert target_month is not None
|
||||
target_year, target_month_number = _parse_archive_target_month(target_month)
|
||||
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
|
||||
bundle_restorer = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
|
||||
summary = bundle_restorer.restore_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
start_date=start_from,
|
||||
end_date=end_before,
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
after_catalog_id=catalog_cursor,
|
||||
limit=limit,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary)
|
||||
return
|
||||
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
|
||||
if summary.bundles_failed:
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
|
||||
@click.command(
|
||||
@@ -979,23 +1008,41 @@ def restore_workflow_runs(
|
||||
)
|
||||
@click.option("--run-id", required=False, help="Workflow run ID to delete.")
|
||||
@click.option(
|
||||
"--start-from",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--target-month",
|
||||
metavar="YYYY-MM",
|
||||
default=None,
|
||||
help="Optional lower bound (inclusive) for created_at; must be paired with --end-before.",
|
||||
help="V2 catalog month to delete; required unless --run-id is used.",
|
||||
)
|
||||
@click.option(
|
||||
"--end-before",
|
||||
type=click.DateTime(formats=["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"]),
|
||||
"--after-catalog-id",
|
||||
default=None,
|
||||
help="Optional upper bound (exclusive) for created_at; must be paired with --start-from.",
|
||||
help="Exclusive V2 cursor from the same delete month and tenant scope.",
|
||||
)
|
||||
@click.option(
|
||||
"--run-shard-index",
|
||||
default=None,
|
||||
type=click.IntRange(min=0),
|
||||
help="Zero-based archive shard index. Must be paired with --run-shard-total.",
|
||||
)
|
||||
@click.option(
|
||||
"--run-shard-total",
|
||||
default=None,
|
||||
type=click.IntRange(min=1, max=16),
|
||||
help="Total archive shard count. Must be paired with --run-shard-index.",
|
||||
)
|
||||
@click.option("--all-pages", is_flag=True, help="Process catalog pages until an empty page is reached.")
|
||||
@click.option(
|
||||
"--limit",
|
||||
type=click.IntRange(min=1),
|
||||
default=100,
|
||||
show_default=True,
|
||||
help="Maximum V2 catalog rows per page.",
|
||||
)
|
||||
@click.option("--limit", type=int, default=100, show_default=True, help="Maximum number of V2 bundles to delete.")
|
||||
@click.option("--dry-run", is_flag=True, help="Preview without deleting.")
|
||||
@click.option(
|
||||
"--skip-bad-archives",
|
||||
is_flag=True,
|
||||
help="Continue batch deletion when one archive object fails validation.",
|
||||
help="V1 --run-id only: continue when one archive object fails validation.",
|
||||
)
|
||||
@click.option(
|
||||
"--restore-sample-interval",
|
||||
@@ -1007,8 +1054,11 @@ def restore_workflow_runs(
|
||||
def delete_archived_workflow_runs(
|
||||
tenant_ids: str | None,
|
||||
run_id: str | None,
|
||||
start_from: datetime.datetime | None,
|
||||
end_before: datetime.datetime | None,
|
||||
target_month: str | None,
|
||||
after_catalog_id: str | None,
|
||||
run_shard_index: int | None,
|
||||
run_shard_total: int | None,
|
||||
all_pages: bool,
|
||||
limit: int,
|
||||
dry_run: bool,
|
||||
skip_bad_archives: bool,
|
||||
@@ -1018,26 +1068,38 @@ def delete_archived_workflow_runs(
|
||||
Delete archived workflow runs from the database.
|
||||
|
||||
Batch delete uses V2 bundle metadata and validates object existence, manifest schema, object size, checksum, row
|
||||
counts, and source/archive content checksums before deleting source rows. `--run-id` keeps the V1 per-run path.
|
||||
counts, and source/archive content checksums before deleting source rows. Parallel workers may select one exact
|
||||
archive shard; all-pages mode keeps only the current bounded page in memory. `--run-id` keeps the V1 per-run path.
|
||||
"""
|
||||
from services.retention.workflow_run.bundle_archive_maintenance import WorkflowRunBundleArchiveMaintenance
|
||||
from services.retention.workflow_run.delete_archived_workflow_run import ArchivedWorkflowRunDeletion
|
||||
|
||||
parsed_tenant_ids = None
|
||||
if tenant_ids:
|
||||
parsed_tenant_ids = [tid.strip() for tid in tenant_ids.split(",") if tid.strip()]
|
||||
if not parsed_tenant_ids:
|
||||
raise click.BadParameter("tenant-ids must not be empty")
|
||||
parsed_tenant_ids = _parse_comma_separated_ids(tenant_ids, param_name="tenant-ids")
|
||||
|
||||
if (start_from is None) ^ (end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before must be provided together.")
|
||||
if run_id is None and (start_from is None or end_before is None):
|
||||
raise click.UsageError("--start-from and --end-before are required for batch delete.")
|
||||
if restore_sample_interval < 0:
|
||||
raise click.BadParameter("restore-sample-interval must be >= 0")
|
||||
if run_id is not None and (
|
||||
target_month is not None
|
||||
or after_catalog_id is not None
|
||||
or run_shard_index is not None
|
||||
or run_shard_total is not None
|
||||
or all_pages
|
||||
):
|
||||
raise click.UsageError(
|
||||
"--target-month, --after-catalog-id, --run-shard-index, --run-shard-total, and --all-pages "
|
||||
"are only valid for V2 batch delete."
|
||||
)
|
||||
if run_id is None and target_month is None:
|
||||
raise click.UsageError("--target-month is required for V2 batch delete.")
|
||||
if run_id is None and skip_bad_archives:
|
||||
raise click.UsageError("--skip-bad-archives is not supported for V2 catalog batches; they fail fast.")
|
||||
if (run_shard_index is None) ^ (run_shard_total is None):
|
||||
raise click.UsageError("--run-shard-index and --run-shard-total must be provided together.")
|
||||
if run_shard_index is not None and run_shard_total is not None and run_shard_index >= run_shard_total:
|
||||
raise click.UsageError("--run-shard-index must be less than --run-shard-total.")
|
||||
|
||||
start_time = datetime.datetime.now(datetime.UTC)
|
||||
target_desc = f"workflow run {run_id}" if run_id else "workflow runs"
|
||||
target_desc = f"workflow run {run_id}" if run_id else f"workflow archive catalog month {target_month}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Starting delete of {target_desc} at {start_time.isoformat()}.",
|
||||
@@ -1110,20 +1172,104 @@ def delete_archived_workflow_runs(
|
||||
|
||||
if restore_sample_interval:
|
||||
click.echo(click.style("--restore-sample-interval is ignored for V2 bundle delete.", fg="yellow"))
|
||||
assert start_from is not None
|
||||
assert end_before is not None
|
||||
bundle_deleter = WorkflowRunBundleArchiveMaintenance(
|
||||
dry_run=dry_run,
|
||||
strict_content_validation=True,
|
||||
stop_on_error=not skip_bad_archives,
|
||||
assert target_month is not None
|
||||
target_year, target_month_number = _parse_archive_target_month(target_month)
|
||||
catalog_cursor = _parse_archive_catalog_cursor(after_catalog_id)
|
||||
shard = (
|
||||
f"{run_shard_index:02d}-of-{run_shard_total:02d}"
|
||||
if run_shard_index is not None and run_shard_total is not None
|
||||
else None
|
||||
)
|
||||
summary = bundle_deleter.delete_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
start_date=start_from,
|
||||
end_date=end_before,
|
||||
limit=limit,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary)
|
||||
bundle_deleter = WorkflowRunBundleArchiveMaintenance(dry_run=dry_run, strict_content_validation=True)
|
||||
if run_shard_total is not None:
|
||||
try:
|
||||
bundle_deleter.validate_catalog_shards(
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
shard_total=run_shard_total,
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.exception(
|
||||
"Archive catalog shard preflight failed: target_month=%s shard=%s",
|
||||
target_month,
|
||||
shard,
|
||||
)
|
||||
raise click.ClickException(
|
||||
f"Archive catalog shard preflight failed for target_month={target_month} shard={shard}: {exc}"
|
||||
) from exc
|
||||
|
||||
initial_catalog_cursor = catalog_cursor
|
||||
pages_processed = 0
|
||||
bundles_succeeded = 0
|
||||
runs_processed = 0
|
||||
rows_processed = 0
|
||||
archive_bytes = 0
|
||||
while True:
|
||||
summary = bundle_deleter.delete_batch(
|
||||
tenant_ids=parsed_tenant_ids,
|
||||
target_year=target_year,
|
||||
target_month=target_month_number,
|
||||
after_catalog_id=catalog_cursor,
|
||||
limit=limit,
|
||||
shard=shard,
|
||||
)
|
||||
_echo_bundle_archive_operation_summary(summary, dry_run=dry_run)
|
||||
if summary.bundles_failed:
|
||||
failed_result = next((result for result in summary.results if not result.success), None)
|
||||
failed_catalog_id = failed_result.catalog_id if failed_result is not None else "unknown"
|
||||
page_resume_cursor = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
resume_cursor = page_resume_cursor or catalog_cursor
|
||||
if dry_run:
|
||||
cursor_details = (
|
||||
f"preview_after_catalog_id={resume_cursor or 'none'} "
|
||||
f"destructive_retry_after_catalog_id={initial_catalog_cursor or 'none'}"
|
||||
)
|
||||
else:
|
||||
cursor_details = f"resume_after_catalog_id={resume_cursor or 'none'}"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete stopped: target_month={target_month} shard={shard or 'all'} "
|
||||
f"failed_catalog_id={failed_catalog_id} "
|
||||
f"{cursor_details}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
raise click.exceptions.Exit(1)
|
||||
|
||||
if not all_pages:
|
||||
break
|
||||
if summary.bundles_processed == 0:
|
||||
break
|
||||
|
||||
pages_processed += 1
|
||||
bundles_succeeded += summary.bundles_succeeded
|
||||
runs_processed += summary.runs_processed
|
||||
rows_processed += summary.rows_processed
|
||||
archive_bytes += summary.archive_bytes
|
||||
next_catalog_id = summary.preview_next_catalog_id if dry_run else summary.next_catalog_id
|
||||
if next_catalog_id is None or (catalog_cursor is not None and next_catalog_id <= catalog_cursor):
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete cursor did not advance: target_month={target_month} shard={shard or 'all'} "
|
||||
f"after_catalog_id={catalog_cursor or 'none'} next_catalog_id={next_catalog_id or 'none'}",
|
||||
fg="red",
|
||||
)
|
||||
)
|
||||
raise click.exceptions.Exit(1)
|
||||
catalog_cursor = next_catalog_id
|
||||
|
||||
if all_pages:
|
||||
final_cursor_label = "preview_final_catalog_id" if dry_run else "final_catalog_id"
|
||||
click.echo(
|
||||
click.style(
|
||||
f"Delete all-pages completed successfully. target_month={target_month} shard={shard or 'all'} "
|
||||
f"pages={pages_processed} bundles_success={bundles_succeeded} runs={runs_processed} "
|
||||
f"rows={rows_processed} archive_bytes={archive_bytes} "
|
||||
f"{final_cursor_label}={catalog_cursor or 'none'}",
|
||||
fg="green",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _find_orphaned_draft_variables(batch_size: int = 1000) -> list[str]:
|
||||
|
||||
@@ -14,6 +14,12 @@ class EnterpriseFeatureConfig(BaseSettings):
|
||||
default=False,
|
||||
)
|
||||
|
||||
WEBAPP_PUBLIC_ACCESS_ENABLED: bool = Field(
|
||||
description="Whether admins are allowed to set a webapp's access mode to public (anyone with the link, "
|
||||
"no auth). Disable in security-sensitive on-prem deployments.",
|
||||
default=True,
|
||||
)
|
||||
|
||||
CAN_REPLACE_LOGO: bool = Field(
|
||||
description="Allow customization of the enterprise logo.",
|
||||
default=False,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from configs.extra.agent_backend_config import AgentBackendConfig
|
||||
from configs.extra.archive_config import ArchiveStorageConfig
|
||||
from configs.extra.knowledge_fs_config import KnowledgeFSConfig
|
||||
from configs.extra.notion_config import NotionConfig
|
||||
from configs.extra.sentry_config import SentryConfig
|
||||
|
||||
@@ -8,6 +9,7 @@ class ExtraServiceConfig(
|
||||
# place the configs in alphabet order
|
||||
AgentBackendConfig,
|
||||
ArchiveStorageConfig,
|
||||
KnowledgeFSConfig,
|
||||
NotionConfig,
|
||||
SentryConfig,
|
||||
):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from pydantic import Field, NonNegativeFloat
|
||||
from pydantic import Field, NonNegativeFloat, NonNegativeInt, PositiveFloat
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
@@ -22,6 +22,21 @@ class AgentBackendConfig(BaseSettings):
|
||||
default="success",
|
||||
)
|
||||
|
||||
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS: PositiveFloat = Field(
|
||||
description="Read timeout for one Agent backend SSE connection.",
|
||||
default=30,
|
||||
)
|
||||
|
||||
AGENT_BACKEND_STREAM_MAX_RECONNECTS: NonNegativeInt = Field(
|
||||
description="Maximum Agent backend SSE reconnects before failing the run.",
|
||||
default=3,
|
||||
)
|
||||
|
||||
AGENT_BACKEND_RUN_TIMEOUT_SECONDS: PositiveFloat = Field(
|
||||
description="Total deadline for one Agent backend run event stream.",
|
||||
default=1200,
|
||||
)
|
||||
|
||||
AGENT_SHELL_ENABLED: bool = Field(
|
||||
description=(
|
||||
"Inject the dify.shell layer (sandboxed bash workspace) into Agent runs. "
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Configuration for the optional KnowledgeFS Console bridge."""
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import Field, PositiveFloat, SecretStr, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class KnowledgeFSConfig(BaseSettings):
|
||||
"""Server-only settings for the KnowledgeFS production connection."""
|
||||
|
||||
KNOWLEDGE_FS_ENABLED: bool = Field(
|
||||
default=False,
|
||||
description="Enable the private KnowledgeFS Console bridge.",
|
||||
)
|
||||
KNOWLEDGE_FS_BASE_URL: str | None = Field(default=None, description="KnowledgeFS gateway base URL.")
|
||||
KNOWLEDGE_FS_JWT_SECRET: SecretStr | None = Field(
|
||||
default=None,
|
||||
min_length=32,
|
||||
description="Shared secret used to sign short-lived KnowledgeFS service JWTs.",
|
||||
)
|
||||
KNOWLEDGE_FS_SSE_READ_TIMEOUT_SECONDS: PositiveFloat = Field(default=300.0, le=3600.0, allow_inf_nan=False)
|
||||
KNOWLEDGE_FS_TIMEOUT_SECONDS: PositiveFloat = Field(default=10.0, le=60.0, allow_inf_nan=False)
|
||||
|
||||
@field_validator(
|
||||
"KNOWLEDGE_FS_BASE_URL",
|
||||
"KNOWLEDGE_FS_JWT_SECRET",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def normalize_optional_string(cls, value: object) -> object:
|
||||
if isinstance(value, SecretStr):
|
||||
normalized = value.get_secret_value().strip()
|
||||
return SecretStr(normalized) if normalized else None
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
return value
|
||||
|
||||
@field_validator("KNOWLEDGE_FS_BASE_URL")
|
||||
@classmethod
|
||||
def validate_base_url(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must be an absolute HTTP(S) URL")
|
||||
try:
|
||||
_ = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must include a valid port") from exc
|
||||
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL must not include credentials, query, or fragment")
|
||||
return value.rstrip("/")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_enabled_connection(self) -> "KnowledgeFSConfig":
|
||||
if not self.KNOWLEDGE_FS_ENABLED:
|
||||
return self
|
||||
if bool(self.KNOWLEDGE_FS_BASE_URL) != bool(self.KNOWLEDGE_FS_JWT_SECRET):
|
||||
raise ValueError("KNOWLEDGE_FS_BASE_URL and KNOWLEDGE_FS_JWT_SECRET must be configured together")
|
||||
if not self.KNOWLEDGE_FS_BASE_URL:
|
||||
raise ValueError("KnowledgeFS connection settings are required when the integration is enabled")
|
||||
return self
|
||||
@@ -1,4 +1,4 @@
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from typing import Literal
|
||||
|
||||
@@ -11,6 +11,7 @@ from pydantic import (
|
||||
PositiveFloat,
|
||||
PositiveInt,
|
||||
computed_field,
|
||||
field_validator,
|
||||
)
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
@@ -275,6 +276,63 @@ class PluginConfig(BaseSettings):
|
||||
default=50 * 1024 * 1024,
|
||||
)
|
||||
|
||||
NEW_USER_DEFAULT_PLUGIN_IDS: str = Field(
|
||||
description="Comma-separated marketplace plugin IDs whose latest versions are installed for new users",
|
||||
default="",
|
||||
)
|
||||
|
||||
@field_validator("PLUGIN_REMOTE_INSTALL_PORT", mode="before")
|
||||
@classmethod
|
||||
def _reject_host_port_shaped_plugin_remote_install_port(cls, v):
|
||||
"""Reject ``host:port``-shaped values with an actionable hint.
|
||||
|
||||
``EXPOSE_PLUGIN_DEBUGGING_PORT`` is overloaded: it feeds both the
|
||||
plugin_daemon ``ports:`` mapping (where ``127.0.0.1:5003`` is valid
|
||||
compose syntax) and this integer app setting advertised in the console.
|
||||
Without this guard a loopback bind spec crashloops the api container
|
||||
with an opaque ``int_parsing`` traceback. See issue #39323.
|
||||
"""
|
||||
if isinstance(v, str) and ":" in v.strip():
|
||||
raise ValueError(
|
||||
"PLUGIN_REMOTE_INSTALL_PORT must be a bare port number, got "
|
||||
f"{v!r}. A 'host:port' value usually means "
|
||||
"EXPOSE_PLUGIN_DEBUGGING_PORT was set to a compose publish spec "
|
||||
"like '127.0.0.1:5003'; bind loopback via a "
|
||||
"docker-compose.override.yaml instead of overloading this var."
|
||||
)
|
||||
return v
|
||||
|
||||
@property
|
||||
def NEW_USER_DEFAULT_PLUGIN_ID_LIST(self) -> list[str]:
|
||||
return [item.strip() for item in self.NEW_USER_DEFAULT_PLUGIN_IDS.split(",") if item.strip()]
|
||||
|
||||
NEW_USER_DEFAULT_MODELS: str = Field(
|
||||
description=("Comma-separated default models for new users in 'model_type:provider:model' format"),
|
||||
default="",
|
||||
)
|
||||
|
||||
@property
|
||||
def NEW_USER_DEFAULT_MODEL_LIST(self) -> list[tuple[str, str, str]]:
|
||||
default_models: list[tuple[str, str, str]] = []
|
||||
configured_model_types: set[str] = set()
|
||||
|
||||
for item in self.NEW_USER_DEFAULT_MODELS.split(","):
|
||||
if not item.strip():
|
||||
continue
|
||||
|
||||
parts = tuple(part.strip() for part in item.split(":", 2))
|
||||
if len(parts) != 3 or not all(parts):
|
||||
raise ValueError("NEW_USER_DEFAULT_MODELS entries must use 'model_type:provider:model' format")
|
||||
|
||||
model_type, provider, model = parts
|
||||
if model_type in configured_model_types:
|
||||
raise ValueError(f"NEW_USER_DEFAULT_MODELS contains duplicate model type: {model_type}")
|
||||
|
||||
configured_model_types.add(model_type)
|
||||
default_models.append((model_type, provider, model))
|
||||
|
||||
return default_models
|
||||
|
||||
|
||||
class MarketplaceConfig(BaseSettings):
|
||||
"""
|
||||
@@ -784,6 +842,11 @@ class WorkflowConfig(BaseSettings):
|
||||
default=500,
|
||||
)
|
||||
|
||||
WORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS: PositiveInt = Field(
|
||||
description="Maximum concurrent node-builder LLM calls per workflow generation request",
|
||||
default=6,
|
||||
)
|
||||
|
||||
WORKFLOW_MAX_EXECUTION_TIME: PositiveInt = Field(
|
||||
description="Maximum execution time in seconds for a single workflow",
|
||||
default=1200,
|
||||
@@ -1097,6 +1160,16 @@ class HomepageConfig(BaseSettings):
|
||||
default=True,
|
||||
)
|
||||
|
||||
ENABLE_STEP_BY_STEP_TOUR: bool = Field(
|
||||
description="Enable account-level Step-by-step Tour eligibility checks",
|
||||
default=False,
|
||||
)
|
||||
|
||||
STEP_BY_STEP_TOUR_ROLLOUT_STARTED_AT: datetime | None = Field(
|
||||
description="UTC timestamp after which newly initialized accounts are eligible for Step-by-step Tour",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
class RagEtlConfig(BaseSettings):
|
||||
"""
|
||||
@@ -1424,6 +1497,11 @@ class LoginConfig(BaseSettings):
|
||||
|
||||
|
||||
class AccountConfig(BaseSettings):
|
||||
ENABLE_CHANGE_EMAIL: bool = Field(
|
||||
description="whether users can change their email address",
|
||||
default=True,
|
||||
)
|
||||
|
||||
ACCOUNT_DELETION_TOKEN_EXPIRY_MINUTES: PositiveInt = Field(
|
||||
description="Duration in minutes for which a account deletion token remains valid",
|
||||
default=5,
|
||||
|
||||
@@ -10,6 +10,7 @@ from extensions.ext_database import db
|
||||
from libs.login import current_account_with_tenant
|
||||
from models.dataset import Dataset
|
||||
from models.model import App
|
||||
from services.agent.roster_service import AgentRosterService
|
||||
from services.enterprise.rbac_service import RBACService
|
||||
|
||||
__all__ = ["RBACPermission", "RBACResourceScope", "enforce_rbac_access", "rbac_permission_required"]
|
||||
@@ -51,7 +52,7 @@ def enforce_rbac_access(
|
||||
check_resource_type = None if resource_type == RBACResourceScope.WORKSPACE else resource_type
|
||||
resource_id = None
|
||||
if resource_required and check_resource_type:
|
||||
resource_id = _extract_resource_id(resource_type, path_args)
|
||||
resource_id = _extract_resource_id(resource_type, tenant_id, path_args)
|
||||
if _is_resource_owned_by_current_user(tenant_id, account_id, resource_type, resource_id):
|
||||
return
|
||||
allowed = RBACService.CheckAccess.check(
|
||||
@@ -131,11 +132,14 @@ def _is_resource_owned_by_current_user(
|
||||
return False
|
||||
|
||||
|
||||
def _extract_resource_id(resource_type: RBACResourceScope, path_args: dict[str, object] | None = None) -> str:
|
||||
def _extract_resource_id(
|
||||
resource_type: RBACResourceScope, tenant_id: str, path_args: dict[str, object] | None = None
|
||||
) -> str:
|
||||
"""Extract the resource ID from matched path arguments.
|
||||
|
||||
Some legacy route classes use neutral names such as ``resource_id`` for
|
||||
app/dataset resources, and Agent App routes use ``agent_id`` as the app id.
|
||||
app/dataset resources, and Agent routes carry ``agent_id``, which is
|
||||
resolved to the App backing that Agent.
|
||||
Dataset endpoints behind a rag-pipeline route contain ``pipeline_id``
|
||||
instead of ``dataset_id``. In that case we look up the associated
|
||||
``Dataset`` row via ``Dataset.pipeline_id``.
|
||||
@@ -146,10 +150,19 @@ def _extract_resource_id(resource_type: RBACResourceScope, path_args: dict[str,
|
||||
matched_args = {**view_args, **(path_args or {})}
|
||||
|
||||
if resource_type == RBACResourceScope.APP:
|
||||
app_id = matched_args.get("app_id") or matched_args.get("agent_id") or matched_args.get("resource_id")
|
||||
if not app_id:
|
||||
raise ValueError("Missing app_id in request path")
|
||||
return str(app_id)
|
||||
app_id = matched_args.get("app_id")
|
||||
if app_id:
|
||||
return str(app_id)
|
||||
|
||||
agent_id = matched_args.get("agent_id")
|
||||
if agent_id:
|
||||
authz_app_id = AgentRosterService(db.session).peek_authz_app_id(tenant_id=tenant_id, agent_id=str(agent_id))
|
||||
return authz_app_id or str(agent_id)
|
||||
|
||||
resource_id = matched_args.get("resource_id")
|
||||
if resource_id:
|
||||
return str(resource_id)
|
||||
raise ValueError("Missing app_id in request path")
|
||||
|
||||
if resource_type == RBACResourceScope.DATASET:
|
||||
dataset_id = matched_args.get("dataset_id") or matched_args.get("resource_id")
|
||||
|
||||
@@ -38,7 +38,9 @@ from . import (
|
||||
feature,
|
||||
human_input_form,
|
||||
init_validate,
|
||||
knowledge_fs_proxy,
|
||||
notification,
|
||||
onboarding,
|
||||
ping,
|
||||
setup,
|
||||
spec,
|
||||
@@ -195,6 +197,7 @@ __all__ = [
|
||||
"human_input_form",
|
||||
"init_validate",
|
||||
"installed_app",
|
||||
"knowledge_fs_proxy",
|
||||
"load_balancing_config",
|
||||
"login",
|
||||
"mcp_server",
|
||||
@@ -207,6 +210,7 @@ __all__ = [
|
||||
"notification",
|
||||
"oauth",
|
||||
"oauth_server",
|
||||
"onboarding",
|
||||
"ops_trace",
|
||||
"parameter",
|
||||
"ping",
|
||||
|
||||
@@ -230,6 +230,7 @@ class WorkflowAgentComposerSaveToRosterApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -439,6 +440,7 @@ class SnippetAgentComposerSaveToRosterApi(Resource):
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
|
||||
)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -478,6 +480,7 @@ class AgentComposerApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user_id
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
|
||||
@@ -62,7 +62,7 @@ from libs.datetime_utils import parse_time_range
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from models.agent import Agent, AgentStatus
|
||||
from models.agent import Agent, AgentConfigDraftType, AgentStatus
|
||||
from models.agent_config_entities import AgentSoulConfig
|
||||
from models.enums import ApiTokenType
|
||||
from models.model import ApiToken, App, IconType
|
||||
@@ -266,6 +266,13 @@ class AgentDebugConversationRefreshResponse(BaseModel):
|
||||
debug_conversation_message_count: int = 0
|
||||
|
||||
|
||||
class AgentDebugConversationRefreshPayload(BaseModel):
|
||||
draft_type: AgentConfigDraftType = Field(
|
||||
default=AgentConfigDraftType.DEBUG_BUILD,
|
||||
description="Agent draft surface whose conversation should be refreshed",
|
||||
)
|
||||
|
||||
|
||||
class AgentPublishPayload(BaseModel):
|
||||
version_note: str | None = Field(default=None, description="Optional note for this published Agent version")
|
||||
|
||||
@@ -309,6 +316,7 @@ register_schema_models(
|
||||
AgentAppCopyPayload,
|
||||
AgentPublishPayload,
|
||||
AgentBuildDraftCheckoutPayload,
|
||||
AgentDebugConversationRefreshPayload,
|
||||
ComposerSavePayload,
|
||||
AgentApiStatusPayload,
|
||||
AgentInviteOptionsQuery,
|
||||
@@ -392,6 +400,7 @@ def _serialize_agent_app_detail(
|
||||
tenant_id=app_model.tenant_id,
|
||||
agent_id=agent.id,
|
||||
account_id=current_user.id,
|
||||
draft_type=AgentConfigDraftType.DEBUG_BUILD,
|
||||
commit=False,
|
||||
)
|
||||
message_count = roster_service.count_agent_app_debug_conversation_messages(
|
||||
@@ -439,6 +448,7 @@ def _serialize_agent_app_pagination(session: Session, app_pagination, *, tenant_
|
||||
tenant_id=tenant_id,
|
||||
agents=list(agents_by_app_id.values()),
|
||||
account_id=current_user.id,
|
||||
draft_type=AgentConfigDraftType.DEBUG_BUILD,
|
||||
)
|
||||
payload = AgentAppPagination.model_validate(
|
||||
app_pagination,
|
||||
@@ -542,6 +552,7 @@ class AgentAppListApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -579,6 +590,7 @@ class AgentAppListApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -620,6 +632,7 @@ class AgentAppApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -645,6 +658,7 @@ class AgentAppApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
def delete(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -655,6 +669,16 @@ class AgentAppApi(Resource):
|
||||
|
||||
@console_ns.route("/agent/<uuid:agent_id>/debug-conversation/refresh")
|
||||
class AgentDebugConversationRefreshApi(Resource):
|
||||
@console_ns.expect(console_ns.models[AgentDebugConversationRefreshPayload.__name__])
|
||||
@console_ns.doc(
|
||||
params={
|
||||
"payload": {
|
||||
"in": "body",
|
||||
"required": False,
|
||||
"schema": {"$ref": f"#/components/schemas/{AgentDebugConversationRefreshPayload.__name__}"},
|
||||
}
|
||||
}
|
||||
)
|
||||
@console_ns.response(
|
||||
200,
|
||||
"Agent debug conversation refreshed",
|
||||
@@ -669,10 +693,12 @@ class AgentDebugConversationRefreshApi(Resource):
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, current_user: Account, agent_id: UUID):
|
||||
args = AgentDebugConversationRefreshPayload.model_validate(request.get_json(silent=True) or {})
|
||||
debug_conversation_id = _agent_roster_service(session).refresh_agent_app_debug_conversation_id(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=str(agent_id),
|
||||
account_id=current_user.id,
|
||||
draft_type=args.draft_type,
|
||||
)
|
||||
return AgentDebugConversationRefreshResponse(
|
||||
debug_conversation_id=debug_conversation_id,
|
||||
@@ -690,6 +716,7 @@ class AgentPublishApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -712,6 +739,7 @@ class AgentBuildDraftCheckoutApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -729,6 +757,7 @@ class AgentBuildDraftCheckoutApi(Resource):
|
||||
@console_ns.route("/agent/<uuid:agent_id>/build-draft")
|
||||
class AgentBuildDraftApi(Resource):
|
||||
@console_ns.response(200, "Agent build draft", console_ns.models[AgentBuildDraftResponse.__name__])
|
||||
@console_ns.response(404, "Agent build draft not found")
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@@ -787,6 +816,7 @@ class AgentBuildDraftApplyApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -809,6 +839,7 @@ class AgentAppCopyApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -834,6 +865,7 @@ class AgentApiAccessApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -850,6 +882,7 @@ class AgentApiStatusApi(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
@@ -868,6 +901,7 @@ class AgentApiKeyListApi(BaseApiKeyListResource):
|
||||
token_prefix = "app-"
|
||||
|
||||
@console_ns.response(200, "Agent service API keys", console_ns.models[ApiKeyList.__name__])
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID) -> dict[str, object]:
|
||||
@@ -878,6 +912,7 @@ class AgentApiKeyListApi(BaseApiKeyListResource):
|
||||
@console_ns.response(400, "Maximum keys exceeded")
|
||||
@with_current_tenant_id
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_session
|
||||
def post(self, session: Session, tenant_id: str, agent_id: UUID) -> tuple[dict[str, object], int]:
|
||||
@@ -897,6 +932,7 @@ class AgentApiKeyApi(BaseApiKeyResource):
|
||||
@console_ns.response(204, "Agent service API key deleted")
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@with_session
|
||||
def delete(
|
||||
@@ -942,6 +978,7 @@ class AgentLogsApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -980,6 +1017,7 @@ class AgentLogMessagesApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1018,6 +1056,7 @@ class AgentLogSourcesApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1038,6 +1077,7 @@ class AgentStatisticsSummaryApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
@@ -1063,6 +1103,7 @@ class AgentRosterVersionsApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID):
|
||||
@@ -1078,6 +1119,7 @@ class AgentRosterVersionDetailApi(Resource):
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_tenant_id
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, tenant_id: str, agent_id: UUID, version_id: UUID):
|
||||
@@ -1098,6 +1140,7 @@ class AgentRosterVersionRestoreApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.AGENT_MANAGE, resource_required=False)
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@with_session
|
||||
|
||||
@@ -12,6 +12,7 @@ from werkzeug.exceptions import Forbidden
|
||||
from configs import dify_config
|
||||
from controllers.common.schema import register_response_schema_models
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import dump_response, to_timestamp
|
||||
from libs.login import login_required
|
||||
@@ -194,6 +195,7 @@ class AppApiKeyListResource(BaseApiKeyListResource):
|
||||
@console_ns.doc(params={"resource_id": "App ID"})
|
||||
@console_ns.response(200, "API keys retrieved successfully", console_ns.models[ApiKeyList.__name__])
|
||||
@with_current_tenant_id
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session(write=False)
|
||||
def get(self, session: Session, current_tenant_id: str, resource_id: UUID) -> dict[str, object]:
|
||||
"""Get all API keys for an app"""
|
||||
@@ -210,6 +212,7 @@ class AppApiKeyListResource(BaseApiKeyListResource):
|
||||
@with_current_tenant_id
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
def post(self, session: Session, current_tenant_id: str, resource_id: UUID) -> tuple[dict[str, object], int]:
|
||||
"""Create a new API key for an app"""
|
||||
@@ -233,6 +236,7 @@ class AppApiKeyResource(BaseApiKeyResource):
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
def delete(
|
||||
self,
|
||||
|
||||
@@ -9,7 +9,7 @@ from flask_restx import Resource
|
||||
from pydantic import AliasChoices, BaseModel, Field, ValidationInfo, computed_field, field_validator, model_validator
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import BadRequest, NotFound
|
||||
from werkzeug.exceptions import BadRequest, Forbidden, NotFound
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.app_access import resolve_app_access_filter
|
||||
@@ -23,7 +23,7 @@ from controllers.common.schema import (
|
||||
register_schema_models,
|
||||
)
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.wraps import get_app_model, with_session
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app, get_app_model, with_session
|
||||
from controllers.console.workspace.models import LoadBalancingPayload
|
||||
from controllers.console.wraps import (
|
||||
RBACPermission,
|
||||
@@ -75,6 +75,7 @@ from services.entities.knowledge_entities.knowledge_entities import (
|
||||
WeightModel,
|
||||
WeightVectorSetting,
|
||||
)
|
||||
from services.errors.account import NoPermissionError
|
||||
from services.feature_service import FeatureService
|
||||
from tasks.initialize_created_app_rbac_access_task import initialize_created_app_rbac_access_task
|
||||
|
||||
@@ -246,7 +247,7 @@ class ModelConfigPartial(ResponseModel):
|
||||
return to_timestamp(value)
|
||||
|
||||
|
||||
class ModelConfig(ResponseModel):
|
||||
class AppModelConfigResponse(ResponseModel):
|
||||
opening_statement: str | None = None
|
||||
suggested_questions: Any | None = Field(
|
||||
default=None, validation_alias=AliasChoices("suggested_questions_list", "suggested_questions")
|
||||
@@ -419,7 +420,7 @@ class AppDetail(AppResponseModel):
|
||||
icon_background: str | None = None
|
||||
enable_site: bool
|
||||
enable_api: bool
|
||||
model_config_: ModelConfig | None = Field(
|
||||
model_config_: AppModelConfigResponse | None = Field(
|
||||
default=None,
|
||||
validation_alias=AliasChoices("app_model_config", "model_config"),
|
||||
alias="model_config",
|
||||
@@ -525,7 +526,13 @@ def _enrich_app_list_items(session: Session, *, apps: Sequence[App], tenant_id:
|
||||
|
||||
register_enum_models(console_ns, RetrievalMethod, WorkflowExecutionStatus, DatasetPermissionEnum)
|
||||
register_response_schema_models(
|
||||
console_ns, RedirectUrlResponse, SimpleResultResponse, AppImportResponse, AppTraceResponse
|
||||
console_ns,
|
||||
RedirectUrlResponse,
|
||||
SimpleResultResponse,
|
||||
AppImportResponse,
|
||||
AppTraceResponse,
|
||||
AppModelConfigResponse,
|
||||
AppDetail,
|
||||
)
|
||||
|
||||
register_schema_models(
|
||||
@@ -544,10 +551,8 @@ register_schema_models(
|
||||
Tag,
|
||||
WorkflowPartial,
|
||||
ModelConfigPartial,
|
||||
ModelConfig,
|
||||
AppDetailSiteResponse,
|
||||
DeletedTool,
|
||||
AppDetail,
|
||||
AppExportResponse,
|
||||
Segmentation,
|
||||
PreProcessingRule,
|
||||
@@ -823,6 +828,7 @@ class AppApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def put(self, session: Session, app_model: App):
|
||||
@@ -857,6 +863,7 @@ class AppApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_DELETE)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model
|
||||
def delete(self, session: Session, app_model: App):
|
||||
@@ -881,6 +888,7 @@ class AppCopyApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_CREATE_AND_MANAGEMENT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
@get_app_model(mode=None)
|
||||
@@ -892,16 +900,19 @@ class AppCopyApi(Resource):
|
||||
with Session(db.engine, expire_on_commit=False) as session:
|
||||
import_service = AppDslService(session)
|
||||
yaml_content = import_service.export_dsl(app_model=app_model, session=session, include_secret=True)
|
||||
result = import_service.import_app(
|
||||
account=current_user,
|
||||
import_mode=ImportMode.YAML_CONTENT,
|
||||
yaml_content=yaml_content,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
)
|
||||
try:
|
||||
result = import_service.import_app(
|
||||
account=current_user,
|
||||
import_mode=ImportMode.YAML_CONTENT,
|
||||
yaml_content=yaml_content,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
return dump_response(AppImportResponse, result), 400
|
||||
@@ -955,6 +966,7 @@ class AppExportApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_IMPORT_EXPORT_DSL)
|
||||
@agent_manage_required_for_agent_app
|
||||
@get_app_model
|
||||
def get(self, app_model: App):
|
||||
"""Export app"""
|
||||
@@ -979,6 +991,7 @@ class AppPublishToCreatorsPlatformApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_IMPORT_EXPORT_DSL)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_current_user_id
|
||||
@get_app_model(mode=None)
|
||||
def post(self, current_user_id: str, app_model: App):
|
||||
@@ -1009,6 +1022,7 @@ class AppNameApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1036,6 +1050,7 @@ class AppIconApi(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1069,6 +1084,7 @@ class AppSiteStatus(Resource):
|
||||
@account_initialization_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
@@ -1096,6 +1112,7 @@ class AppApiStatus(Resource):
|
||||
@is_admin_or_owner_required
|
||||
@account_initialization_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@with_session
|
||||
@get_app_model(mode=None)
|
||||
def post(self, session: Session, app_model: App):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.orm import Session
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.schema import register_enum_models, register_schema_models
|
||||
@@ -28,6 +29,7 @@ from services.app_dsl_service import (
|
||||
)
|
||||
from services.enterprise.enterprise_service import EnterpriseService
|
||||
from services.entities.dsl_entities import CheckDependenciesResult, ImportStatus
|
||||
from services.errors.account import NoPermissionError
|
||||
from services.feature_service import FeatureService
|
||||
|
||||
from .. import console_ns
|
||||
@@ -91,18 +93,21 @@ class AppImportApi(Resource):
|
||||
import_service = AppDslService(session)
|
||||
# Import app
|
||||
account = current_user
|
||||
result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode=args.mode,
|
||||
yaml_content=args.yaml_content,
|
||||
yaml_url=args.yaml_url,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
app_id=args.app_id,
|
||||
)
|
||||
try:
|
||||
result = import_service.import_app(
|
||||
account=account,
|
||||
import_mode=args.mode,
|
||||
yaml_content=args.yaml_content,
|
||||
yaml_url=args.yaml_url,
|
||||
name=args.name,
|
||||
description=args.description,
|
||||
icon_type=args.icon_type,
|
||||
icon=args.icon,
|
||||
icon_background=args.icon_background,
|
||||
app_id=args.app_id,
|
||||
)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
else:
|
||||
@@ -157,7 +162,10 @@ class AppImportConfirmApi(Resource):
|
||||
import_service = AppDslService(session)
|
||||
# Confirm import
|
||||
account = current_user
|
||||
result = import_service.confirm_import(import_id=import_id, account=account)
|
||||
try:
|
||||
result = import_service.confirm_import(import_id=import_id, account=account)
|
||||
except NoPermissionError as e:
|
||||
raise Forbidden(str(e))
|
||||
if result.status == ImportStatus.FAILED:
|
||||
session.rollback()
|
||||
else:
|
||||
|
||||
@@ -49,6 +49,7 @@ from libs import helper
|
||||
from libs.helper import uuid_value
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from models.agent import AgentConfigDraftType
|
||||
from models.model import App, AppMode
|
||||
from services.agent.errors import AgentNotFoundError
|
||||
from services.agent.roster_service import AgentRosterService
|
||||
@@ -343,14 +344,23 @@ class AgentChatMessageStopApi(Resource):
|
||||
|
||||
|
||||
def _resolve_current_user_agent_debug_conversation_id(
|
||||
*, session: Session, current_tenant_id: str, current_user: Account, app_model: App, agent_id: str | None
|
||||
*,
|
||||
session: Session,
|
||||
current_tenant_id: str,
|
||||
current_user: Account,
|
||||
app_model: App,
|
||||
agent_id: str | None,
|
||||
draft_type: AgentConfigDraftType,
|
||||
) -> str:
|
||||
"""Resolve the current editor's conversation without crossing draft surfaces."""
|
||||
|
||||
roster_service = AgentRosterService(session)
|
||||
if agent_id:
|
||||
return roster_service.get_or_create_agent_app_debug_conversation_id(
|
||||
tenant_id=current_tenant_id,
|
||||
agent_id=agent_id,
|
||||
account_id=current_user.id,
|
||||
draft_type=draft_type,
|
||||
)
|
||||
|
||||
agent = roster_service.get_app_backing_agent(tenant_id=current_tenant_id, app_id=str(app_model.id))
|
||||
@@ -360,6 +370,7 @@ def _resolve_current_user_agent_debug_conversation_id(
|
||||
tenant_id=current_tenant_id,
|
||||
agent_id=agent.id,
|
||||
account_id=current_user.id,
|
||||
draft_type=draft_type,
|
||||
)
|
||||
|
||||
|
||||
@@ -382,6 +393,7 @@ def _create_chat_message(
|
||||
current_user=current_user,
|
||||
app_model=app_model,
|
||||
agent_id=agent_id,
|
||||
draft_type=AgentConfigDraftType(args_model.draft_type),
|
||||
)
|
||||
if args_model.conversation_id and args_model.conversation_id != debug_conversation_id:
|
||||
raise NotFound("Conversation Not Exists.")
|
||||
@@ -418,6 +430,7 @@ def _create_build_chat_finalization_message(
|
||||
current_user=current_user,
|
||||
app_model=app_model,
|
||||
agent_id=agent_id,
|
||||
draft_type=AgentConfigDraftType.DEBUG_BUILD,
|
||||
)
|
||||
args: dict[str, Any] = {
|
||||
"query": _BUILD_CHAT_FINALIZATION_QUERY,
|
||||
|
||||
@@ -10,7 +10,7 @@ from constants.languages import supported_language
|
||||
from controllers.common.schema import register_schema_models
|
||||
from controllers.common.session import with_session
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.app.wraps import get_app_model
|
||||
from controllers.console.app.wraps import agent_manage_required_for_agent_app, get_app_model
|
||||
from controllers.console.wraps import (
|
||||
RBACPermission,
|
||||
RBACResourceScope,
|
||||
@@ -93,6 +93,7 @@ class AppSite(Resource):
|
||||
@login_required
|
||||
@edit_permission_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_session
|
||||
@@ -145,6 +146,7 @@ class AppSiteAccessTokenReset(Resource):
|
||||
@login_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_RELEASE_AND_VERSION)
|
||||
@agent_manage_required_for_agent_app
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_session
|
||||
|
||||
@@ -58,6 +58,7 @@ from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from factories import file_factory, variable_factory
|
||||
from fields.base import ResponseModel
|
||||
from fields.conversation_variable_fields import WorkflowConversationVariableResponse
|
||||
from fields.member_fields import SimpleAccount
|
||||
from fields.workflow_run_fields import WorkflowRunNodeExecutionResponse
|
||||
from graphon.enums import NodeType
|
||||
@@ -104,6 +105,7 @@ class SyncDraftWorkflowPayload(BaseModel):
|
||||
graph: dict[str, Any]
|
||||
features: dict[str, Any]
|
||||
hash: str | None = None
|
||||
is_collaborative: bool = Field(default=False, alias="_is_collaborative")
|
||||
environment_variables: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
)
|
||||
@@ -238,21 +240,6 @@ class WorkflowOnlineUsersPayload(BaseModel):
|
||||
return list(dict.fromkeys(app_id.strip() for app_id in app_ids if app_id.strip()))
|
||||
|
||||
|
||||
class WorkflowConversationVariableResponse(ResponseModel):
|
||||
id: str
|
||||
name: str
|
||||
value_type: str
|
||||
value: Any
|
||||
description: str
|
||||
|
||||
@field_validator("value_type", mode="before")
|
||||
@classmethod
|
||||
def _serialize_value_type(cls, value: Any) -> str:
|
||||
if hasattr(value, "exposed_type"):
|
||||
return str(value.exposed_type())
|
||||
return str(value)
|
||||
|
||||
|
||||
class PipelineVariableResponse(ResponseModel):
|
||||
label: str
|
||||
variable: str
|
||||
@@ -372,6 +359,12 @@ class WorkflowPublishResponse(ResponseModel):
|
||||
created_at: int
|
||||
|
||||
|
||||
class SyncDraftWorkflowResponse(ResponseModel):
|
||||
result: str
|
||||
hash: str
|
||||
updated_at: int
|
||||
|
||||
|
||||
class WorkflowRestoreResponse(ResponseModel):
|
||||
result: str
|
||||
hash: str
|
||||
@@ -454,6 +447,7 @@ register_response_schema_models(
|
||||
WorkflowOnlineUsersByApp,
|
||||
WorkflowOnlineUsersResponse,
|
||||
WorkflowPublishResponse,
|
||||
SyncDraftWorkflowResponse,
|
||||
WorkflowRestoreResponse,
|
||||
DefaultBlockConfigsResponse,
|
||||
DefaultBlockConfigResponse,
|
||||
@@ -569,14 +563,7 @@ class DraftWorkflowApi(Resource):
|
||||
@console_ns.response(
|
||||
200,
|
||||
"Draft workflow synced successfully",
|
||||
console_ns.model(
|
||||
"SyncDraftWorkflowResponse",
|
||||
{
|
||||
"result": fields.String,
|
||||
"hash": fields.String,
|
||||
"updated_at": fields.String,
|
||||
},
|
||||
),
|
||||
console_ns.models[SyncDraftWorkflowResponse.__name__],
|
||||
)
|
||||
@console_ns.response(400, "Invalid workflow configuration")
|
||||
@console_ns.response(403, "Permission denied")
|
||||
@@ -624,17 +611,21 @@ class DraftWorkflowApi(Resource):
|
||||
environment_variables=environment_variables,
|
||||
conversation_variables=conversation_variables,
|
||||
session=db.session(),
|
||||
graph_only=args["is_collaborative"],
|
||||
)
|
||||
except WorkflowHashNotEqualError:
|
||||
raise DraftWorkflowNotSync()
|
||||
except VariableError as e:
|
||||
raise InvalidArgumentError(description=str(e))
|
||||
|
||||
return {
|
||||
"result": "success",
|
||||
"hash": workflow.unique_hash,
|
||||
"updated_at": TimestampField().format(workflow.updated_at or workflow.created_at),
|
||||
}
|
||||
return dump_response(
|
||||
SyncDraftWorkflowResponse,
|
||||
{
|
||||
"result": "success",
|
||||
"hash": workflow.unique_hash,
|
||||
"updated_at": TimestampField().format(workflow.updated_at or workflow.created_at),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflows/draft/run")
|
||||
|
||||
@@ -12,14 +12,22 @@ from typing import cast, overload
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.common.session import with_session
|
||||
from controllers.common.wraps import RBACPermission, RBACResourceScope, enforce_rbac_access
|
||||
from controllers.console.app.error import AppNotFoundError
|
||||
from extensions.ext_database import db
|
||||
from libs.login import current_account_with_tenant
|
||||
from models import App, AppMode, TrialApp
|
||||
from models.agent import AgentScope
|
||||
from services.recommended_app_service import RecommendedAppService
|
||||
|
||||
__all__ = ["get_app_model", "get_app_model_with_trial", "with_session"]
|
||||
__all__ = [
|
||||
"agent_manage_required_for_agent_app",
|
||||
"get_app_model",
|
||||
"get_app_model_with_trial",
|
||||
"with_session",
|
||||
]
|
||||
|
||||
|
||||
def _load_app_model(session: Session, app_id: str) -> App | None:
|
||||
@@ -48,6 +56,45 @@ def _load_app_model_with_trial(session: Session, app_id: str) -> App | None:
|
||||
return app_model
|
||||
|
||||
|
||||
def agent_manage_required_for_agent_app[**P, R](view: Callable[P, R]) -> Callable[P, R]:
|
||||
"""Gate generic app management routes that target an Agent App.
|
||||
|
||||
A hidden workflow-only backing App only reuses the App runtime and is not
|
||||
part of the general app management plane, so generic routes reject it
|
||||
outright. Managing a roster Agent App mutates the roster Agent behind it
|
||||
(rename/icon sync, archive, API enablement), so it additionally requires
|
||||
workspace ``agent.manage`` on top of the route's existing App permission
|
||||
checks when RBAC is enabled. A no-op for non-agent Apps. Must be placed
|
||||
above ``get_app_model`` so the ``app_id`` path parameter is still present.
|
||||
"""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> R:
|
||||
raw_app_id = kwargs.get("app_id") or kwargs.get("resource_id")
|
||||
if raw_app_id is not None:
|
||||
app_model = _load_app_model_from_scoped_session(str(raw_app_id))
|
||||
binding = (
|
||||
app_model.agent_app_binding_with_session(session=db.session(), include_archived=True)
|
||||
if app_model is not None
|
||||
else None
|
||||
)
|
||||
if binding is not None:
|
||||
if binding.scope == AgentScope.WORKFLOW_ONLY:
|
||||
raise AppNotFoundError()
|
||||
if dify_config.RBAC_ENABLED:
|
||||
current_user, current_tenant_id = current_account_with_tenant()
|
||||
enforce_rbac_access(
|
||||
tenant_id=current_tenant_id,
|
||||
account_id=current_user.id,
|
||||
resource_type=RBACResourceScope.WORKSPACE,
|
||||
scene=RBACPermission.AGENT_MANAGE,
|
||||
resource_required=False,
|
||||
)
|
||||
return view(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _get_injected_session(args: tuple[object, ...]) -> Session | None:
|
||||
"""Return the request session inserted by `with_session`, if this handler has been migrated."""
|
||||
if len(args) < 2:
|
||||
|
||||
@@ -7,10 +7,13 @@ from configs import dify_config
|
||||
from constants.languages import supported_language
|
||||
from controllers.common.schema import query_params_from_model, register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.auth.error import InvitationAccountMismatchError
|
||||
from controllers.console.error import AccountInFreezeError, AlreadyActivateError
|
||||
from extensions.ext_database import db
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from libs.helper import EmailStr, timezone
|
||||
from libs.login import current_account_with_tenant
|
||||
from libs.token import extract_access_token
|
||||
from models import AccountStatus
|
||||
from models.account import TenantAccountJoin, TenantAccountRole
|
||||
from services.account_service import RegisterService, TenantService
|
||||
@@ -136,6 +139,12 @@ class ActivateApi(Resource):
|
||||
)
|
||||
@console_ns.response(400, "Already activated or invalid token")
|
||||
def post(self):
|
||||
"""Accept an invitation without letting an existing session act for another account.
|
||||
|
||||
Token-only activation remains available for legacy clients. When the request already
|
||||
carries a console session, that session must belong to the account encoded in the
|
||||
invitation before the token is consumed or tenant membership is changed.
|
||||
"""
|
||||
args = ActivatePayload.model_validate(console_ns.payload)
|
||||
|
||||
normalized_request_email = args.email.lower() if args.email else None
|
||||
@@ -146,6 +155,11 @@ class ActivateApi(Resource):
|
||||
raise AlreadyActivateError()
|
||||
|
||||
account = invitation["account"]
|
||||
if extract_access_token(request):
|
||||
current_account, _ = current_account_with_tenant()
|
||||
if current_account.id != account.id:
|
||||
raise InvitationAccountMismatchError()
|
||||
|
||||
if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(account.email):
|
||||
raise AccountInFreezeError()
|
||||
|
||||
|
||||
@@ -13,6 +13,12 @@ class InvalidEmailError(BaseHTTPException):
|
||||
code = 400
|
||||
|
||||
|
||||
class InvitationAccountMismatchError(BaseHTTPException):
|
||||
error_code = "invitation_account_mismatch"
|
||||
description = "This invitation was sent to another account. Please sign in with the invited account."
|
||||
code = 403
|
||||
|
||||
|
||||
class PasswordMismatchError(BaseHTTPException):
|
||||
error_code = "password_mismatch"
|
||||
description = "The passwords do not match."
|
||||
|
||||
@@ -16,7 +16,6 @@ from controllers.console.auth.error import (
|
||||
)
|
||||
from controllers.console.error import AccountNotFound, EmailSendIpLimitError
|
||||
from controllers.console.wraps import email_password_login_enabled, setup_required
|
||||
from events.tenant_event import tenant_was_created
|
||||
from extensions.ext_database import db
|
||||
from libs.helper import EmailStr, extract_remote_ip
|
||||
from libs.password import hash_password
|
||||
@@ -201,7 +200,4 @@ class ForgotPasswordResetApi(Resource):
|
||||
not TenantService.get_join_tenants(account, session=db.session())
|
||||
and FeatureService.get_system_features().is_allow_create_workspace
|
||||
):
|
||||
tenant = TenantService.create_tenant(f"{account.name}'s Workspace", session=db.session())
|
||||
TenantService.create_tenant_member(tenant, account, db.session(), role="owner")
|
||||
account.set_current_tenant_with_session(tenant, session=db.session())
|
||||
tenant_was_created.send(tenant)
|
||||
TenantService.create_owner_tenant(account, session=db.session())
|
||||
|
||||
@@ -42,7 +42,6 @@ from controllers.console.wraps import (
|
||||
setup_required,
|
||||
with_current_user,
|
||||
)
|
||||
from events.tenant_event import tenant_was_created
|
||||
from extensions.ext_database import db
|
||||
from libs.helper import EmailStr, extract_remote_ip
|
||||
from libs.helper import timezone as validate_timezone_string
|
||||
@@ -317,10 +316,7 @@ class EmailCodeLoginApi(Resource):
|
||||
if not FeatureService.get_system_features().is_allow_create_workspace:
|
||||
raise NotAllowedCreateWorkspace()
|
||||
else:
|
||||
new_tenant = TenantService.create_tenant(f"{account.name}'s Workspace", session=db.session())
|
||||
TenantService.create_tenant_member(new_tenant, account, db.session(), role="owner")
|
||||
account.set_current_tenant_with_session(new_tenant, session=db.session())
|
||||
tenant_was_created.send(new_tenant)
|
||||
TenantService.create_owner_tenant(account, session=db.session())
|
||||
|
||||
if account is None:
|
||||
try:
|
||||
|
||||
@@ -6,12 +6,12 @@ from flask import current_app, redirect, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Unauthorized
|
||||
from werkzeug.wrappers import Response
|
||||
|
||||
from configs import dify_config
|
||||
from constants.languages import languages
|
||||
from controllers.common.fields import RedirectResponse
|
||||
from controllers.common.schema import query_params_from_model, register_response_schema_model, register_schema_models
|
||||
from events.tenant_event import tenant_was_created
|
||||
from extensions.ext_database import db
|
||||
from libs.datetime_utils import naive_utc_now
|
||||
from libs.helper import extract_remote_ip
|
||||
@@ -128,6 +128,20 @@ def _preferred_interface_language(language: str | None = None) -> str:
|
||||
return languages[0]
|
||||
|
||||
|
||||
def _redirect_with_console_session(account: Account, target_url: str) -> Response:
|
||||
"""Create a console session and attach its cookies to a redirect response."""
|
||||
token_pair = AccountService.login(
|
||||
account=account,
|
||||
session=db.session(),
|
||||
ip_address=extract_remote_ip(request),
|
||||
)
|
||||
response = redirect(target_url)
|
||||
set_access_token_to_cookie(request, response, token_pair.access_token)
|
||||
set_refresh_token_to_cookie(request, response, token_pair.refresh_token)
|
||||
set_csrf_token_to_cookie(request, response, token_pair.csrf_token)
|
||||
return response
|
||||
|
||||
|
||||
@console_ns.route("/oauth/login/<provider>")
|
||||
class OAuthLogin(Resource):
|
||||
@console_ns.doc("oauth_login")
|
||||
@@ -196,16 +210,26 @@ class OAuthCallback(Resource):
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message={urllib.parse.quote(str(e))}")
|
||||
|
||||
if invite_token and RegisterService.is_valid_invite_token(invite_token):
|
||||
invitation = RegisterService.get_invitation_by_token(token=invite_token)
|
||||
if invitation:
|
||||
invitation_email = invitation.get("email", None)
|
||||
invitation_email_normalized = (
|
||||
invitation_email.lower() if isinstance(invitation_email, str) else invitation_email
|
||||
)
|
||||
if invitation_email_normalized != user_info.email.lower():
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Invalid invitation token.")
|
||||
invitation = RegisterService.get_invitation_if_token_valid(
|
||||
None,
|
||||
None,
|
||||
invite_token,
|
||||
session=db.session(),
|
||||
)
|
||||
if not invitation:
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Invalid invitation token.")
|
||||
if invitation["data"]["email"].lower() != user_info.email.lower():
|
||||
message = "This invitation was sent to another account. Please sign in with the invited account."
|
||||
query = urllib.parse.urlencode({"message": message, "invite_token": invite_token})
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?{query}")
|
||||
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin/invite-settings?invite_token={invite_token}")
|
||||
account = invitation["account"]
|
||||
if account.status == AccountStatus.BANNED:
|
||||
return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Account is banned.")
|
||||
|
||||
AccountService.link_account_integrate(provider, user_info.id, account, session=db.session())
|
||||
target_url = f"{dify_config.CONSOLE_WEB_URL}/signin/invite-settings?invite_token={invite_token}"
|
||||
return _redirect_with_console_session(account, target_url)
|
||||
|
||||
try:
|
||||
account, oauth_new_user = _generate_account(provider, user_info, timezone=timezone, language=language)
|
||||
@@ -240,21 +264,10 @@ class OAuthCallback(Resource):
|
||||
"?message=Workspace not found, please contact system admin to invite you to join in a workspace."
|
||||
)
|
||||
|
||||
token_pair = AccountService.login(
|
||||
account=account,
|
||||
session=db.session(),
|
||||
ip_address=extract_remote_ip(request),
|
||||
)
|
||||
|
||||
target_url = _get_redirect_target(redirect_url)
|
||||
query_char = "&" if "?" in target_url else "?"
|
||||
target_url = f"{target_url}{query_char}oauth_new_user={str(oauth_new_user).lower()}"
|
||||
response = redirect(target_url)
|
||||
|
||||
set_access_token_to_cookie(request, response, token_pair.access_token)
|
||||
set_refresh_token_to_cookie(request, response, token_pair.refresh_token)
|
||||
set_csrf_token_to_cookie(request, response, token_pair.csrf_token)
|
||||
return response
|
||||
return _redirect_with_console_session(account, target_url)
|
||||
|
||||
|
||||
def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Account | None:
|
||||
@@ -282,10 +295,7 @@ def _generate_account(
|
||||
if not FeatureService.get_system_features().is_allow_create_workspace:
|
||||
raise WorkSpaceNotAllowedCreateError()
|
||||
else:
|
||||
new_tenant = TenantService.create_tenant(f"{account.name}'s Workspace", session=db.session())
|
||||
TenantService.create_tenant_member(new_tenant, account, db.session(), role="owner")
|
||||
account.set_current_tenant_with_session(new_tenant, session=db.session())
|
||||
tenant_was_created.send(new_tenant)
|
||||
TenantService.create_owner_tenant(account, session=db.session())
|
||||
|
||||
if not account:
|
||||
normalized_email = user_info.email.lower()
|
||||
|
||||
@@ -607,6 +607,13 @@ class DatasetListApi(Resource):
|
||||
ReplaceMemberBindings(scope=RBACResourceWhitelistScope.ALL),
|
||||
)
|
||||
initialize_created_app_rbac_access_task.delay(current_tenant_id, current_user.id, dataset_id=dataset.id)
|
||||
else:
|
||||
enterprise_rbac_service.RBACService.DatasetAccess.replace_whitelist(
|
||||
current_tenant_id,
|
||||
current_user.id,
|
||||
dataset.id,
|
||||
ReplaceMemberBindings(scope=RBACResourceWhitelistScope.SPECIFIC),
|
||||
)
|
||||
|
||||
permission_keys_map = enterprise_rbac_service.RBACService.DatasetPermissions.batch_get(
|
||||
current_tenant_id,
|
||||
@@ -875,7 +882,7 @@ class DatasetIndexingEstimateApi(Resource):
|
||||
file_details = session.scalars(
|
||||
select(UploadFile).where(UploadFile.tenant_id == current_tenant_id, UploadFile.id.in_(file_ids))
|
||||
).all()
|
||||
if file_details is None:
|
||||
if not file_details:
|
||||
raise NotFound("File not found.")
|
||||
|
||||
if file_details:
|
||||
|
||||
@@ -47,7 +47,9 @@ from controllers.console.explore.error import (
|
||||
NotWorkflowAppError,
|
||||
)
|
||||
from controllers.console.explore.wraps import TrialAppResource, trial_feature_enable
|
||||
from controllers.console.wraps import with_current_user
|
||||
from controllers.console.files import FILE_UPLOAD_PARAMS, upload_file_from_request
|
||||
from controllers.console.remote_files import RemoteFileUploadPayload, upload_remote_file_from_request
|
||||
from controllers.console.wraps import cloud_edition_billing_resource_check, with_current_user
|
||||
from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
|
||||
from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
|
||||
from core.app.apps.base_app_queue_manager import AppQueueManager
|
||||
@@ -60,12 +62,14 @@ from core.errors.error import (
|
||||
from extensions.ext_database import db
|
||||
from extensions.ext_redis import redis_client
|
||||
from fields.base import ResponseModel
|
||||
from fields.conversation_variable_fields import WorkflowConversationVariableResponse
|
||||
from fields.file_fields import FileResponse, FileWithSignedUrl
|
||||
from fields.message_fields import SuggestedQuestionsResponse
|
||||
from graphon.graph_engine.manager import GraphEngineManager
|
||||
from graphon.model_runtime.errors.invoke import InvokeError
|
||||
from libs import helper
|
||||
from libs.helper import dump_response, to_timestamp, uuid_value
|
||||
from models import Account
|
||||
from models import Account, App
|
||||
from models.account import TenantStatus
|
||||
from models.model import AppMode, Site, load_annotation_reply_config
|
||||
from models.workflow import Workflow
|
||||
@@ -373,7 +377,7 @@ class TrialWorkflowResponse(ResponseModel):
|
||||
updated_at: int | None = None
|
||||
tool_published: bool | None = None
|
||||
environment_variables: list[JsonObject] = Field(default_factory=list)
|
||||
conversation_variables: list[JsonObject] = Field(default_factory=list)
|
||||
conversation_variables: list[WorkflowConversationVariableResponse] = Field(default_factory=list)
|
||||
rag_pipeline_variables: list[JsonObject] = Field(default_factory=list)
|
||||
|
||||
@field_validator("created_at", "updated_at", mode="before")
|
||||
@@ -427,6 +431,36 @@ register_response_schema_models(
|
||||
simple_account_model = console_ns.models[TrialSimpleAccount.__name__]
|
||||
|
||||
|
||||
class TrialAppFileUploadApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account, app_model: App):
|
||||
"""Upload a file into the tenant that owns the trial app."""
|
||||
upload_file = upload_file_from_request(
|
||||
current_user=current_user,
|
||||
resource_tenant_id=app_model.tenant_id,
|
||||
)
|
||||
return dump_response(FileResponse, upload_file), 201
|
||||
|
||||
|
||||
class TrialAppRemoteFileUploadApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileWithSignedUrl.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account, app_model: App):
|
||||
"""Upload a remote file into the tenant that owns the trial app."""
|
||||
remote_file = upload_remote_file_from_request(
|
||||
current_user=current_user,
|
||||
resource_tenant_id=app_model.tenant_id,
|
||||
)
|
||||
return remote_file.model_dump(mode="json"), 201
|
||||
|
||||
|
||||
class TrialAppWorkflowRunApi(TrialAppResource):
|
||||
@trial_feature_enable
|
||||
@console_ns.expect(console_ns.models[WorkflowRunRequest.__name__])
|
||||
@@ -611,7 +645,7 @@ class TrialChatAudioApi(TrialAppResource):
|
||||
def post(self, current_user: Account, trial_app):
|
||||
app_model = trial_app
|
||||
|
||||
file = request.files["file"]
|
||||
file = request.files.get("file")
|
||||
|
||||
try:
|
||||
# Get IDs before they might be detached from session
|
||||
@@ -886,6 +920,18 @@ class DatasetListApi(Resource):
|
||||
|
||||
console_ns.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialAppFileUploadApi,
|
||||
"/trial-apps/<uuid:app_id>/files/upload",
|
||||
endpoint="trial_app_file_upload",
|
||||
)
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialAppRemoteFileUploadApi,
|
||||
"/trial-apps/<uuid:app_id>/remote-files/upload",
|
||||
endpoint="trial_app_remote_file_upload",
|
||||
)
|
||||
|
||||
console_ns.add_resource(
|
||||
TrialMessageSuggestedQuestionApi,
|
||||
"/trial-apps/<uuid:app_id>/messages/<uuid:message_id>/suggested-questions",
|
||||
|
||||
@@ -29,7 +29,7 @@ from extensions.ext_database import db
|
||||
from fields.file_fields import FileResponse, UploadConfig
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from models import Account, UploadFile
|
||||
from services.file_service import FileService
|
||||
|
||||
from . import console_ns
|
||||
@@ -39,7 +39,7 @@ register_response_schema_models(console_ns, AllowedExtensionsResponse, TextConte
|
||||
|
||||
PREVIEW_WORDS_LIMIT = 3000
|
||||
|
||||
_FILE_UPLOAD_PARAMS = {
|
||||
FILE_UPLOAD_PARAMS = {
|
||||
"file": {
|
||||
"description": "File to upload",
|
||||
"in": "formData",
|
||||
@@ -56,6 +56,43 @@ _FILE_UPLOAD_PARAMS = {
|
||||
}
|
||||
|
||||
|
||||
def upload_file_from_request(*, current_user: Account, resource_tenant_id: str | None = None) -> UploadFile:
|
||||
"""Validate the multipart request and persist the file under the requested resource tenant."""
|
||||
source_str = request.form.get("source")
|
||||
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
|
||||
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
|
||||
if len(request.files) > 1:
|
||||
raise TooManyFilesError()
|
||||
file = request.files["file"]
|
||||
|
||||
if not file.filename:
|
||||
raise FilenameNotExistsError
|
||||
if source == "datasets" and not current_user.is_dataset_editor:
|
||||
raise Forbidden()
|
||||
|
||||
if source not in ("datasets", None):
|
||||
source = None
|
||||
|
||||
try:
|
||||
return FileService(db.engine).upload_file(
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mimetype=file.mimetype,
|
||||
user=current_user,
|
||||
tenant_id=resource_tenant_id,
|
||||
source=source,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
|
||||
raise BlockedFileExtensionError(blocked_extension_error.description)
|
||||
|
||||
|
||||
@console_ns.route("/files/upload")
|
||||
class FileApi(Resource):
|
||||
@setup_required
|
||||
@@ -81,42 +118,11 @@ class FileApi(Resource):
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@cloud_edition_billing_resource_check("documents")
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=FILE_UPLOAD_PARAMS)
|
||||
@console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
|
||||
@with_current_user
|
||||
def post(self, current_user: Account):
|
||||
source_str = request.form.get("source")
|
||||
source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
|
||||
|
||||
if "file" not in request.files:
|
||||
raise NoFileUploadedError()
|
||||
|
||||
if len(request.files) > 1:
|
||||
raise TooManyFilesError()
|
||||
file = request.files["file"]
|
||||
|
||||
if not file.filename:
|
||||
raise FilenameNotExistsError
|
||||
if source == "datasets" and not current_user.is_dataset_editor:
|
||||
raise Forbidden()
|
||||
|
||||
if source not in ("datasets", None):
|
||||
source = None
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file.filename,
|
||||
content=file.stream.read(),
|
||||
mimetype=file.mimetype,
|
||||
user=current_user,
|
||||
source=source,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
except services.errors.file.BlockedFileExtensionError as blocked_extension_error:
|
||||
raise BlockedFileExtensionError(blocked_extension_error.description)
|
||||
upload_file = upload_file_from_request(current_user=current_user)
|
||||
|
||||
return dump_response(FileResponse, upload_file), 201
|
||||
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
"""Authenticated transport adapter for the Console-to-KnowledgeFS proxy.
|
||||
|
||||
These raw Blueprint routes deliberately stay outside Dify's OpenAPI surface:
|
||||
KnowledgeFS owns the wire contract consumed by the frontend. The catch-all path
|
||||
avoids resource-specific Dify controllers, while the forwarding module consumes
|
||||
only the operations explicitly enabled by Dify's product registry. The registry
|
||||
can be validated explicitly against the pinned KnowledgeFS contract during development.
|
||||
Console auth and contract-specific dataset RBAC run before forwarding. Request
|
||||
bodies are capped at 64 MiB, JSON and binary responses have separate bounds,
|
||||
SSE responses remain streaming with a bounded idle read timeout, and only safe
|
||||
response headers are exposed. Operation-specific upstream error mappings are
|
||||
applied before Console JSON error handling; the default maps 401 to 502 so it
|
||||
cannot trigger browser-session recovery and preserves resource-level 403.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable, Iterator
|
||||
from functools import wraps
|
||||
from http import HTTPStatus
|
||||
from typing import NoReturn, cast
|
||||
|
||||
import httpx
|
||||
from flask import Response, request, stream_with_context
|
||||
from flask.typing import ResponseReturnValue
|
||||
from werkzeug.exceptions import (
|
||||
BadGateway,
|
||||
Forbidden,
|
||||
GatewayTimeout,
|
||||
HTTPException,
|
||||
NotFound,
|
||||
RequestEntityTooLarge,
|
||||
ServiceUnavailable,
|
||||
default_exceptions,
|
||||
)
|
||||
|
||||
from configs import dify_config
|
||||
from controllers.console import api, bp
|
||||
from controllers.console.wraps import (
|
||||
account_initialization_required,
|
||||
cloud_edition_billing_rate_limit_check,
|
||||
setup_required,
|
||||
)
|
||||
from core.helper import ssrf_proxy
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from services.knowledge_fs_operations import KnowledgeFSMethod
|
||||
from services.knowledge_fs_proxy import (
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSAuthorization,
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
KnowledgeFSUpstreamResponse,
|
||||
authorize_knowledge_fs_request,
|
||||
get_knowledge_fs_operation,
|
||||
proxy_authorized_knowledge_fs_request,
|
||||
proxy_knowledge_fs_request,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
type _KnowledgeFSRequestForwarder = Callable[
|
||||
[str | None, str | None, bytes | None, bytes | None],
|
||||
KnowledgeFSUpstreamResponse,
|
||||
]
|
||||
|
||||
_MAX_PROXY_BODY_BYTES = 64 * 1024 * 1024
|
||||
_RESPONSE_HEADER_ALLOWLIST = (
|
||||
"Cache-Control",
|
||||
"Content-Disposition",
|
||||
"Content-Type",
|
||||
"Retry-After",
|
||||
"X-Trace-Id",
|
||||
)
|
||||
_RESPONSE_HEADER_DENYLIST = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"connection",
|
||||
"cookie",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"set-cookie",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _console_api_errors[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Route raw Blueprint exceptions through the Console API JSON handlers."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
try:
|
||||
return view(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
return api.handle_error(exc)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _knowledge_fs_enabled[**P](
|
||||
view: Callable[P, ResponseReturnValue],
|
||||
) -> Callable[P, ResponseReturnValue]:
|
||||
"""Hide the complete KnowledgeFS route surface while the bridge is disabled."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(*args: P.args, **kwargs: P.kwargs) -> ResponseReturnValue:
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
return view(*args, **kwargs)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _translate_proxy_error(exc: Exception, *, tenant_id: str) -> NoReturn:
|
||||
"""Map forwarding failures to the stable Console HTTP error surface."""
|
||||
if isinstance(exc, KnowledgeFSRouteNotAllowedError):
|
||||
raise NotFound() from exc
|
||||
if isinstance(exc, KnowledgeFSAccessDeniedError):
|
||||
raise Forbidden() from exc
|
||||
if isinstance(exc, KnowledgeFSConfigurationError):
|
||||
logger.error("KnowledgeFS request was blocked by invalid configuration for tenant_id=%s", tenant_id)
|
||||
raise ServiceUnavailable("KnowledgeFS integration is misconfigured") from exc
|
||||
if isinstance(exc, KnowledgeFSTimeoutError):
|
||||
raise GatewayTimeout("KnowledgeFS request timed out") from exc
|
||||
if isinstance(exc, KnowledgeFSTransportError):
|
||||
logger.warning("KnowledgeFS transport request failed for tenant_id=%s", tenant_id)
|
||||
raise BadGateway("KnowledgeFS is unavailable") from exc
|
||||
raise exc
|
||||
|
||||
|
||||
def _knowledge_fs_operation_access_required(
|
||||
view: Callable[[KnowledgeFSAuthorization], ResponseReturnValue],
|
||||
) -> Callable[[KnowledgeFSMethod, str], ResponseReturnValue]:
|
||||
"""Authorize one declared operation before billing and request-body work."""
|
||||
|
||||
@wraps(view)
|
||||
def decorated(method: KnowledgeFSMethod, upstream_path: str) -> ResponseReturnValue:
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
try:
|
||||
authorization = authorize_knowledge_fs_request(
|
||||
account=current_user,
|
||||
tenant_id=tenant_id,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
except KnowledgeFSAccessDeniedError as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return view(authorization)
|
||||
|
||||
return decorated
|
||||
|
||||
|
||||
def _request_body() -> bytes:
|
||||
"""Read the raw body up to the proxy limit or raise RequestEntityTooLarge."""
|
||||
body = request.stream.read(_MAX_PROXY_BODY_BYTES + 1)
|
||||
if len(body) > _MAX_PROXY_BODY_BYTES:
|
||||
raise RequestEntityTooLarge("KnowledgeFS proxy request body is too large")
|
||||
return body
|
||||
|
||||
|
||||
def _stream_response_body(
|
||||
upstream: httpx.Response,
|
||||
*,
|
||||
tenant_id: str,
|
||||
max_response_bytes: int,
|
||||
) -> Iterator[bytes]:
|
||||
"""Yield one bounded SSE response and always release its pooled connection."""
|
||||
total_bytes = 0
|
||||
try:
|
||||
for chunk in upstream.iter_bytes():
|
||||
total_bytes += len(chunk)
|
||||
if total_bytes > max_response_bytes:
|
||||
logger.warning("KnowledgeFS stream exceeded the proxy limit for tenant_id=%s", tenant_id)
|
||||
raise ssrf_proxy.ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
|
||||
yield chunk
|
||||
finally:
|
||||
upstream.close()
|
||||
|
||||
|
||||
def _proxy_response(
|
||||
upstream_result: KnowledgeFSUpstreamResponse,
|
||||
*,
|
||||
tenant_id: str,
|
||||
contract_response_headers: tuple[str, ...],
|
||||
max_response_bytes: int,
|
||||
) -> Response:
|
||||
"""Expose raw content, status, and allowlisted headers from KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: KnowledgeFS returns a status normalized by the operation contract.
|
||||
"""
|
||||
upstream = upstream_result.response
|
||||
mapped_status = dict(upstream_result.operation.error_status_map).get(upstream.status_code)
|
||||
if mapped_status is not None:
|
||||
upstream.close()
|
||||
description = "KnowledgeFS upstream request failed"
|
||||
if upstream.status_code == HTTPStatus.UNAUTHORIZED:
|
||||
description = "KnowledgeFS authentication failed"
|
||||
logger.error(
|
||||
"KnowledgeFS rejected the Dify server credential with HTTP %s for tenant_id=%s",
|
||||
upstream.status_code,
|
||||
tenant_id,
|
||||
)
|
||||
exception_type = default_exceptions.get(mapped_status)
|
||||
if exception_type is None:
|
||||
exception = HTTPException(description)
|
||||
exception.code = mapped_status
|
||||
raise exception
|
||||
raise exception_type(description)
|
||||
|
||||
allowed_header_names = dict.fromkeys(
|
||||
name.lower() for name in (*_RESPONSE_HEADER_ALLOWLIST, *contract_response_headers)
|
||||
)
|
||||
headers = {
|
||||
name: value
|
||||
for name in allowed_header_names
|
||||
if name not in _RESPONSE_HEADER_DENYLIST
|
||||
if (value := upstream.headers.get(name)) is not None
|
||||
}
|
||||
if upstream_result.response_kind == "stream":
|
||||
response = Response(
|
||||
stream_with_context( # pyrefly: ignore[no-matching-overload]
|
||||
_stream_response_body(
|
||||
upstream,
|
||||
tenant_id=tenant_id,
|
||||
max_response_bytes=max_response_bytes,
|
||||
)
|
||||
),
|
||||
status=upstream.status_code,
|
||||
headers=headers,
|
||||
)
|
||||
response.call_on_close(upstream.close)
|
||||
return response
|
||||
|
||||
try:
|
||||
content = upstream.content
|
||||
finally:
|
||||
upstream.close()
|
||||
return Response(content, status=upstream.status_code, headers=headers)
|
||||
|
||||
|
||||
def _proxy_current_request(
|
||||
*,
|
||||
method: KnowledgeFSMethod,
|
||||
tenant_id: str,
|
||||
forward: _KnowledgeFSRequestForwarder,
|
||||
) -> Response:
|
||||
"""Forward the current raw request through one preconfigured service entry."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
try:
|
||||
proxy_result = forward(
|
||||
request.headers.get("Accept"),
|
||||
request.content_type,
|
||||
request.query_string or None,
|
||||
_request_body() if method != "GET" else None,
|
||||
)
|
||||
except (
|
||||
KnowledgeFSConfigurationError,
|
||||
KnowledgeFSAccessDeniedError,
|
||||
KnowledgeFSRouteNotAllowedError,
|
||||
KnowledgeFSTimeoutError,
|
||||
KnowledgeFSTransportError,
|
||||
) as exc:
|
||||
_translate_proxy_error(exc, tenant_id=tenant_id)
|
||||
return _proxy_response(
|
||||
proxy_result,
|
||||
tenant_id=tenant_id,
|
||||
contract_response_headers=proxy_result.operation.response_headers,
|
||||
max_response_bytes=proxy_result.operation.max_response_bytes,
|
||||
)
|
||||
|
||||
|
||||
def _proxy_request(
|
||||
method: KnowledgeFSMethod,
|
||||
upstream_path: str,
|
||||
) -> Response:
|
||||
"""Authorize and forward the current request through the combined service use case."""
|
||||
if not dify_config.KNOWLEDGE_FS_ENABLED:
|
||||
raise NotFound()
|
||||
current_user, tenant_id = current_account_with_tenant()
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_knowledge_fs_request(
|
||||
account=current_user,
|
||||
method=method,
|
||||
path=upstream_path,
|
||||
tenant_id=tenant_id,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
def _proxy_authorized_request(authorization: KnowledgeFSAuthorization) -> Response:
|
||||
"""Forward the current request using one previously authorized operation capability.
|
||||
|
||||
Args:
|
||||
authorization: Request-scoped capability produced before billing and body parsing.
|
||||
|
||||
Returns:
|
||||
The filtered response returned by KnowledgeFS.
|
||||
|
||||
Raises:
|
||||
HTTPException: The integration is disabled or forwarding fails.
|
||||
"""
|
||||
operation = authorization.operation
|
||||
tenant_id = authorization.tenant_id
|
||||
|
||||
def forward(
|
||||
accept: str | None,
|
||||
content_type: str | None,
|
||||
query: bytes | None,
|
||||
body: bytes | None,
|
||||
) -> KnowledgeFSUpstreamResponse:
|
||||
return proxy_authorized_knowledge_fs_request(
|
||||
authorization=authorization,
|
||||
accept=accept,
|
||||
content_type=content_type,
|
||||
query=query,
|
||||
body=body,
|
||||
request_headers=request.headers,
|
||||
)
|
||||
|
||||
return _proxy_current_request(method=operation.method, tenant_id=tenant_id, forward=forward)
|
||||
|
||||
|
||||
@_knowledge_fs_enabled
|
||||
@_knowledge_fs_operation_access_required
|
||||
@cloud_edition_billing_rate_limit_check("knowledge")
|
||||
def _proxy_knowledge_fs_non_get(
|
||||
authorization: KnowledgeFSAuthorization,
|
||||
) -> ResponseReturnValue:
|
||||
"""Apply knowledge billing checks to one allowlisted non-GET operation."""
|
||||
return _proxy_authorized_request(authorization)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["OPTIONS"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
def proxy_knowledge_fs_options(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Complete a CORS preflight only for an enabled Console operation."""
|
||||
requested_method = cast(KnowledgeFSMethod, request.headers.get("Access-Control-Request-Method", "").upper())
|
||||
try:
|
||||
get_knowledge_fs_operation(requested_method, upstream_path)
|
||||
except KnowledgeFSRouteNotAllowedError as exc:
|
||||
raise NotFound() from exc
|
||||
return Response(status=HTTPStatus.NO_CONTENT)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["GET"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_get(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated, dataset-readable GET request.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
if request.method != "GET":
|
||||
raise NotFound()
|
||||
return _proxy_request("GET", upstream_path)
|
||||
|
||||
|
||||
@bp.route(
|
||||
"/knowledge-fs/<path:upstream_path>",
|
||||
methods=["DELETE", "PATCH", "POST", "PUT"],
|
||||
provide_automatic_options=False,
|
||||
)
|
||||
@_console_api_errors
|
||||
@_knowledge_fs_enabled
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
def proxy_knowledge_fs_write(upstream_path: str) -> ResponseReturnValue:
|
||||
"""Forward one authenticated non-GET request under its contract access policy.
|
||||
|
||||
Args:
|
||||
upstream_path: Relative KFS path captured after the Console proxy prefix.
|
||||
|
||||
Returns:
|
||||
The filtered raw KnowledgeFS response or a Console JSON error response.
|
||||
"""
|
||||
method = cast(KnowledgeFSMethod, request.method)
|
||||
return _proxy_knowledge_fs_non_get(method, upstream_path)
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Console onboarding APIs.
|
||||
|
||||
This module keeps Step-by-step Tour persistence account-scoped. Workspace IDs
|
||||
are accepted only as presentation overrides; UI-only state such as minimized
|
||||
panels or the currently active task stays on the frontend. PATCH requests are
|
||||
action-based so callers do not replace server-side arrays with stale snapshots.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal, cast
|
||||
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from controllers.common.schema import register_response_schema_models, register_schema_models
|
||||
from extensions.ext_database import db
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import dump_response
|
||||
from libs.login import login_required
|
||||
from models import Account
|
||||
from services.step_by_step_tour_service import StepByStepTourPatch, StepByStepTourService
|
||||
|
||||
from . import console_ns
|
||||
from .wraps import account_initialization_required, setup_required, with_current_tenant_id, with_current_user
|
||||
|
||||
StepByStepTourAction = Literal[
|
||||
"skip",
|
||||
"complete_task",
|
||||
"uncomplete_task",
|
||||
"enable_current_workspace",
|
||||
"disable_current_workspace",
|
||||
]
|
||||
StepByStepTourTaskId = Literal["home", "studio", "knowledge", "integration"]
|
||||
|
||||
|
||||
class StepByStepTourStatePatchPayload(BaseModel):
|
||||
action: StepByStepTourAction = Field(description="State update action")
|
||||
task_id: StepByStepTourTaskId | None = Field(default=None, description="Task ID for task actions")
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_patch_shape(self) -> "StepByStepTourStatePatchPayload":
|
||||
task_actions = {"complete_task", "uncomplete_task"}
|
||||
if self.action in task_actions and self.task_id is None:
|
||||
raise ValueError("task_id is required for task actions")
|
||||
if self.action not in task_actions and self.task_id is not None:
|
||||
raise ValueError("task_id is only supported for task actions")
|
||||
|
||||
return self
|
||||
|
||||
|
||||
class StepByStepTourStateResponse(ResponseModel):
|
||||
first_workspace_id: str | None = None
|
||||
skipped: bool = False
|
||||
completed_task_ids: list[StepByStepTourTaskId] = Field(default_factory=list)
|
||||
manually_enabled_workspace_ids: list[str] = Field(default_factory=list)
|
||||
manually_disabled_workspace_ids: list[str] = Field(default_factory=list)
|
||||
updated_at: datetime | None = None
|
||||
|
||||
|
||||
register_schema_models(console_ns, StepByStepTourStatePatchPayload)
|
||||
register_response_schema_models(console_ns, StepByStepTourStateResponse)
|
||||
|
||||
|
||||
@console_ns.route("/onboarding/step-by-step-tour/state")
|
||||
class StepByStepTourStateApi(Resource):
|
||||
@console_ns.doc("get_step_by_step_tour_state")
|
||||
@console_ns.doc(description="Get account-level Step-by-step Tour state")
|
||||
@console_ns.response(200, "Success", console_ns.models[StepByStepTourStateResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def get(self, current_tenant_id: str, current_user: Account):
|
||||
return dump_response(
|
||||
StepByStepTourStateResponse,
|
||||
StepByStepTourService.get_state(
|
||||
account=current_user,
|
||||
current_tenant_id=current_tenant_id,
|
||||
session=db.session,
|
||||
),
|
||||
)
|
||||
|
||||
@console_ns.doc("patch_step_by_step_tour_state")
|
||||
@console_ns.doc(description="Update account-level Step-by-step Tour state")
|
||||
@console_ns.expect(console_ns.models[StepByStepTourStatePatchPayload.__name__])
|
||||
@console_ns.response(200, "Success", console_ns.models[StepByStepTourStateResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
@account_initialization_required
|
||||
@with_current_user
|
||||
@with_current_tenant_id
|
||||
def patch(self, current_tenant_id: str, current_user: Account):
|
||||
payload = StepByStepTourStatePatchPayload.model_validate(console_ns.payload or {})
|
||||
patch = cast(StepByStepTourPatch, payload.model_dump(exclude_unset=True, exclude_none=True))
|
||||
return dump_response(
|
||||
StepByStepTourStateResponse,
|
||||
StepByStepTourService.patch_state(
|
||||
account=current_user,
|
||||
current_tenant_id=current_tenant_id,
|
||||
patch=patch,
|
||||
session=db.session,
|
||||
),
|
||||
)
|
||||
@@ -46,6 +46,61 @@ class GetRemoteFileInfo(Resource):
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
def upload_remote_file_from_request(
|
||||
*,
|
||||
current_user: Account,
|
||||
resource_tenant_id: str | None = None,
|
||||
) -> FileWithSignedUrl:
|
||||
"""Validate the JSON request, fetch its remote file, and persist it under the requested tenant."""
|
||||
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
|
||||
url = payload.url
|
||||
|
||||
# Try to fetch remote file metadata/content first
|
||||
try:
|
||||
resp = remote_fetcher.make_request("HEAD", url=url)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
# Normalize into a user-friendly error message expected by tests
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
|
||||
except httpx.RequestError as e:
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
|
||||
|
||||
file_info = helpers.guess_file_info_from_response(resp)
|
||||
|
||||
# Enforce file size limit with 400 (Bad Request) per tests' expectation
|
||||
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
|
||||
raise FileTooLargeError()
|
||||
|
||||
# Load content if needed
|
||||
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file_info.filename,
|
||||
content=content,
|
||||
mimetype=file_info.mimetype,
|
||||
user=current_user,
|
||||
tenant_id=resource_tenant_id,
|
||||
source_url=url,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
return FileWithSignedUrl(
|
||||
id=upload_file.id,
|
||||
name=upload_file.name,
|
||||
size=upload_file.size,
|
||||
extension=upload_file.extension,
|
||||
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
|
||||
mime_type=upload_file.mime_type,
|
||||
created_by=upload_file.created_by,
|
||||
created_at=int(upload_file.created_at.timestamp()),
|
||||
)
|
||||
|
||||
|
||||
@console_ns.route("/remote-files/upload")
|
||||
class RemoteFileUpload(Resource):
|
||||
@console_ns.expect(console_ns.models[RemoteFileUploadPayload.__name__])
|
||||
@@ -53,53 +108,8 @@ class RemoteFileUpload(Resource):
|
||||
@login_required
|
||||
@with_current_user
|
||||
def post(self, current_user: Account):
|
||||
payload = RemoteFileUploadPayload.model_validate(console_ns.payload)
|
||||
url = payload.url
|
||||
|
||||
# Try to fetch remote file metadata/content first
|
||||
try:
|
||||
resp = remote_fetcher.make_request("HEAD", url=url)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
resp = remote_fetcher.make_request("GET", url=url, timeout=3, follow_redirects=True)
|
||||
if resp.status_code != httpx.codes.OK:
|
||||
# Normalize into a user-friendly error message expected by tests
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {resp.text}")
|
||||
except httpx.RequestError as e:
|
||||
raise RemoteFileUploadError(f"Failed to fetch file from {url}: {str(e)}")
|
||||
|
||||
file_info = helpers.guess_file_info_from_response(resp)
|
||||
|
||||
# Enforce file size limit with 400 (Bad Request) per tests' expectation
|
||||
if not FileService.is_file_size_within_limit(extension=file_info.extension, file_size=file_info.size):
|
||||
raise FileTooLargeError()
|
||||
|
||||
# Load content if needed
|
||||
content = resp.content if resp.request.method == "GET" else remote_fetcher.make_request("GET", url).content
|
||||
|
||||
try:
|
||||
upload_file = FileService(db.engine).upload_file(
|
||||
filename=file_info.filename,
|
||||
content=content,
|
||||
mimetype=file_info.mimetype,
|
||||
user=current_user,
|
||||
source_url=url,
|
||||
)
|
||||
except services.errors.file.FileTooLargeError as file_too_large_error:
|
||||
raise FileTooLargeError(file_too_large_error.description)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
# Success: return created resource with 201 status
|
||||
remote_file = upload_remote_file_from_request(current_user=current_user)
|
||||
return (
|
||||
FileWithSignedUrl(
|
||||
id=upload_file.id,
|
||||
name=upload_file.name,
|
||||
size=upload_file.size,
|
||||
extension=upload_file.extension,
|
||||
url=file_helpers.get_signed_file_url(upload_file_id=upload_file.id),
|
||||
mime_type=upload_file.mime_type,
|
||||
created_by=upload_file.created_by,
|
||||
created_at=int(upload_file.created_at.timestamp()),
|
||||
).model_dump(mode="json"),
|
||||
remote_file.model_dump(mode="json"),
|
||||
201,
|
||||
)
|
||||
|
||||
@@ -98,6 +98,7 @@ def handle_collaboration_event(sid, data):
|
||||
6. workflow_update
|
||||
7. comments_update
|
||||
8. node_panel_presence
|
||||
9. graph_view_state (session reports tab visibility; drives leader election)
|
||||
"""
|
||||
return collaboration_service.relay_collaboration_event(sid, data)
|
||||
|
||||
|
||||
@@ -4,20 +4,16 @@ from http import HTTPStatus
|
||||
from flask import redirect
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Conflict, NotFound
|
||||
from werkzeug.exceptions import Conflict, Forbidden, NotFound
|
||||
|
||||
from controllers.common.fields import RedirectResponse
|
||||
from controllers.common.schema import register_response_schema_models, register_schema_models
|
||||
from controllers.console import console_ns
|
||||
from controllers.console.wraps import (
|
||||
RBACPermission,
|
||||
RBACResourceScope,
|
||||
account_initialization_required,
|
||||
cloud_edition_billing_enabled,
|
||||
cloud_edition_billing_paid_plan_required,
|
||||
is_admin_or_owner_required,
|
||||
only_edition_cloud,
|
||||
rbac_permission_required,
|
||||
setup_required,
|
||||
)
|
||||
from extensions.ext_database import db
|
||||
@@ -25,6 +21,7 @@ from fields.base import ResponseModel
|
||||
from libs.archive_storage import get_export_storage
|
||||
from libs.helper import dump_response
|
||||
from libs.login import current_account_with_tenant, login_required
|
||||
from models import TenantAccountRole
|
||||
from services.retention.workflow_run.archive_download_preparation import ARCHIVE_DOWNLOAD_MIME_TYPE
|
||||
from services.retention.workflow_run.archive_download_task_cache import (
|
||||
WorkflowRunArchiveDownloadStatus,
|
||||
@@ -98,11 +95,13 @@ register_response_schema_models(
|
||||
)
|
||||
|
||||
|
||||
def _current_ids() -> tuple[str, str]:
|
||||
"""Return current `(tenant_id, account_id)` or raise when no workspace is selected."""
|
||||
def _current_owner_or_admin_ids() -> tuple[str, str]:
|
||||
"""Return current Cloud workspace IDs for an owner or admin, independently of enterprise RBAC."""
|
||||
current_user, current_tenant_id = current_account_with_tenant()
|
||||
if not current_tenant_id:
|
||||
raise NotFound("Current workspace not found")
|
||||
if not TenantAccountRole.is_privileged_role(current_user.current_role):
|
||||
raise Forbidden()
|
||||
return current_tenant_id, current_user.id
|
||||
|
||||
|
||||
@@ -124,12 +123,8 @@ class WorkflowRunArchivesApi(Resource):
|
||||
@only_edition_cloud
|
||||
@cloud_edition_billing_enabled
|
||||
@cloud_edition_billing_paid_plan_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.WORKSPACE_ROLE_MANAGE, resource_required=False
|
||||
)
|
||||
def get(self):
|
||||
tenant_id, _ = _current_ids()
|
||||
tenant_id, _ = _current_owner_or_admin_ids()
|
||||
return dump_response(WorkflowRunArchiveListResponse, list_workflow_run_archives(db.session(), tenant_id))
|
||||
|
||||
|
||||
@@ -149,12 +144,8 @@ class WorkflowRunArchiveDownloadsApi(Resource):
|
||||
@only_edition_cloud
|
||||
@cloud_edition_billing_enabled
|
||||
@cloud_edition_billing_paid_plan_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.WORKSPACE_ROLE_MANAGE, resource_required=False
|
||||
)
|
||||
def post(self):
|
||||
tenant_id, account_id = _current_ids()
|
||||
tenant_id, account_id = _current_owner_or_admin_ids()
|
||||
payload = WorkflowRunArchiveDownloadPayload.model_validate(console_ns.payload or {})
|
||||
try:
|
||||
task = create_workflow_run_archive_download_task(
|
||||
@@ -180,12 +171,8 @@ class WorkflowRunArchiveDownloadApi(Resource):
|
||||
@only_edition_cloud
|
||||
@cloud_edition_billing_enabled
|
||||
@cloud_edition_billing_paid_plan_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.WORKSPACE_ROLE_MANAGE, resource_required=False
|
||||
)
|
||||
def get(self, download_id: str):
|
||||
tenant_id, _ = _current_ids()
|
||||
tenant_id, _ = _current_owner_or_admin_ids()
|
||||
try:
|
||||
task = get_workflow_run_archive_download_task(tenant_id=tenant_id, download_id=download_id)
|
||||
except WorkflowRunArchiveDownloadTaskNotFoundError as exc:
|
||||
@@ -209,12 +196,8 @@ class WorkflowRunArchiveDownloadFileApi(Resource):
|
||||
@only_edition_cloud
|
||||
@cloud_edition_billing_enabled
|
||||
@cloud_edition_billing_paid_plan_required
|
||||
@is_admin_or_owner_required
|
||||
@rbac_permission_required(
|
||||
RBACResourceScope.WORKSPACE, RBACPermission.WORKSPACE_ROLE_MANAGE, resource_required=False
|
||||
)
|
||||
def get(self, download_id: str):
|
||||
tenant_id, _ = _current_ids()
|
||||
tenant_id, _ = _current_owner_or_admin_ids()
|
||||
try:
|
||||
task = get_ready_workflow_run_archive_download_task(tenant_id=tenant_id, download_id=download_id)
|
||||
except WorkflowRunArchiveDownloadTaskNotFoundError as exc:
|
||||
|
||||
@@ -67,6 +67,15 @@ from services.plugin.plugin_parameter_service import PluginParameterService
|
||||
from services.plugin.plugin_permission_service import PluginPermissionService
|
||||
from services.tools.tools_transform_service import ToolTransformService
|
||||
|
||||
_PLUGIN_PACKAGE_UPLOAD_PARAMS = {
|
||||
"pkg": {
|
||||
"description": "Plugin package to upload",
|
||||
"in": "formData",
|
||||
"type": "file",
|
||||
"required": True,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class AutoUpgradeSettingsResponse(TypedDict):
|
||||
strategy_setting: TenantPluginAutoUpgradeStrategySetting
|
||||
@@ -645,6 +654,7 @@ class PluginAssetApi(Resource):
|
||||
|
||||
@console_ns.route("/workspaces/current/plugin/upload/pkg")
|
||||
class PluginUploadFromPkgApi(Resource):
|
||||
@console_ns.doc(consumes=["multipart/form-data"], params=_PLUGIN_PACKAGE_UPLOAD_PARAMS)
|
||||
@console_ns.response(200, "Success", console_ns.models[PluginDecodeResponse.__name__])
|
||||
@setup_required
|
||||
@login_required
|
||||
|
||||
@@ -88,6 +88,7 @@ class TenantInfoResponse(ResponseModel):
|
||||
custom_config: WorkspaceCustomConfigResponse | None = None
|
||||
trial_credits: int | None = None
|
||||
trial_credits_used: int | None = None
|
||||
trial_credits_exhausted_at: int | None = None
|
||||
next_credit_reset_date: int | None = None
|
||||
|
||||
@field_validator("plan", "status", "trial_end_reason", mode="before")
|
||||
|
||||
@@ -47,10 +47,12 @@ class EnterpriseWorkspace(Resource):
|
||||
if account is None:
|
||||
return {"message": "owner account not found."}, 404
|
||||
|
||||
tenant = TenantService.create_tenant(args.name, is_from_dashboard=True, session=db.session())
|
||||
TenantService.create_tenant_member(tenant, account, db.session(), role="owner")
|
||||
|
||||
tenant_was_created.send(tenant)
|
||||
tenant = TenantService.create_owner_tenant(
|
||||
account,
|
||||
name=args.name,
|
||||
is_from_dashboard=True,
|
||||
session=db.session(),
|
||||
)
|
||||
|
||||
resp = {
|
||||
"id": tenant.id,
|
||||
|
||||
@@ -101,7 +101,7 @@ class AudioApi(Resource):
|
||||
|
||||
Accepts an audio file upload and returns the transcribed text.
|
||||
"""
|
||||
file = request.files["file"]
|
||||
file = request.files.get("file")
|
||||
|
||||
try:
|
||||
response = AudioService.transcript_asr(
|
||||
|
||||
@@ -531,14 +531,22 @@ class DatasetListApi(DatasetApiResource):
|
||||
except services.errors.dataset.DatasetNameDuplicateError:
|
||||
raise DatasetNameDuplicateError()
|
||||
|
||||
if payload.permission == DatasetPermissionEnum.ALL_TEAM and dify_config.RBAC_ENABLED:
|
||||
RBACService.DatasetAccess.replace_whitelist(
|
||||
tenant_id,
|
||||
current_user.id,
|
||||
dataset.id,
|
||||
ReplaceMemberBindings(scope=RBACResourceWhitelistScope.ALL),
|
||||
)
|
||||
initialize_created_app_rbac_access_task.delay(tenant_id, current_user.id, dataset_id=dataset.id)
|
||||
if dify_config.RBAC_ENABLED:
|
||||
if payload.permission == DatasetPermissionEnum.ALL_TEAM:
|
||||
RBACService.DatasetAccess.replace_whitelist(
|
||||
tenant_id,
|
||||
current_user.id,
|
||||
dataset.id,
|
||||
ReplaceMemberBindings(scope=RBACResourceWhitelistScope.ALL),
|
||||
)
|
||||
initialize_created_app_rbac_access_task.delay(tenant_id, current_user.id, dataset_id=dataset.id)
|
||||
else:
|
||||
RBACService.DatasetAccess.replace_whitelist(
|
||||
tenant_id,
|
||||
current_user.id,
|
||||
dataset.id,
|
||||
ReplaceMemberBindings(scope=RBACResourceWhitelistScope.SPECIFIC),
|
||||
)
|
||||
|
||||
return _dump_service_dataset_detail(dataset, session=session), 200
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ class AudioApi(WebApiResource):
|
||||
@web_ns.response(200, "Success", web_ns.models[AudioToTextResponse.__name__])
|
||||
def post(self, app_model: App, end_user: EndUser):
|
||||
"""Convert audio to text"""
|
||||
file = request.files["file"]
|
||||
file = request.files.get("file")
|
||||
|
||||
try:
|
||||
response = AudioService.transcript_asr(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from typing import Any, Self
|
||||
|
||||
from pydantic import AliasChoices, Field, computed_field
|
||||
from pydantic import AliasChoices, Field
|
||||
from sqlalchemy import select
|
||||
from werkzeug.exceptions import Forbidden
|
||||
|
||||
@@ -9,11 +9,13 @@ from controllers.common.schema import register_response_schema_models
|
||||
from controllers.web import web_ns
|
||||
from controllers.web.wraps import WebApiResource
|
||||
from extensions.ext_database import db
|
||||
from extensions.storage.storage_type import StorageType
|
||||
from fields.base import ResponseModel
|
||||
from libs.helper import build_icon_url
|
||||
from models.account import Tenant, TenantStatus
|
||||
from models.model import App, EndUser, Site
|
||||
from models.model import App, EndUser, IconType, Site
|
||||
from services.feature_service import FeatureModel, FeatureService
|
||||
from services.file_service import FileService
|
||||
|
||||
|
||||
class WebSiteResponse(ResponseModel):
|
||||
@@ -32,11 +34,7 @@ class WebSiteResponse(ResponseModel):
|
||||
prompt_public: bool | None = None
|
||||
show_workflow_steps: bool | None = None
|
||||
use_icon_as_answer_icon: bool | None = None
|
||||
|
||||
@computed_field(return_type=str | None) # type: ignore[prop-decorator]
|
||||
@property
|
||||
def icon_url(self) -> str | None:
|
||||
return build_icon_url(self.icon_type, self.icon)
|
||||
icon_url: str | None = None
|
||||
|
||||
|
||||
class WebModelConfigResponse(ResponseModel):
|
||||
@@ -88,6 +86,7 @@ class WebAppSiteResponse(ResponseModel):
|
||||
end_user_id: str | None,
|
||||
features: FeatureModel,
|
||||
can_replace_logo: bool,
|
||||
icon_url: str | None = None,
|
||||
) -> Self:
|
||||
custom_config = None
|
||||
if can_replace_logo:
|
||||
@@ -102,6 +101,7 @@ class WebAppSiteResponse(ResponseModel):
|
||||
)
|
||||
|
||||
site_response = WebSiteResponse.model_validate(site, from_attributes=True)
|
||||
site_response.icon_url = icon_url if icon_url is not None else build_icon_url(site.icon_type, site.icon)
|
||||
if features.billing.enabled and not features.webapp_copyright_enabled:
|
||||
site_response.copyright = None
|
||||
site_response.input_placeholder = None
|
||||
@@ -123,6 +123,15 @@ register_response_schema_models(
|
||||
)
|
||||
|
||||
|
||||
def _build_site_icon_url(*, site: Site, tenant_id: str) -> str | None:
|
||||
"""Use direct S3 URLs only in Cloud Mode and preserve preview URLs elsewhere."""
|
||||
if site.icon_type != IconType.IMAGE or not site.icon:
|
||||
return None
|
||||
if dify_config.EDITION == "CLOUD" and StorageType(dify_config.STORAGE_TYPE) == StorageType.S3:
|
||||
return FileService(db.engine).get_file_presigned_url(file_id=site.icon, tenant_id=tenant_id)
|
||||
return build_icon_url(site.icon_type, site.icon)
|
||||
|
||||
|
||||
@web_ns.route("/site")
|
||||
class AppSiteApi(WebApiResource):
|
||||
@web_ns.doc("Get App Site Info")
|
||||
@@ -159,4 +168,5 @@ class AppSiteApi(WebApiResource):
|
||||
end_user_id=end_user.id,
|
||||
features=features,
|
||||
can_replace_logo=features.can_replace_logo,
|
||||
icon_url=_build_site_icon_url(site=site, tenant_id=tenant.id),
|
||||
).model_dump(mode="json")
|
||||
|
||||
@@ -242,6 +242,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=workflow_triggered_from,
|
||||
@@ -249,6 +250,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
@@ -375,6 +377,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
# Create workflow execution(aka workflow run) repository
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
|
||||
@@ -382,6 +385,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
@@ -466,6 +470,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
# Create workflow execution(aka workflow run) repository
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
|
||||
@@ -473,6 +478,7 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
@@ -619,7 +625,11 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
|
||||
message=message_snapshot,
|
||||
user=user,
|
||||
stream=stream,
|
||||
draft_var_saver_factory=self._get_draft_var_saver_factory(invoke_from, account=user),
|
||||
draft_var_saver_factory=self._get_draft_var_saver_factory(
|
||||
invoke_from,
|
||||
account=user,
|
||||
tenant_id=application_generate_entity.app_config.tenant_id,
|
||||
),
|
||||
)
|
||||
|
||||
return AdvancedChatAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)
|
||||
|
||||
@@ -540,6 +540,9 @@ class AgentAppGenerator(MessageBasedAppGenerator):
|
||||
base_url=dify_config.AGENT_BACKEND_BASE_URL,
|
||||
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
|
||||
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
|
||||
stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
|
||||
stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
|
||||
stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
|
||||
),
|
||||
event_adapter=AgentBackendRunEventAdapter(),
|
||||
session_store=AgentAppRuntimeSessionStore(),
|
||||
@@ -679,42 +682,27 @@ class AgentAppGenerator(MessageBasedAppGenerator):
|
||||
if draft_type == AgentConfigDraftType.DEBUG_BUILD.value
|
||||
else AgentConfigDraftType.DRAFT
|
||||
)
|
||||
if effective_draft_type == AgentConfigDraftType.DRAFT:
|
||||
from services.agent.composer_service import AgentComposerService
|
||||
|
||||
return AgentComposerService.get_or_create_normal_agent_draft(
|
||||
session=session,
|
||||
tenant_id=tenant_id,
|
||||
agent=agent,
|
||||
created_by=agent.updated_by or agent.created_by,
|
||||
)
|
||||
if not account_id:
|
||||
raise AgentAppGeneratorError("Build draft requires an account user")
|
||||
stmt = select(AgentConfigDraft).where(
|
||||
AgentConfigDraft.tenant_id == tenant_id,
|
||||
AgentConfigDraft.agent_id == agent.id,
|
||||
AgentConfigDraft.draft_type == effective_draft_type,
|
||||
AgentConfigDraft.draft_type == AgentConfigDraftType.DEBUG_BUILD,
|
||||
AgentConfigDraft.account_id == account_id,
|
||||
)
|
||||
if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
|
||||
if not account_id:
|
||||
raise AgentAppGeneratorError("Build draft requires an account user")
|
||||
stmt = stmt.where(AgentConfigDraft.account_id == account_id)
|
||||
else:
|
||||
stmt = stmt.where(AgentConfigDraft.account_id.is_(None))
|
||||
draft = session.scalar(stmt.order_by(AgentConfigDraft.updated_at.desc()).limit(1))
|
||||
if draft is not None:
|
||||
return draft
|
||||
if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
|
||||
raise AgentAppGeneratorError("Agent build draft not found")
|
||||
_, snapshot, agent_soul = AgentAppGenerator._resolve_agent_by_id(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent.id,
|
||||
snapshot_id=agent.active_config_snapshot_id,
|
||||
session=session,
|
||||
)
|
||||
draft = AgentConfigDraft(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent.id,
|
||||
draft_type=AgentConfigDraftType.DRAFT,
|
||||
account_id=None,
|
||||
draft_owner_key="",
|
||||
base_snapshot_id=snapshot.id,
|
||||
config_snapshot=agent_soul,
|
||||
created_by=agent.created_by,
|
||||
updated_by=agent.updated_by,
|
||||
)
|
||||
session.add(draft)
|
||||
session.flush()
|
||||
return draft
|
||||
raise AgentAppGeneratorError("Agent build draft not found")
|
||||
|
||||
@staticmethod
|
||||
def _resolve_agent_by_id(
|
||||
|
||||
@@ -941,48 +941,64 @@ class AgentAppRunner:
|
||||
if pending_text:
|
||||
persist_answer_text(pending_text)
|
||||
|
||||
for public_event in self._agent_backend_client.stream_events(run_id):
|
||||
if queue_manager.is_stopped():
|
||||
flush_pending_agent_message_text()
|
||||
self._cancel_run(run_id)
|
||||
raise GenerateTaskStoppedError()
|
||||
for internal_event in self._event_adapter.adapt(public_event):
|
||||
try:
|
||||
public_events = self._agent_backend_client.stream_events(
|
||||
run_id,
|
||||
should_stop=queue_manager.is_stopped,
|
||||
)
|
||||
for public_event in public_events:
|
||||
if queue_manager.is_stopped():
|
||||
flush_pending_agent_message_text()
|
||||
self._cancel_run(run_id)
|
||||
raise GenerateTaskStoppedError()
|
||||
if internal_event.type in (
|
||||
AgentBackendInternalEventType.RUN_STARTED,
|
||||
AgentBackendInternalEventType.STREAM_EVENT,
|
||||
AgentBackendInternalEventType.AGENT_MESSAGE_DELTA,
|
||||
):
|
||||
if isinstance(internal_event, AgentBackendAgentMessageDeltaInternalEvent):
|
||||
debounced_delta = text_delta_debouncer.push(internal_event.delta)
|
||||
if debounced_delta:
|
||||
persist_answer_text(debounced_delta)
|
||||
continue
|
||||
|
||||
if isinstance(internal_event, AgentBackendStreamInternalEvent):
|
||||
for internal_event in self._event_adapter.adapt(public_event):
|
||||
if queue_manager.is_stopped():
|
||||
flush_pending_agent_message_text()
|
||||
try:
|
||||
process_recorder.handle_stream_event(internal_event)
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
logger.warning(
|
||||
"Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
|
||||
run_id,
|
||||
message_id,
|
||||
internal_event.event_kind,
|
||||
exc_info=True,
|
||||
)
|
||||
self._cancel_run(run_id)
|
||||
raise GenerateTaskStoppedError()
|
||||
if internal_event.type in (
|
||||
AgentBackendInternalEventType.RUN_STARTED,
|
||||
AgentBackendInternalEventType.STREAM_EVENT,
|
||||
AgentBackendInternalEventType.AGENT_MESSAGE_DELTA,
|
||||
):
|
||||
if isinstance(internal_event, AgentBackendAgentMessageDeltaInternalEvent):
|
||||
debounced_delta = text_delta_debouncer.push(internal_event.delta)
|
||||
if debounced_delta:
|
||||
persist_answer_text(debounced_delta)
|
||||
continue
|
||||
|
||||
if isinstance(internal_event, AgentBackendStreamInternalEvent):
|
||||
flush_pending_agent_message_text()
|
||||
try:
|
||||
process_recorder.handle_stream_event(internal_event)
|
||||
except Exception:
|
||||
db.session.rollback()
|
||||
logger.warning(
|
||||
"Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
|
||||
run_id,
|
||||
message_id,
|
||||
internal_event.event_kind,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
continue
|
||||
continue
|
||||
flush_pending_agent_message_text()
|
||||
terminal = internal_event
|
||||
break
|
||||
if terminal is not None:
|
||||
break
|
||||
flush_pending_agent_message_text()
|
||||
terminal = internal_event
|
||||
break
|
||||
if terminal is not None:
|
||||
break
|
||||
except GenerateTaskStoppedError:
|
||||
raise
|
||||
except Exception as error:
|
||||
flush_pending_agent_message_text()
|
||||
self._cancel_run(run_id)
|
||||
if queue_manager.is_stopped():
|
||||
raise GenerateTaskStoppedError() from error
|
||||
raise
|
||||
flush_pending_agent_message_text()
|
||||
if queue_manager.is_stopped():
|
||||
self._cancel_run(run_id)
|
||||
raise GenerateTaskStoppedError()
|
||||
return terminal, process_recorder
|
||||
|
||||
def _cancel_run(self, run_id: str) -> None:
|
||||
|
||||
@@ -32,6 +32,7 @@ class _DebuggerDraftVariableSaver:
|
||||
self,
|
||||
*,
|
||||
account: Account,
|
||||
tenant_id: str,
|
||||
app_id: str,
|
||||
node_id: str,
|
||||
node_type: NodeType,
|
||||
@@ -39,6 +40,7 @@ class _DebuggerDraftVariableSaver:
|
||||
enclosing_node_id: str | None = None,
|
||||
) -> None:
|
||||
self._account = account
|
||||
self._tenant_id = tenant_id
|
||||
self._app_id = app_id
|
||||
self._node_id = node_id
|
||||
self._node_type = node_type
|
||||
@@ -49,6 +51,7 @@ class _DebuggerDraftVariableSaver:
|
||||
with Session(db.engine) as session, session.begin():
|
||||
DraftVariableSaverImpl(
|
||||
session=session,
|
||||
tenant_id=self._tenant_id,
|
||||
app_id=self._app_id,
|
||||
node_id=self._node_id,
|
||||
node_type=self._node_type,
|
||||
@@ -287,7 +290,12 @@ class BaseAppGenerator:
|
||||
|
||||
@final
|
||||
@staticmethod
|
||||
def _get_draft_var_saver_factory(invoke_from: InvokeFrom, account: Account | EndUser) -> DraftVariableSaverFactory:
|
||||
def _get_draft_var_saver_factory(
|
||||
invoke_from: InvokeFrom,
|
||||
account: Account | EndUser,
|
||||
*,
|
||||
tenant_id: str,
|
||||
) -> DraftVariableSaverFactory:
|
||||
if invoke_from == InvokeFrom.DEBUGGER:
|
||||
assert isinstance(account, Account)
|
||||
|
||||
@@ -300,6 +308,7 @@ class BaseAppGenerator:
|
||||
) -> DraftVariableSaver:
|
||||
return _DebuggerDraftVariableSaver(
|
||||
account=account,
|
||||
tenant_id=tenant_id,
|
||||
app_id=app_id,
|
||||
node_id=node_id,
|
||||
node_type=node_type,
|
||||
|
||||
@@ -21,6 +21,7 @@ from core.app.entities.queue_entities import (
|
||||
WorkflowQueueMessage,
|
||||
)
|
||||
from extensions.ext_redis import redis_client
|
||||
from graphon.graph_engine.manager import GraphEngineManager
|
||||
from graphon.runtime import GraphRuntimeState
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -51,6 +52,9 @@ class AppQueueManager(ABC):
|
||||
self._graph_runtime_state: GraphRuntimeState | None = None
|
||||
self._stopped_cache: TTLCache[tuple, bool] = TTLCache(maxsize=1, ttl=1)
|
||||
self._cache_lock = threading.Lock()
|
||||
self._execution_terminal = threading.Event()
|
||||
self._abort_sent = threading.Event()
|
||||
self._lifecycle_lock = threading.Lock()
|
||||
|
||||
def listen(self):
|
||||
"""
|
||||
@@ -59,7 +63,7 @@ class AppQueueManager(ABC):
|
||||
"""
|
||||
# wait for APP_MAX_EXECUTION_TIME seconds to stop listen
|
||||
listen_timeout = dify_config.APP_MAX_EXECUTION_TIME
|
||||
start_time = time.time()
|
||||
start_time = time.monotonic()
|
||||
last_ping_time: int | float = 0
|
||||
try:
|
||||
while True:
|
||||
@@ -72,8 +76,14 @@ class AppQueueManager(ABC):
|
||||
except queue.Empty:
|
||||
continue
|
||||
finally:
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time >= listen_timeout or self._is_stopped():
|
||||
elapsed_time = time.monotonic() - start_time
|
||||
timed_out = elapsed_time >= listen_timeout
|
||||
manually_stopped = self._is_stopped()
|
||||
if not self._execution_terminal.is_set() and (timed_out or manually_stopped):
|
||||
reason = (
|
||||
f"App execution exceeded {listen_timeout} seconds" if timed_out else "App task was stopped"
|
||||
)
|
||||
self._abort_execution(reason)
|
||||
# publish two messages to make sure the client can receive the stop signal
|
||||
# and stop listening after the stop signal processed
|
||||
self.publish(
|
||||
@@ -84,16 +94,33 @@ class AppQueueManager(ABC):
|
||||
self.publish(QueuePingEvent(), PublishFrom.TASK_PIPELINE)
|
||||
last_ping_time = elapsed_time // 10
|
||||
finally:
|
||||
if not self._execution_terminal.is_set():
|
||||
self._abort_execution("Client response stream closed before app execution completed")
|
||||
self._graph_runtime_state = None # Release reference once consumers finish or close the generator.
|
||||
|
||||
def stop_listen(self):
|
||||
def stop_listen(self, *, execution_terminal: bool = False):
|
||||
"""
|
||||
Stop listen to queue
|
||||
:return:
|
||||
"""
|
||||
if execution_terminal:
|
||||
self._execution_terminal.set()
|
||||
self._clear_task_belong_cache()
|
||||
self._q.put(None)
|
||||
|
||||
def _abort_execution(self, reason: str) -> None:
|
||||
"""Propagate response timeout/disconnect to legacy and GraphEngine runners."""
|
||||
with self._lifecycle_lock:
|
||||
if self._execution_terminal.is_set() or self._abort_sent.is_set():
|
||||
return
|
||||
self._abort_sent.set()
|
||||
|
||||
try:
|
||||
self.set_stop_flag_no_user_check(self._task_id)
|
||||
GraphEngineManager(redis_client).send_stop_command(self._task_id, reason=reason)
|
||||
except Exception:
|
||||
logger.exception("Failed to abort app execution for task %s", self._task_id)
|
||||
|
||||
def _clear_task_belong_cache(self) -> None:
|
||||
"""
|
||||
Remove the task belong cache key once listening is finished.
|
||||
|
||||
@@ -45,7 +45,7 @@ class MessageBasedAppQueueManager(AppQueueManager):
|
||||
if isinstance(
|
||||
event, QueueStopEvent | QueueErrorEvent | QueueMessageEndEvent | QueueAdvancedChatMessageEndEvent
|
||||
):
|
||||
self.stop_listen()
|
||||
self.stop_listen(execution_terminal=True)
|
||||
|
||||
if pub_from == PublishFrom.APPLICATION_MANAGER and self._is_stopped():
|
||||
if self._app_mode == AppMode.ADVANCED_CHAT.value:
|
||||
|
||||
@@ -216,6 +216,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
session_factory = sessionmaker(bind=db.engine, expire_on_commit=False)
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=workflow_triggered_from,
|
||||
@@ -223,6 +224,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.RAG_PIPELINE_RUN,
|
||||
@@ -347,6 +349,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
draft_var_saver_factory = self._get_draft_var_saver_factory(
|
||||
invoke_from,
|
||||
user,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
)
|
||||
# return response or stream generator
|
||||
response = self._handle_response(
|
||||
@@ -425,6 +428,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_DEBUGGING,
|
||||
@@ -432,6 +436,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
@@ -524,6 +529,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.RAG_PIPELINE_DEBUGGING,
|
||||
@@ -531,6 +537,7 @@ class PipelineGenerator(BaseAppGenerator):
|
||||
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=pipeline.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
|
||||
@@ -42,7 +42,7 @@ class PipelineQueueManager(AppQueueManager):
|
||||
| QueueWorkflowFailedEvent
|
||||
| QueueWorkflowPartialSuccessEvent,
|
||||
):
|
||||
self.stop_listen()
|
||||
self.stop_listen(execution_terminal=True)
|
||||
|
||||
if pub_from == PublishFrom.APPLICATION_MANAGER and self._is_stopped():
|
||||
raise GenerateTaskStoppedError()
|
||||
|
||||
@@ -243,6 +243,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
workflow_triggered_from = WorkflowRunTriggeredFrom.APP_RUN
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=workflow_triggered_from,
|
||||
@@ -250,6 +251,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.WORKFLOW_RUN,
|
||||
@@ -397,7 +399,11 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
|
||||
worker_thread.start()
|
||||
|
||||
draft_var_saver_factory = self._get_draft_var_saver_factory(invoke_from, user)
|
||||
draft_var_saver_factory = self._get_draft_var_saver_factory(
|
||||
invoke_from,
|
||||
user,
|
||||
tenant_id=app_model.tenant_id,
|
||||
)
|
||||
|
||||
# return response or stream generator
|
||||
response = self._handle_response(
|
||||
@@ -470,6 +476,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
# Create workflow execution(aka workflow run) repository
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
|
||||
@@ -477,6 +484,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
@@ -560,6 +568,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
# Create workflow execution(aka workflow run) repository
|
||||
workflow_execution_repository = DifyCoreRepositoryFactory.create_workflow_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowRunTriggeredFrom.DEBUGGING,
|
||||
@@ -567,6 +576,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
|
||||
# Create workflow node execution repository
|
||||
workflow_node_execution_repository = DifyCoreRepositoryFactory.create_workflow_node_execution_repository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=app_model.tenant_id,
|
||||
user=user,
|
||||
app_id=application_generate_entity.app_config.app_id,
|
||||
triggered_from=WorkflowNodeExecutionTriggeredFrom.SINGLE_STEP,
|
||||
|
||||
@@ -41,4 +41,4 @@ class WorkflowAppQueueManager(AppQueueManager):
|
||||
| QueueWorkflowFailedEvent
|
||||
| QueueWorkflowPartialSuccessEvent,
|
||||
):
|
||||
self.stop_listen()
|
||||
self.stop_listen(execution_terminal=True)
|
||||
|
||||
@@ -26,6 +26,7 @@ from core.app.entities.queue_entities import (
|
||||
QueueNodeSucceededEvent,
|
||||
QueueReasoningChunkEvent,
|
||||
QueueRetrieverResourcesEvent,
|
||||
QueueStopEvent,
|
||||
QueueTextChunkEvent,
|
||||
QueueWorkflowFailedEvent,
|
||||
QueueWorkflowPartialSuccessEvent,
|
||||
@@ -424,7 +425,12 @@ class WorkflowBasedAppRunner:
|
||||
QueueWorkflowFailedEvent(error=event.error, exceptions_count=event.exceptions_count)
|
||||
)
|
||||
case GraphRunAbortedEvent():
|
||||
self._publish_event(QueueWorkflowFailedEvent(error=event.reason or "Unknown error", exceptions_count=0))
|
||||
self._publish_event(
|
||||
QueueStopEvent(
|
||||
stopped_by=QueueStopEvent.StopBy.USER_MANUAL,
|
||||
reason=event.reason or "Workflow execution aborted",
|
||||
)
|
||||
)
|
||||
case GraphRunPausedEvent():
|
||||
runtime_state = workflow_entry.graph_engine.graph_runtime_state
|
||||
paused_nodes = runtime_state.get_paused_nodes()
|
||||
|
||||
@@ -500,11 +500,15 @@ class QueueStopEvent(AppQueueEvent):
|
||||
|
||||
event: QueueEvent = QueueEvent.STOP
|
||||
stopped_by: StopBy
|
||||
reason: str | None = None
|
||||
|
||||
def get_stop_reason(self) -> str:
|
||||
"""
|
||||
To stop reason
|
||||
"""
|
||||
if self.reason:
|
||||
return self.reason
|
||||
|
||||
reason_mapping = {
|
||||
QueueStopEvent.StopBy.USER_MANUAL: "Stopped by user.",
|
||||
QueueStopEvent.StopBy.ANNOTATION_REPLY: "Stopped by annotation reply.",
|
||||
|
||||
@@ -47,6 +47,24 @@ class MaxRetriesExceededError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ResponseLimitError(ValueError):
|
||||
"""Base error for responses that cannot be safely bounded."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ResponseTooLargeError(ResponseLimitError):
|
||||
"""Raised when an identity response exceeds the configured byte limit."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class UnsupportedResponseEncodingError(ResponseLimitError):
|
||||
"""Raised when response encoding prevents safe decoded-size enforcement."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
request_error = httpx.RequestError
|
||||
max_retries_exceeded_error = MaxRetriesExceededError
|
||||
|
||||
@@ -142,7 +160,31 @@ def _inject_trace_headers(headers: Headers | None) -> Headers:
|
||||
return headers
|
||||
|
||||
|
||||
def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
|
||||
def make_request(
|
||||
method: str,
|
||||
url: str,
|
||||
max_retries: int = SSRF_DEFAULT_MAX_RETRIES,
|
||||
stream_response: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> httpx.Response:
|
||||
"""Send one SSRF-protected request with optional streaming.
|
||||
|
||||
Args:
|
||||
method: HTTP method sent through the configured SSRF client.
|
||||
url: Absolute request URL.
|
||||
max_retries: Number of retry attempts after the initial request.
|
||||
stream_response: Return an open streaming response that the caller must close.
|
||||
**kwargs: Additional keyword arguments forwarded to ``httpx.Client``.
|
||||
|
||||
Returns:
|
||||
A buffered response, or an open response when ``stream_response`` is true.
|
||||
|
||||
Raises:
|
||||
ToolSSRFError: The configured SSRF proxy rejects the destination.
|
||||
MaxRetriesExceededError: All configured request attempts fail.
|
||||
httpx.RequestError: A request fails while retries are disabled.
|
||||
ValueError: The SSL verification option or request headers are invalid.
|
||||
"""
|
||||
# Convert requests-style allow_redirects to httpx-style follow_redirects
|
||||
if "allow_redirects" in kwargs:
|
||||
allow_redirects = kwargs.pop("allow_redirects")
|
||||
@@ -175,6 +217,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
# When using a forward proxy, httpx may override the Host header based on the URL.
|
||||
# We extract and preserve any explicitly set Host header to support virtual hosting.
|
||||
user_provided_host = _get_user_provided_host_header(headers)
|
||||
send_kwargs: dict[str, Any] = {}
|
||||
if "auth" in kwargs:
|
||||
send_kwargs["auth"] = kwargs.pop("auth")
|
||||
if "follow_redirects" in kwargs:
|
||||
send_kwargs["follow_redirects"] = kwargs.pop("follow_redirects")
|
||||
|
||||
retries = 0
|
||||
while retries <= max_retries:
|
||||
@@ -185,7 +232,11 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
if user_provided_host is not None:
|
||||
headers["host"] = user_provided_host
|
||||
kwargs["headers"] = headers
|
||||
response = client.request(method=method, url=url, **kwargs)
|
||||
request = client.build_request(method=method, url=url, **kwargs)
|
||||
if stream_response:
|
||||
response = client.send(request, stream=True, **send_kwargs)
|
||||
else:
|
||||
response = client.send(request, **send_kwargs)
|
||||
|
||||
# Check for SSRF protection by Squid proxy
|
||||
if response.status_code in (401, 403):
|
||||
@@ -195,6 +246,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
|
||||
# Squid typically identifies itself in Server or Via headers
|
||||
if "squid" in server_header or "squid" in via_header:
|
||||
response.close()
|
||||
raise ToolSSRFError(
|
||||
f"Access to '{url}' was blocked by SSRF protection. "
|
||||
f"The URL may point to a private or local network address. "
|
||||
@@ -208,6 +260,7 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
response.status_code,
|
||||
url,
|
||||
)
|
||||
response.close()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
logger.warning("Request to URL %s failed on attempt %s: %s", url, retries + 1, e)
|
||||
@@ -220,6 +273,42 @@ def make_request(method: str, url: str, max_retries: int = SSRF_DEFAULT_MAX_RETR
|
||||
raise MaxRetriesExceededError(f"Reached maximum retries ({max_retries}) for URL {url}")
|
||||
|
||||
|
||||
def buffer_response(response: httpx.Response, *, max_response_bytes: int) -> httpx.Response:
|
||||
"""Consume one open identity response under a decoded byte limit and close its stream."""
|
||||
if max_response_bytes <= 0:
|
||||
raise ValueError("max_response_bytes must be positive")
|
||||
|
||||
try:
|
||||
content_encoding = response.headers.get("content-encoding", "identity").strip().lower()
|
||||
if content_encoding not in {"", "identity"}:
|
||||
raise UnsupportedResponseEncodingError(f"content encoding {content_encoding} cannot be safely bounded")
|
||||
content = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
if len(content) + len(chunk) > max_response_bytes:
|
||||
raise ResponseTooLargeError(f"response exceeded {max_response_bytes} bytes")
|
||||
content.extend(chunk)
|
||||
decoded_headers = {
|
||||
name: value
|
||||
for name, value in response.headers.items()
|
||||
if name.lower() not in {"content-encoding", "content-length", "transfer-encoding"}
|
||||
}
|
||||
try:
|
||||
request = response.request
|
||||
except RuntimeError:
|
||||
request = None
|
||||
return httpx.Response(
|
||||
response.status_code,
|
||||
headers=decoded_headers,
|
||||
content=bytes(content),
|
||||
request=request,
|
||||
extensions=response.extensions,
|
||||
history=response.history,
|
||||
default_encoding=response.default_encoding,
|
||||
)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def get(url: str, max_retries: int = SSRF_DEFAULT_MAX_RETRIES, **kwargs: Any) -> httpx.Response:
|
||||
return make_request("GET", url, max_retries=max_retries, **kwargs)
|
||||
|
||||
|
||||
@@ -403,55 +403,6 @@ class LLMGenerator:
|
||||
return ""
|
||||
return "\n\n".join(sections) + "\n\n"
|
||||
|
||||
@classmethod
|
||||
def classify_workflow_mode(
|
||||
cls,
|
||||
tenant_id: str,
|
||||
instruction: str,
|
||||
model_config: ModelConfig,
|
||||
) -> Literal["workflow", "advanced-chat"]:
|
||||
"""Classify a free-text instruction into a concrete app mode.
|
||||
|
||||
One tiny LLM call using the model the user already picked (so no extra
|
||||
provider setup is needed). Parsed leniently; defaults to
|
||||
``advanced-chat`` on anything unexpected or any error, so a
|
||||
``mode="auto"`` request never blocks generation. NEVER raises.
|
||||
"""
|
||||
default_mode: Literal["workflow", "advanced-chat"] = "advanced-chat"
|
||||
try:
|
||||
model_instance = ModelManager.for_tenant(tenant_id=tenant_id).get_model_instance(
|
||||
tenant_id=tenant_id,
|
||||
model_type=ModelType.LLM,
|
||||
provider=model_config.provider,
|
||||
model=model_config.name,
|
||||
)
|
||||
prompt_messages: list[PromptMessage] = [
|
||||
UserPromptMessage(
|
||||
content=(
|
||||
"Reply with exactly one word: 'workflow' (one-shot automation, no chat) "
|
||||
"or 'advanced-chat' (conversational multi-turn). "
|
||||
f"Instruction: {instruction.strip()}"
|
||||
)
|
||||
),
|
||||
]
|
||||
response: LLMResult = model_instance.invoke_llm(
|
||||
prompt_messages=prompt_messages,
|
||||
model_parameters={"max_tokens": 4, "temperature": 0},
|
||||
stream=False,
|
||||
)
|
||||
text = (response.message.get_text_content() or "").strip().lower()
|
||||
except Exception:
|
||||
logger.info("Workflow mode classification failed; defaulting to %s", default_mode, exc_info=True)
|
||||
return default_mode
|
||||
|
||||
# Lenient parse: an affirmative "workflow" wins; everything else
|
||||
# (including a truncated / empty / garbled reply) falls back to the
|
||||
# conversational default. "advanced-chat" needs no positive match
|
||||
# because it IS the default.
|
||||
if "workflow" in text:
|
||||
return "workflow"
|
||||
return default_mode
|
||||
|
||||
@classmethod
|
||||
def generate_rule_config(cls, tenant_id: str, args: RuleGeneratePayload):
|
||||
output_parser = RuleConfigGeneratorOutputParser()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import inspect
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Generator
|
||||
from collections.abc import Callable, Generator, Mapping
|
||||
from typing import Any, cast
|
||||
from urllib.parse import unquote
|
||||
|
||||
@@ -23,6 +23,7 @@ from core.plugin.impl.exc import (
|
||||
PluginLLMPollingUnsupportedError,
|
||||
PluginNotFoundError,
|
||||
PluginPermissionDeniedError,
|
||||
PluginRuntimeError,
|
||||
PluginUniqueIdentifierError,
|
||||
)
|
||||
from core.trigger.errors import (
|
||||
@@ -375,6 +376,18 @@ class BasePluginClient:
|
||||
# type `PluginLLMPollingUnsupportedError`.
|
||||
case PluginLLMPollingUnsupportedError.__name__:
|
||||
raise PluginLLMPollingUnsupportedError(description=error_object.get("message"))
|
||||
case PluginRuntimeError.__name__:
|
||||
args = error_object.get("args")
|
||||
lambda_request_id = args.get("request_id") if isinstance(args, Mapping) else None
|
||||
if not isinstance(lambda_request_id, str):
|
||||
lambda_request_id = None
|
||||
runtime_message = error_object.get("message")
|
||||
if not isinstance(runtime_message, str):
|
||||
runtime_message = "Plugin runtime request failed"
|
||||
raise PluginRuntimeError(
|
||||
description=runtime_message,
|
||||
lambda_request_id=lambda_request_id,
|
||||
)
|
||||
case _:
|
||||
raise PluginInvokeError(description=message)
|
||||
case PluginDaemonInternalServerError.__name__:
|
||||
|
||||
@@ -49,6 +49,18 @@ class PluginDaemonBadRequestError(PluginDaemonClientSideError):
|
||||
description: str = "Bad Request"
|
||||
|
||||
|
||||
class PluginRuntimeError(PluginDaemonInternalError):
|
||||
"""A plugin runtime failed before it could return a valid plugin response."""
|
||||
|
||||
lambda_request_id: str | None
|
||||
|
||||
def __init__(self, description: str, lambda_request_id: str | None = None) -> None:
|
||||
self.lambda_request_id = lambda_request_id
|
||||
if lambda_request_id:
|
||||
description = description.replace(f"RequestId: {lambda_request_id} Error: ", "", 1)
|
||||
super().__init__(description)
|
||||
|
||||
|
||||
class PluginInvokeError(PluginDaemonClientSideError, ValueError):
|
||||
description: str = "Invoke Error"
|
||||
|
||||
|
||||
@@ -131,7 +131,10 @@ class WaterCrawlAPIClient(BaseAPIClient):
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
media_type = content_type.split(";", 1)[0].strip().lower()
|
||||
if media_type == "application/json":
|
||||
return response.json() or {}
|
||||
try:
|
||||
return response.json() or {}
|
||||
except ValueError as exc:
|
||||
raise ValueError("Invalid JSON response from WaterCrawl") from exc
|
||||
|
||||
if media_type == "application/octet-stream":
|
||||
return response.content
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
"""WaterCrawl domain exceptions.
|
||||
|
||||
These exceptions are constructed from upstream HTTP responses, which may be
|
||||
JSON API errors or plain text/HTML proxy errors. Keep the exception type stable
|
||||
even when the body is not JSON so callers can handle WaterCrawl failures by
|
||||
domain type instead of low-level parser errors.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import override
|
||||
from typing import Any, override
|
||||
|
||||
from httpx import Response
|
||||
|
||||
|
||||
class WaterCrawlError(Exception):
|
||||
@@ -7,11 +17,16 @@ class WaterCrawlError(Exception):
|
||||
|
||||
|
||||
class WaterCrawlBadRequestError(WaterCrawlError):
|
||||
def __init__(self, response):
|
||||
def __init__(self, response: Response):
|
||||
self.status_code = response.status_code
|
||||
self.response = response
|
||||
data = response.json()
|
||||
self.message = data.get("message", "Unknown error occurred")
|
||||
try:
|
||||
data: Any = response.json()
|
||||
except ValueError:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
self.message = data.get("message") or response.text or "Unknown error occurred"
|
||||
self.errors = data.get("errors", {})
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ class RBACPermission(StrEnum):
|
||||
WORKSPACE_ROLE_MANAGE = "workspace_role_manage"
|
||||
API_EXTENSION_MANAGE = "api_extension_manage"
|
||||
CUSTOMIZATION_MANAGE = "customization_manage"
|
||||
AGENT_MANAGE = "agent_manage"
|
||||
|
||||
SNIPPETS_CREATE_AND_MODIFY = "snippets_create_and_modify"
|
||||
SNIPPETS_MANAGE = "snippets_management"
|
||||
|
||||
@@ -13,7 +13,6 @@ from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from core.repositories.factory import WorkflowExecutionRepository
|
||||
from graphon.entities import WorkflowExecution
|
||||
from libs.helper import extract_tenant_id
|
||||
from models import Account, CreatorUserRole, EndUser
|
||||
from models.enums import WorkflowRunTriggeredFrom
|
||||
from tasks.workflow_execution_tasks import (
|
||||
@@ -47,6 +46,7 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowRunTriggeredFrom | None,
|
||||
@@ -56,7 +56,8 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for fallback operations
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (DEBUGGING or APP_RUN)
|
||||
"""
|
||||
@@ -71,10 +72,8 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
f"Invalid session_factory type {type(session_factory).__name__}; expected sessionmaker or Engine"
|
||||
)
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
|
||||
@@ -17,7 +17,6 @@ from core.repositories.factory import (
|
||||
WorkflowNodeExecutionRepository,
|
||||
)
|
||||
from graphon.entities import WorkflowNodeExecution
|
||||
from libs.helper import extract_tenant_id
|
||||
from models import Account, CreatorUserRole, EndUser
|
||||
from models.workflow import WorkflowNodeExecutionTriggeredFrom
|
||||
from tasks.workflow_node_execution_tasks import (
|
||||
@@ -54,6 +53,7 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowNodeExecutionTriggeredFrom | None,
|
||||
@@ -63,7 +63,8 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for fallback operations
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow node execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (SINGLE_STEP or WORKFLOW_RUN)
|
||||
"""
|
||||
@@ -78,10 +79,8 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
f"Invalid session_factory type {type(session_factory).__name__}; expected sessionmaker or Engine"
|
||||
)
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
|
||||
@@ -62,6 +62,7 @@ class DifyCoreRepositoryFactory:
|
||||
def create_workflow_execution_repository(
|
||||
cls,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str,
|
||||
triggered_from: WorkflowRunTriggeredFrom,
|
||||
@@ -71,7 +72,8 @@ class DifyCoreRepositoryFactory:
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine
|
||||
user: Account or EndUser object
|
||||
tenant_id: Tenant that owns the workflow execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: Application ID
|
||||
triggered_from: Source of the execution trigger
|
||||
|
||||
@@ -87,6 +89,7 @@ class DifyCoreRepositoryFactory:
|
||||
repository_class = import_string(class_path)
|
||||
return repository_class(
|
||||
session_factory=session_factory,
|
||||
tenant_id=tenant_id,
|
||||
user=user,
|
||||
app_id=app_id,
|
||||
triggered_from=triggered_from,
|
||||
@@ -98,6 +101,7 @@ class DifyCoreRepositoryFactory:
|
||||
def create_workflow_node_execution_repository(
|
||||
cls,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str,
|
||||
triggered_from: WorkflowNodeExecutionTriggeredFrom,
|
||||
@@ -107,7 +111,8 @@ class DifyCoreRepositoryFactory:
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine
|
||||
user: Account or EndUser object
|
||||
tenant_id: Tenant that owns the workflow node execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: Application ID
|
||||
triggered_from: Source of the execution trigger
|
||||
|
||||
@@ -123,6 +128,7 @@ class DifyCoreRepositoryFactory:
|
||||
repository_class = import_string(class_path)
|
||||
return repository_class(
|
||||
session_factory=session_factory,
|
||||
tenant_id=tenant_id,
|
||||
user=user,
|
||||
app_id=app_id,
|
||||
triggered_from=triggered_from,
|
||||
|
||||
@@ -13,7 +13,6 @@ from core.repositories.factory import WorkflowExecutionRepository
|
||||
from graphon.entities import WorkflowExecution
|
||||
from graphon.enums import WorkflowExecutionStatus, WorkflowType
|
||||
from graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter
|
||||
from libs.helper import extract_tenant_id
|
||||
from models import (
|
||||
Account,
|
||||
CreatorUserRole,
|
||||
@@ -40,6 +39,7 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowRunTriggeredFrom | None,
|
||||
@@ -49,7 +49,8 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for creating sessions
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (DEBUGGING or APP_RUN)
|
||||
"""
|
||||
@@ -64,10 +65,8 @@ class SQLAlchemyWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
f"Invalid session_factory type {type(session_factory).__name__}; expected sessionmaker or Engine"
|
||||
)
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
|
||||
@@ -23,7 +23,6 @@ from graphon.entities import WorkflowNodeExecution
|
||||
from graphon.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
|
||||
from graphon.model_runtime.utils.encoders import jsonable_encoder
|
||||
from graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter
|
||||
from libs.helper import extract_tenant_id
|
||||
from libs.uuid_utils import uuidv7
|
||||
from models import (
|
||||
Account,
|
||||
@@ -63,6 +62,7 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowNodeExecutionTriggeredFrom | None,
|
||||
@@ -72,7 +72,8 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for creating sessions
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow node execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (SINGLE_STEP or WORKFLOW_RUN)
|
||||
"""
|
||||
@@ -87,10 +88,8 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)
|
||||
f"Invalid session_factory type {type(session_factory).__name__}; expected sessionmaker or Engine"
|
||||
)
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
@@ -299,6 +298,7 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)
|
||||
content=value_json.encode("utf-8"),
|
||||
mimetype="application/json",
|
||||
user=self._user,
|
||||
tenant_id=self._tenant_id,
|
||||
)
|
||||
offload = WorkflowNodeExecutionOffload(
|
||||
id=uuidv7(),
|
||||
|
||||
@@ -58,7 +58,6 @@ class Tool(ABC):
|
||||
if self.runtime and self.runtime.runtime_parameters:
|
||||
tool_parameters.update(self.runtime.runtime_parameters)
|
||||
|
||||
# try parse tool parameters into the correct type
|
||||
tool_parameters = self._transform_tool_parameters_type(tool_parameters)
|
||||
|
||||
result = self._invoke(
|
||||
@@ -87,14 +86,14 @@ class Tool(ABC):
|
||||
return result
|
||||
|
||||
def _transform_tool_parameters_type(self, tool_parameters: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Transform tool parameters type
|
||||
"""
|
||||
# Temp fix for the issue that the tool parameters will be converted to empty while validating the credentials
|
||||
"""Transform declared tool parameter values without resolving runtime schemas."""
|
||||
result = deepcopy(tool_parameters)
|
||||
for parameter in self.entity.parameters or []:
|
||||
if parameter.name in tool_parameters:
|
||||
result[parameter.name] = parameter.type.cast_value(tool_parameters[parameter.name])
|
||||
if parameter.multiple:
|
||||
result[parameter.name] = parameter.init_frontend_parameter(result.get(parameter.name))
|
||||
else:
|
||||
result[parameter.name] = parameter.type.cast_value(tool_parameters[parameter.name])
|
||||
|
||||
return result
|
||||
|
||||
@@ -196,17 +195,31 @@ class Tool(ABC):
|
||||
}:
|
||||
continue
|
||||
|
||||
parameter_schema: dict[str, Any] = (
|
||||
{
|
||||
"type": parameter.type.as_normal_type(),
|
||||
"description": parameter.llm_description or "",
|
||||
}
|
||||
if parameter.input_schema is None
|
||||
else deepcopy(parameter.input_schema)
|
||||
)
|
||||
is_multiple_select = parameter.multiple and parameter.type in {
|
||||
ToolParameter.ToolParameterType.SELECT,
|
||||
ToolParameter.ToolParameterType.DYNAMIC_SELECT,
|
||||
}
|
||||
if is_multiple_select:
|
||||
item_schema: dict[str, Any] = {"type": "string"}
|
||||
if parameter.type == ToolParameter.ToolParameterType.SELECT and parameter.options:
|
||||
item_schema["enum"] = [option.value for option in parameter.options]
|
||||
parameter_schema: dict[str, Any] = {"type": "array", "items": item_schema}
|
||||
else:
|
||||
parameter_schema = (
|
||||
{
|
||||
"type": parameter.type.as_normal_type(),
|
||||
"description": parameter.llm_description or "",
|
||||
}
|
||||
if parameter.input_schema is None
|
||||
else deepcopy(parameter.input_schema)
|
||||
)
|
||||
parameter_schema.setdefault("description", parameter.llm_description or "")
|
||||
|
||||
if parameter.type == ToolParameter.ToolParameterType.SELECT and parameter.options:
|
||||
if (
|
||||
not is_multiple_select
|
||||
and parameter.type == ToolParameter.ToolParameterType.SELECT
|
||||
and parameter.options
|
||||
):
|
||||
parameter_schema["enum"] = [option.value for option in parameter.options]
|
||||
|
||||
schema["properties"][parameter.name] = parameter_schema
|
||||
|
||||
@@ -292,9 +292,7 @@ class ToolInvokeMessageBinary(BaseModel):
|
||||
|
||||
|
||||
class ToolParameter(PluginParameter):
|
||||
"""
|
||||
Overrides type
|
||||
"""
|
||||
"""Tool-specific parameter declaration and invocation-value normalization."""
|
||||
|
||||
class ToolParameterType(StrEnum):
|
||||
"""
|
||||
@@ -333,12 +331,28 @@ class ToolParameter(PluginParameter):
|
||||
LLM = auto() # will be set by LLM
|
||||
|
||||
type: ToolParameterType = Field(..., description="The type of the parameter")
|
||||
multiple: bool = Field(
|
||||
default=False,
|
||||
description="Whether the parameter is multiple select, only valid for select or dynamic-select type",
|
||||
)
|
||||
human_description: I18nObject | None = Field(default=None, description="The description presented to the user")
|
||||
form: ToolParameterForm = Field(..., description="The form of the parameter, schema/form/llm")
|
||||
llm_description: str | None = None
|
||||
# MCP object and array type parameters use this field to store the schema
|
||||
input_schema: dict[str, Any] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_multiple(self) -> ToolParameter:
|
||||
supports_multiple = self.type in {
|
||||
self.ToolParameterType.SELECT,
|
||||
self.ToolParameterType.DYNAMIC_SELECT,
|
||||
}
|
||||
if self.multiple and not supports_multiple:
|
||||
raise ValueError("multiple is only valid for select and dynamic-select parameters")
|
||||
if supports_multiple and self.default is not None and (isinstance(self.default, list) != self.multiple):
|
||||
raise ValueError("default must be a list exactly when multiple is true")
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def get_simple_instance(
|
||||
cls,
|
||||
@@ -378,8 +392,25 @@ class ToolParameter(PluginParameter):
|
||||
options=option_objs,
|
||||
)
|
||||
|
||||
def init_frontend_parameter(self, value: Any):
|
||||
return init_frontend_parameter(self, self.type, value)
|
||||
def init_frontend_parameter(self, value: Any) -> Any:
|
||||
"""Normalize a value against this tool parameter's full declaration."""
|
||||
if not self.multiple:
|
||||
return init_frontend_parameter(self, self.type, value)
|
||||
|
||||
parameter_value = self.default if value is None else value
|
||||
if parameter_value is None:
|
||||
parameter_value = []
|
||||
if not isinstance(parameter_value, list):
|
||||
raise ValueError(f"tool parameter {self.name} must be a list when multiple is true")
|
||||
if not all(isinstance(item, str) for item in parameter_value):
|
||||
raise ValueError(f"tool parameter {self.name} must contain only strings")
|
||||
if self.required and not parameter_value:
|
||||
raise ValueError(f"tool parameter {self.name} not found in tool config")
|
||||
if self.type == self.ToolParameterType.SELECT:
|
||||
options = [option.value for option in self.options]
|
||||
if any(item not in options for item in parameter_value):
|
||||
raise ValueError(f"tool parameter {self.name} value {parameter_value} not in options {options}")
|
||||
return parameter_value
|
||||
|
||||
|
||||
class ToolProviderIdentity(BaseModel):
|
||||
|
||||
@@ -4,12 +4,12 @@ Workflow generator package.
|
||||
Generates a Dify workflow graph (nodes, edges, viewport) from a natural-language
|
||||
instruction. Intended for the cmd+k `/create` slash command's preview/apply flow.
|
||||
|
||||
Pipeline (slim, single-shot variant):
|
||||
Pipeline:
|
||||
|
||||
runner.WorkflowGenerator.generate_workflow_graph(...)
|
||||
├── planner_prompts: short LLM call → high-level node plan
|
||||
└── builder_prompts: structured-output LLM call → full graph JSON
|
||||
└── postprocess: fill defaults, auto-layout viewport, sanity-check edges
|
||||
├── planner_prompts: short LLM call → node and edge plan
|
||||
├── node_builder_prompts: bounded parallel calls → semantic node configs
|
||||
└── postprocess: assemble wrappers, auto-layout, validate graph
|
||||
|
||||
The runner is pure domain logic; ``WorkflowGeneratorService`` (in ``services/``)
|
||||
owns the model-manager dependency and is what controllers call.
|
||||
|
||||
@@ -1,19 +1,4 @@
|
||||
"""
|
||||
Builder prompts.
|
||||
|
||||
The builder is the second step of the slim planner→builder pipeline. It takes
|
||||
the planner's high-level node list and emits the *full* graph JSON consumed by
|
||||
``WorkflowService.sync_draft_workflow``.
|
||||
|
||||
The builder owns: node configuration (prompts, code, headers, etc.), edge wiring,
|
||||
handle ids ("source"/"target"), positions, and the viewport. It is the only
|
||||
prompt that needs to know the concrete shape of each node type — keep its
|
||||
examples accurate or the LLM will invent fields.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
"""Compact semantic configuration references for workflow node builders."""
|
||||
|
||||
# Per-node-type configuration cheatsheet.
|
||||
#
|
||||
@@ -23,50 +8,9 @@ from typing import Any
|
||||
# both ``WorkflowService.sync_draft_workflow``'s structural checks and the
|
||||
# runtime entity validation each node performs when the workflow runs.
|
||||
#
|
||||
# The cheatsheet is assembled DYNAMICALLY per request: the planner decides
|
||||
# which node types the workflow needs, and ``build_node_config_cheatsheet``
|
||||
# stitches together only the snippets for those types (plus the always-needed
|
||||
# wrapper / shared-field / edge-handle preamble, and the containers section
|
||||
# when an iteration / loop is planned). This keeps the builder prompt tight —
|
||||
# a 3-node summariser no longer carries the schema for 12 unrelated node
|
||||
# types — and lets each snippet document its FULL schema (e.g. a "file" start
|
||||
# variable's required ``allowed_file_types``) without bloating every prompt.
|
||||
#
|
||||
# The postprocessor in ``runner.py`` fills missing wrapper fields (``type``,
|
||||
# ``positionAbsolute``, ``width``, ``height``, ``sourcePosition`` /
|
||||
# ``targetPosition``, edge ``data.sourceType`` / ``data.targetType``), so the
|
||||
# LLM only needs to emit semantically meaningful fields.
|
||||
|
||||
# Always-included preamble: the node/edge wrapper shape and the shared
|
||||
# ``data`` fields that apply to every node type, plus the "## Per type" header
|
||||
# the per-type snippets slot under.
|
||||
_CHEATSHEET_PREAMBLE = """\
|
||||
## Node wrapper (every node, top-level)
|
||||
|
||||
{"id": "node1" (digits + letters only — see "Node IDs" below),
|
||||
"type": "custom", # ReactFlow renderer key. Iteration/loop
|
||||
# *start* children use special types
|
||||
# (see Containers below).
|
||||
"position": {"x": <number>, "y": <number>},
|
||||
"data": { ... per-type fields ... }}
|
||||
|
||||
Children of iteration / loop containers additionally need
|
||||
``parentId``, ``zIndex: 1002`` and ``extent: "parent"`` — see Containers.
|
||||
|
||||
## Shared "data" fields (every node)
|
||||
|
||||
{"type": "<node-type>", # e.g. "llm", "start", "if-else"
|
||||
"title": "<short label>",
|
||||
"desc": "<one-liner>",
|
||||
"selected": false}
|
||||
|
||||
## Per type — additional "data" fields (only the node types in your plan are shown)"""
|
||||
|
||||
|
||||
# node_type → its per-type schema snippet. Keyed by the exact ``node_type``
|
||||
# string the planner emits so ``build_node_config_cheatsheet`` can look each
|
||||
# one up directly. Iteration / loop are documented in the Containers section
|
||||
# (they are subgraphs, not leaf nodes) rather than here.
|
||||
# Each snippet mirrors the production node default closely enough for one
|
||||
# model call to emit only meaningful ``data`` fields. The runner owns wrappers,
|
||||
# topology, container metadata, layout, and validation.
|
||||
_NODE_SNIPPETS: dict[str, str] = {
|
||||
"start": """\
|
||||
- start:
|
||||
@@ -248,484 +192,30 @@ _NODE_SNIPPETS: dict[str, str] = {
|
||||
Enable only the sub-features you need; ``conditions`` reuse the if-else
|
||||
condition shape (key / comparison_operator / value). Outputs: ``result``
|
||||
(the processed array), ``first_record``, ``last_record``.""",
|
||||
"assigner": """\
|
||||
- assigner (write to an existing conversation / loop variable):
|
||||
{"version": "2",
|
||||
"items": [{"variable_selector": ["<target-node>", "<target-var>"],
|
||||
"input_type": "variable",
|
||||
"operation": "over-write",
|
||||
"value": ["<source-node>", "<source-var>"]}]}
|
||||
``input_type`` is "variable" (value is a selector) or "constant".
|
||||
Operations: over-write | clear | append | extend | set | += | -= | *= |
|
||||
/= | remove-first | remove-last.""",
|
||||
"human-input": """\
|
||||
- human-input (pause for a person; use webapp delivery by default):
|
||||
{"delivery_methods": [{"id": "webapp", "type": "webapp", "enabled": true}],
|
||||
"form_content": "<short review / approval instructions>",
|
||||
"inputs": [{"type": "paragraph", "output_variable_name": "comment",
|
||||
"default": {"type": "constant", "selector": [], "value": ""}}],
|
||||
"user_actions": [{"id": "approve", "title": "Approve",
|
||||
"button_style": "primary"}],
|
||||
"timeout": 3, "timeout_unit": "day"}
|
||||
Each ``inputs[].output_variable_name`` is an output variable. Outgoing
|
||||
edges use the matching user-action id as ``sourceHandle``.""",
|
||||
}
|
||||
|
||||
|
||||
# Pulled into the cheatsheet only when an iteration / loop appears in the plan.
|
||||
_CONTAINERS_SECTION = """\
|
||||
## Containers — iteration / loop
|
||||
|
||||
These are SUBGRAPH nodes. To use one you MUST emit, in order:
|
||||
|
||||
1. The container node itself, e.g. for iteration:
|
||||
id: "nodeK"
|
||||
type: "custom"
|
||||
data: {"type": "iteration",
|
||||
"title": "<label>",
|
||||
"desc": "",
|
||||
"selected": false,
|
||||
"start_node_id": "nodeKstart",
|
||||
"iterator_selector": ["<src>", "<list-var>"],
|
||||
"output_selector": ["<inner-last-node>", "<out-var>"],
|
||||
"is_parallel": false,
|
||||
"parallel_nums": 10,
|
||||
"error_handle_mode": "terminated",
|
||||
"flatten_output": true}
|
||||
width: 808
|
||||
height: 204
|
||||
zIndex: 1
|
||||
|
||||
For loop, swap "iteration" → "loop" and use:
|
||||
data: {"type": "loop", "title": "...", "desc": "",
|
||||
"selected": false, "start_node_id": "nodeKstart",
|
||||
"break_conditions": [], "loop_count": 10,
|
||||
"logical_operator": "and"}
|
||||
|
||||
2. The auto-start child (one per container):
|
||||
id: "nodeKstart"
|
||||
type: "custom-iteration-start" # loop → "custom-loop-start"
|
||||
parentId: "nodeK"
|
||||
extent: "parent"
|
||||
draggable: false
|
||||
selectable: false
|
||||
zIndex: 1002
|
||||
position: {"x": 60, "y": 78} # relative to parent
|
||||
data: {"type": "iteration-start", # loop → "loop-start"
|
||||
"title": "", "desc": "",
|
||||
"isInIteration": true, # loop → "isInLoop": true
|
||||
"selected": false}
|
||||
|
||||
3. Each inner-pipeline node (any node type, follows normal data rules) MUST add:
|
||||
parentId: "nodeK"
|
||||
extent: "parent"
|
||||
zIndex: 1002
|
||||
position: {x, y} # relative to parent
|
||||
data: {..., "isInIteration": true, # loop → "isInLoop": true
|
||||
"iteration_id": "nodeK"} # loop → "loop_id"
|
||||
|
||||
4. Edges INSIDE a container must add to ``data``:
|
||||
"isInIteration": true # loop → "isInLoop": true
|
||||
"iteration_id": "nodeK" # loop → "loop_id"
|
||||
and use ``zIndex: 1002``. Edges OUTSIDE containers use the default
|
||||
``isInIteration: false`` / ``isInLoop: false``.
|
||||
|
||||
5. The container's incoming/outgoing edges connect to the container's id
|
||||
(``nodeK``), NOT to inner nodes. The first inner edge connects from
|
||||
``nodeKstart``."""
|
||||
|
||||
|
||||
# Always-included trailer: edge handle conventions for every graph.
|
||||
_EDGE_HANDLES_SECTION = """\
|
||||
## Edge handles
|
||||
|
||||
- Most nodes: sourceHandle "source", targetHandle "target".
|
||||
- if-else cases: sourceHandle is the case_id ("true" / "false" / ...).
|
||||
- question-classifier: sourceHandle is the class_id ("1" / "2" / ...).
|
||||
- iteration-start / sourceHandle "source"; the edge from the *start node
|
||||
loop-start: is what kicks off the first inner step."""
|
||||
|
||||
|
||||
# Container node types are described in ``_CONTAINERS_SECTION`` rather than as
|
||||
# leaf snippets; their presence in a plan pulls that section in.
|
||||
_CONTAINER_NODE_TYPES = frozenset({"iteration", "loop"})
|
||||
|
||||
|
||||
def build_node_config_cheatsheet(node_types: Iterable[str] | None = None) -> str:
|
||||
"""
|
||||
Assemble the builder cheatsheet for exactly the node types in the plan.
|
||||
|
||||
``node_types`` is the set of ``node_type`` strings the planner chose. We
|
||||
emit the always-on preamble (wrapper / shared fields), then only the
|
||||
per-type snippets for the requested types (``start`` is always included —
|
||||
every graph has one), the Containers section when an iteration / loop is
|
||||
planned, and the edge-handles trailer. Unknown / unrecognised type strings
|
||||
are ignored (the runtime / structural validator catches genuinely bogus
|
||||
types).
|
||||
|
||||
``None`` returns the FULL cheatsheet (every snippet + containers) — used to
|
||||
build the static back-compat constants below and as a safe fallback.
|
||||
"""
|
||||
if node_types is None:
|
||||
requested: set[str] = set(_NODE_SNIPPETS) | set(_CONTAINER_NODE_TYPES)
|
||||
else:
|
||||
requested = {str(t).strip() for t in node_types if str(t).strip()}
|
||||
requested.add("start") # every workflow has exactly one start node
|
||||
|
||||
parts: list[str] = [_CHEATSHEET_PREAMBLE]
|
||||
# Iterate _NODE_SNIPPETS (not ``requested``) to keep a stable, readable order.
|
||||
parts.extend(snippet for node_type, snippet in _NODE_SNIPPETS.items() if node_type in requested)
|
||||
if requested & _CONTAINER_NODE_TYPES:
|
||||
parts.append(_CONTAINERS_SECTION)
|
||||
parts.append(_EDGE_HANDLES_SECTION)
|
||||
return "\n\n".join(parts) + "\n"
|
||||
|
||||
|
||||
# Full cheatsheet (all node types) — retained as a module constant so callers
|
||||
# and tests that want the complete reference can import it directly. The
|
||||
# dynamic per-request prompt is built by ``get_builder_system_prompt``.
|
||||
NODE_CONFIG_CHEATSHEET = build_node_config_cheatsheet()
|
||||
|
||||
|
||||
_BASE_SYSTEM_PROMPT_HEAD = """You are a Dify workflow builder.
|
||||
|
||||
You are given:
|
||||
1. A user instruction (what the workflow should do).
|
||||
2. A node plan from the planner (which nodes to use, in execution order).
|
||||
|
||||
Your job: emit a complete Dify workflow graph as JSON. The graph will be written
|
||||
directly into a Studio draft, so it must be syntactically valid and structurally
|
||||
correct.
|
||||
|
||||
# Hard rules
|
||||
|
||||
1. The output is a single JSON object — no prose, no Markdown, no code fences.
|
||||
2. NODE IDs MUST USE ONLY ALPHANUMERICS + UNDERSCORES — never hyphens.
|
||||
Dify's run-time placeholder regex (see ``variable_pool.VARIABLE_PATTERN``)
|
||||
is ``\\{\\{#([a-zA-Z0-9_]{1,50}(?:\\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\\}\\}``,
|
||||
so any placeholder pointing at a hyphenated id (e.g. ``{{#node-1.text#}}``)
|
||||
silently fails to match at run time and the literal string survives into
|
||||
the prompt — the user then sees ``{{#node-1.text#}}`` in their output.
|
||||
Use the EXACT ids from the plan, formatted as ``node1``, ``node2``, ... in
|
||||
plan order. Edge ``source`` / ``target`` must reference these ids.
|
||||
3. Every node has top-level fields: id, type, position, data.
|
||||
- "type" is always "custom" (ReactFlow node renderer).
|
||||
- "data.type" is the actual node type ("llm", "start", etc.).
|
||||
4. Every edge has top-level fields: id, source, target, type, sourceHandle, targetHandle.
|
||||
- "type" is always "custom".
|
||||
- "sourceHandle"/"targetHandle" follow the cheatsheet (default: "source"/"target").
|
||||
- Edge id format: "<source>-<sourceHandle>-<target>-<targetHandle>".
|
||||
5. Use the model from the planner context for ALL "llm" / "question-classifier" /
|
||||
"parameter-extractor" nodes (provider, name, mode, completion_params).
|
||||
6. Reference upstream outputs with the literal placeholder syntax
|
||||
``{{#<node-id>.<output-var>#}}`` — that's DOUBLE curly braces with ``#``
|
||||
markers inside (matching Dify's runtime placeholder regex
|
||||
``\\{\\{#[^#]+#\\}\\}``). NEVER emit single-brace ``{#…#}`` — Dify will
|
||||
not interpolate it, so the LLM at run time would see the literal
|
||||
placeholder string in its prompt and echo it back as output. Use
|
||||
``["<node-id>", "<output-var>"]`` for ``value_selector`` /
|
||||
``query_variable_selector`` / etc.
|
||||
7. The "start" node owns input variables; downstream nodes reference them as
|
||||
``["<start-node-id>", "<var-name>"]`` for selectors or
|
||||
``{{#<start-node-id>.<var-name>#}}`` inside prompt strings.
|
||||
8. NEVER emit "code" or "http-request" nodes if a tool from the "Available tools"
|
||||
section below covers the same task — replace them with a "tool" node referencing
|
||||
the exact provider/tool identifier from the catalogue. "code" / "http-request"
|
||||
are last-resort escape hatches for arbitrary transformations and APIs that no
|
||||
installed tool can express.
|
||||
9. EVERY variable reference MUST resolve to a real, declared variable on the
|
||||
source node — never invent a variable name. Specifically:
|
||||
- ``{{#<node-id>.<var>#}}`` inside a prompt / ``answer`` / ``template-transform``
|
||||
template (DOUBLE braces — single ``{#…#}`` is NOT a Dify placeholder
|
||||
and will NOT be substituted), AND ``["<node-id>", "<var>"]`` inside a
|
||||
``value_selector`` /
|
||||
``query_variable_selector`` / ``iterator_selector`` / ``output_selector`` /
|
||||
``tool_parameters[*].value`` (when ``type: "variable"``), MUST point at a
|
||||
value that the source node actually exposes:
|
||||
* ``start`` → one of the ``data.variables[*].variable`` entries you
|
||||
declared on the start node. Add an entry if you need a new input.
|
||||
* ``llm`` → ``text`` (the default LLM output) or, when structured
|
||||
output is enabled, a key from its schema.
|
||||
* ``code`` → a key in ``data.outputs``.
|
||||
* ``knowledge-retrieval`` → ``result`` (the standard array output).
|
||||
* ``parameter-extractor`` → one of the ``data.parameters[*].name``.
|
||||
* ``document-extractor`` → ``text`` (extracted file text; an array of
|
||||
strings when ``is_array_file`` is true).
|
||||
* ``variable-aggregator`` → ``output``.
|
||||
* ``list-operator`` → ``result`` (array), ``first_record``,
|
||||
``last_record``.
|
||||
* ``tool`` → any parameter declared by the tool — the run time
|
||||
validates these, so you can name them freely, but pick from the
|
||||
documented provider/tool.
|
||||
If the planner's "Start inputs" list (see user prompt) is non-empty,
|
||||
copy each entry verbatim into ``start.data.variables`` so the
|
||||
downstream references resolve.
|
||||
- In Advanced-Chat mode you may also reference ``sys.query`` and
|
||||
``sys.files`` without declaring them. In selector fields, spell these as
|
||||
exactly ``["sys", "query"]`` and ``["sys", "files"]`` — never as a
|
||||
one-item array such as ``["sys,query"]`` or ``["sys.query"]``.
|
||||
10. MULTIPLE KNOWLEDGE-RETRIEVAL INPUTS TO ONE LLM require a template fan-in.
|
||||
``context.variable_selector accepts only one selector`` and therefore
|
||||
cannot carry two retrieval outputs. When an LLM must synthesize two or
|
||||
more retrieval results:
|
||||
- Run the retrieval nodes as parallel siblings from the same query input.
|
||||
- Add one ``template-transform`` node after them. Give it one variable per
|
||||
retrieval, such as ``value_selector: ["node2", "result"]`` and
|
||||
``value_selector: ["node3", "result"]``, and render every source's
|
||||
content into one labelled text output.
|
||||
- Add an edge from EACH retrieval node to the template, then one edge from
|
||||
the template to the LLM. The LLM must NOT receive direct retrieval edges.
|
||||
- Enable the LLM's context, set ``context.variable_selector`` to the
|
||||
template's ``["<template-node-id>", "output"]``, and put
|
||||
``{{#context#}}`` in its prompt.
|
||||
- Do not use ``variable-aggregator`` for this: it selects the first value
|
||||
produced by mutually exclusive branches; it does not concatenate two
|
||||
retrieval results that both ran.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
_BASE_SYSTEM_PROMPT_TAIL = """\
|
||||
|
||||
# Layout
|
||||
|
||||
- Place nodes left-to-right with x=80 + 320 * index, y=280.
|
||||
- Viewport: {"x": 0, "y": 0, "zoom": 0.7}.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
_BASE_SYSTEM_PROMPT_FOOTER = """
|
||||
|
||||
# Output schema
|
||||
|
||||
{
|
||||
"nodes": [...],
|
||||
"edges": [...],
|
||||
"viewport": {"x": 0, "y": 0, "zoom": 0.7}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
_WORKFLOW_MODE_RULES = """# Mode-specific rules — Workflow
|
||||
|
||||
- The graph MUST start with exactly one "start" node and end with exactly one "end" node.
|
||||
- Do NOT use "answer" nodes (those are for Advanced Chat only).
|
||||
- The "end" node's outputs[].value_selector must point at a real upstream output.
|
||||
"""
|
||||
|
||||
|
||||
_ADVANCED_CHAT_MODE_RULES = """# Mode-specific rules — Advanced Chat (Chatflow)
|
||||
|
||||
- The graph MUST start with exactly one "start" node and end with exactly one "answer" node.
|
||||
- Do NOT use "end" nodes (those are for plain Workflow apps).
|
||||
- The "start" node should expose "sys.query" / "sys.files" automatically; user-defined
|
||||
variables go in start.data.variables.
|
||||
- The "answer" node's "answer" field references upstream outputs as
|
||||
{{#<node-id>.<var>#}} and is what the user sees in chat.
|
||||
"""
|
||||
|
||||
|
||||
def _assemble_builder_system_prompt(mode: str, node_types: Iterable[str] | None) -> str:
|
||||
"""Stitch the builder system prompt for ``mode`` around a cheatsheet built
|
||||
for ``node_types`` (``None`` → full cheatsheet)."""
|
||||
mode_rules = _ADVANCED_CHAT_MODE_RULES if mode == "advanced-chat" else _WORKFLOW_MODE_RULES
|
||||
return (
|
||||
_BASE_SYSTEM_PROMPT_HEAD
|
||||
+ mode_rules
|
||||
+ _BASE_SYSTEM_PROMPT_TAIL
|
||||
+ build_node_config_cheatsheet(node_types)
|
||||
+ _BASE_SYSTEM_PROMPT_FOOTER
|
||||
)
|
||||
|
||||
|
||||
# Static full-cheatsheet prompts — the back-compat default returned by
|
||||
# ``get_builder_system_prompt`` when the caller doesn't pin a node-type set.
|
||||
BUILDER_SYSTEM_PROMPT_WORKFLOW = _assemble_builder_system_prompt("workflow", None)
|
||||
|
||||
BUILDER_SYSTEM_PROMPT_ADVANCED_CHAT = _assemble_builder_system_prompt("advanced-chat", None)
|
||||
|
||||
|
||||
BUILDER_USER_PROMPT = """# User instruction
|
||||
|
||||
{instruction}
|
||||
|
||||
{ideal_output_section}\
|
||||
{existing_graph_section}\
|
||||
# Selected model (use for all LLM-based nodes)
|
||||
|
||||
provider={provider}, name={name}, mode={mode_label}
|
||||
|
||||
{tool_catalogue_section}\
|
||||
{start_inputs_section}\
|
||||
# Node plan (from planner — use these labels and node_types in this order)
|
||||
|
||||
{plan_block}
|
||||
|
||||
Now emit the complete workflow graph JSON.
|
||||
"""
|
||||
|
||||
|
||||
# Node wrapper fields that carry no meaning the builder needs: pure canvas /
|
||||
# selection state, plus geometry the runner's postprocess recomputes anyway.
|
||||
# Stripping them out of the refine prompt cuts its size roughly in half on
|
||||
# hand-edited graphs — fewer tokens in, and (because the builder echoes
|
||||
# untouched nodes verbatim) far fewer tokens out, which is where the latency
|
||||
# lives.
|
||||
_PRUNED_NODE_KEYS = frozenset(
|
||||
{
|
||||
"positionAbsolute",
|
||||
"sourcePosition",
|
||||
"targetPosition",
|
||||
"selected",
|
||||
"dragging",
|
||||
"measured",
|
||||
}
|
||||
)
|
||||
|
||||
# Additionally pruned from TOP-LEVEL nodes only: the layered auto-layout
|
||||
# recomputes their position and size defaults, so the builder never needs to
|
||||
# reproduce them. Container children keep ``position`` (relative to the
|
||||
# parent, which we cannot recompute) and containers keep ``width`` /
|
||||
# ``height`` (their canvas size is real config, not a default).
|
||||
_PRUNED_TOP_LEVEL_NODE_KEYS = _PRUNED_NODE_KEYS | {"position", "width", "height"}
|
||||
|
||||
_CONTAINER_DATA_TYPES = frozenset({"iteration", "loop"})
|
||||
|
||||
# Edge fields the builder must echo; everything else (ids, zIndex,
|
||||
# sourceType / targetType, isInIteration / isInLoop markers) is recomputed
|
||||
# by the runner's postprocess from the node topology.
|
||||
_KEPT_EDGE_KEYS = ("source", "target", "sourceHandle", "targetHandle")
|
||||
|
||||
|
||||
def compact_graph_for_builder(current_graph: dict) -> dict:
|
||||
"""
|
||||
Strip canvas noise out of a draft graph before prompt injection.
|
||||
|
||||
Keeps everything semantically meaningful — ids, wrapper ``type``,
|
||||
``parentId``, the full ``data`` config, child positions, container
|
||||
sizes — and drops geometry / selection state the postprocess pass
|
||||
recomputes. The builder echoes untouched nodes verbatim, so every byte
|
||||
removed here is removed twice (prompt AND completion).
|
||||
"""
|
||||
nodes_out: list[dict] = []
|
||||
for node in current_graph.get("nodes") or []:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
is_child = bool(node.get("parentId"))
|
||||
is_container = isinstance(node.get("data"), dict) and node["data"].get("type") in _CONTAINER_DATA_TYPES
|
||||
pruned = _PRUNED_NODE_KEYS if (is_child or is_container) else _PRUNED_TOP_LEVEL_NODE_KEYS
|
||||
compact = {k: v for k, v in node.items() if k not in pruned}
|
||||
if is_container:
|
||||
# Container position is still recomputed by the layout pass.
|
||||
compact.pop("position", None)
|
||||
nodes_out.append(compact)
|
||||
edges_out = [
|
||||
{k: edge[k] for k in _KEPT_EDGE_KEYS if k in edge}
|
||||
for edge in (current_graph.get("edges") or [])
|
||||
if isinstance(edge, dict)
|
||||
]
|
||||
return {"nodes": nodes_out, "edges": edges_out}
|
||||
|
||||
|
||||
def format_builder_existing_graph_section(current_graph: dict | None) -> str:
|
||||
"""
|
||||
Refine mode: give the builder the existing graph JSON so it can keep
|
||||
every node and edge the user's change does not touch byte-for-byte — same
|
||||
ids, same config, same prompt templates. Without the full config the
|
||||
builder would regenerate untouched nodes from scratch and silently drop
|
||||
the user's hand-tuned settings. Canvas-only fields are stripped first
|
||||
(see ``compact_graph_for_builder``) — they're recomputed in postprocess,
|
||||
so carrying them only slows the call down.
|
||||
|
||||
Returns an empty string in create mode (no ``current_graph``); the builder
|
||||
then behaves exactly as before, constructing the graph purely from the
|
||||
planner's node plan.
|
||||
"""
|
||||
if not current_graph:
|
||||
return ""
|
||||
graph_json = json.dumps(compact_graph_for_builder(current_graph), ensure_ascii=False, separators=(",", ":"))
|
||||
return (
|
||||
"# Existing graph to refine (JSON)\n\n"
|
||||
"You are REFINING this existing graph, NOT building from scratch. Apply "
|
||||
"ONLY the change the user instruction describes. Every node and edge the "
|
||||
"change does not affect MUST be preserved verbatim — keep the same node "
|
||||
"ids, the same `data` config, and the same prompt templates. The node "
|
||||
"plan below is the target node set after your change; use the existing "
|
||||
"graph as the source of truth for the config of nodes that carry over.\n\n"
|
||||
f"```json\n{graph_json}\n```\n\n"
|
||||
)
|
||||
|
||||
|
||||
def format_start_inputs_section(start_inputs: list[dict[str, Any]]) -> str:
|
||||
"""
|
||||
Surface the planner's ``start_inputs`` list to the builder so it can
|
||||
populate ``start.data.variables`` with the exact set of inputs every
|
||||
downstream variable reference will need. Empty list → empty section,
|
||||
because the builder may then declare no input variables (e.g. an
|
||||
Advanced-Chat workflow that only consumes ``sys.query``).
|
||||
"""
|
||||
if not start_inputs:
|
||||
return ""
|
||||
lines = ["# Start inputs (copy each entry verbatim into start.data.variables)"]
|
||||
lines.append("")
|
||||
for inp in start_inputs:
|
||||
variable = str(inp.get("variable") or "").strip()
|
||||
label = str(inp.get("label") or "").strip()
|
||||
type_ = str(inp.get("type") or "paragraph").strip()
|
||||
if not variable:
|
||||
continue
|
||||
lines.append(f"- variable={variable!r} label={label!r} type={type_!r}")
|
||||
lines.append("")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def format_builder_tool_catalogue_section(catalogue_text: str) -> str:
|
||||
"""
|
||||
Builder-facing catalogue block. The builder needs the same identifiers
|
||||
the planner saw, plus a stern reminder that ``tool`` nodes MUST set
|
||||
``provider_id`` / ``provider_name`` / ``tool_name`` to entries that
|
||||
actually exist in this list — hallucinated tools fail at draft sync.
|
||||
"""
|
||||
if not catalogue_text.strip():
|
||||
return ""
|
||||
return (
|
||||
"# Available tools (use these exact provider/tool identifiers — "
|
||||
"for each 'tool' node, set provider_id and provider_name to the "
|
||||
"provider portion and tool_name to the tool portion)\n\n"
|
||||
f"{catalogue_text}\n\n"
|
||||
)
|
||||
|
||||
|
||||
def format_plan_block(plan_nodes: list[dict[str, Any]]) -> str:
|
||||
"""
|
||||
Render the planner output as a numbered list the builder can quote.
|
||||
|
||||
Node IDs use no separator (``node1``, ``node2``, ...) because Dify's
|
||||
run-time placeholder regex requires ``[a-zA-Z0-9_]`` in the node-id
|
||||
slot — a hyphenated id like ``node-1`` would silently fail to match
|
||||
at run time and the literal ``{{#node-1.var#}}`` survives into the
|
||||
LLM prompt.
|
||||
|
||||
For container children (planner emitted a ``"parent": "<label>"`` key),
|
||||
we resolve the parent label to its ``nodeN`` id and surface it on the
|
||||
same line so the builder knows to set ``parentId`` and the
|
||||
``isInIteration`` / ``isInLoop`` markers on inner nodes.
|
||||
"""
|
||||
# First pass: label → node-id so we can resolve "parent" hints.
|
||||
label_to_id: dict[str, str] = {}
|
||||
for idx, node in enumerate(plan_nodes, start=1):
|
||||
label = str(node.get("label") or "")
|
||||
if label and label not in label_to_id:
|
||||
label_to_id[label] = f"node{idx}"
|
||||
|
||||
lines = []
|
||||
for idx, node in enumerate(plan_nodes, start=1):
|
||||
node_id = f"node{idx}"
|
||||
label = node.get("label", "")
|
||||
node_type = node.get("node_type", "")
|
||||
purpose = node.get("purpose", "")
|
||||
parent_label = str(node.get("parent") or "")
|
||||
parent_clause = ""
|
||||
if parent_label:
|
||||
parent_id = label_to_id.get(parent_label, "")
|
||||
if parent_id:
|
||||
parent_clause = f" parent={parent_id}"
|
||||
else:
|
||||
parent_clause = f" parent={parent_label!r}"
|
||||
lines.append(f"{idx}. id={node_id} type={node_type} label={label!r}{parent_clause}\n purpose: {purpose}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def get_builder_system_prompt(mode: str, node_types: Iterable[str] | None = None) -> str:
|
||||
"""
|
||||
Build the builder system prompt for ``mode``, with a cheatsheet scoped to
|
||||
``node_types`` (the planner's chosen node types).
|
||||
|
||||
When ``node_types`` is ``None`` we return the cached full-cheatsheet
|
||||
constant (back-compat default). When the runner passes the plan's node-type
|
||||
set we assemble a fresh prompt carrying only the relevant per-type schemas,
|
||||
so the builder isn't handed config for node types the workflow never uses.
|
||||
"""
|
||||
if node_types is None:
|
||||
return BUILDER_SYSTEM_PROMPT_ADVANCED_CHAT if mode == "advanced-chat" else BUILDER_SYSTEM_PROMPT_WORKFLOW
|
||||
return _assemble_builder_system_prompt(mode, node_types)
|
||||
def get_node_config_snippet(node_type: str) -> str:
|
||||
"""Return the semantic config reference for one leaf node type."""
|
||||
return _NODE_SNIPPETS.get(node_type, "")
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Compact prompts for parallel, per-node workflow configuration.
|
||||
|
||||
Each call produces only the semantic ``data`` fields for one planned node.
|
||||
Canvas wrappers, shared labels, topology, layout, and edge defaults are owned
|
||||
by ``WorkflowGenerator`` so completion length scales with node configuration
|
||||
rather than with the full ReactFlow graph.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from core.workflow.generator.prompts.builder_prompts import get_node_config_snippet
|
||||
|
||||
_CONTAINER_CONFIG_SNIPPETS = {
|
||||
"iteration": """- iteration:
|
||||
{"iterator_selector": ["<src>", "<list-var>"],
|
||||
"output_selector": ["<last-child>", "<out-var>"],
|
||||
"is_parallel": false, "parallel_nums": 10,
|
||||
"error_handle_mode": "terminated", "flatten_output": true}
|
||||
The runner supplies start_node_id, child wrappers, and the synthetic start node.""",
|
||||
"loop": """- loop:
|
||||
{"break_conditions": [{"id": "c1",
|
||||
"variable_selector": ["<child>", "<var>"],
|
||||
"comparison_operator": "is",
|
||||
"value": "<value>"}],
|
||||
"loop_count": 10, "logical_operator": "and"}
|
||||
The runner supplies start_node_id, child wrappers, and the synthetic start node.""",
|
||||
}
|
||||
|
||||
_NODE_BUILDER_HEAD = """You configure exactly ONE node in a Dify workflow.
|
||||
|
||||
Return one JSON object with exactly this shape: {"config": {...}}.
|
||||
``config`` contains only node-type-specific ``data`` fields. Do NOT repeat id,
|
||||
type, title, desc, selected, position, wrapper fields, edges, or viewport.
|
||||
|
||||
Rules:
|
||||
- Use only ids from the supplied normalized plan.
|
||||
- Placeholder strings use ``{{#node_id.variable#}}``; selector fields use
|
||||
``["node_id", "variable"]``. Never invent an upstream output.
|
||||
- Use the selected model verbatim for llm, question-classifier, and
|
||||
parameter-extractor nodes.
|
||||
- Keep prompts/code concise but complete for the user's requested behavior.
|
||||
- Emit strict JSON only: no prose, Markdown, comments, or trailing commas.
|
||||
|
||||
# Target node schema
|
||||
|
||||
"""
|
||||
|
||||
|
||||
NODE_BUILDER_USER_PROMPT = """# Target node
|
||||
|
||||
id={node_id}, type={node_type}, label={label!r}
|
||||
purpose={purpose}
|
||||
|
||||
# User instruction
|
||||
|
||||
{instruction}
|
||||
|
||||
{ideal_output_section}{mode_section}{model_section}{tool_catalogue_section}{start_inputs_section}{existing_config_section}\
|
||||
# Normalized plan and topology
|
||||
|
||||
{plan_json}
|
||||
|
||||
Return {{"config": {{...}}}} for target node {node_id} now.
|
||||
"""
|
||||
|
||||
|
||||
def get_node_builder_system_prompt(node_type: str) -> str:
|
||||
"""Build a one-node prompt containing only that node's semantic schema."""
|
||||
snippet = _CONTAINER_CONFIG_SNIPPETS.get(node_type) or get_node_config_snippet(node_type)
|
||||
return _NODE_BUILDER_HEAD + (snippet or f"- {node_type}: emit the minimum valid config fields.")
|
||||
|
||||
|
||||
def format_parallel_plan(
|
||||
plan_nodes: list[dict[str, Any]],
|
||||
plan_edges: list[dict[str, Any]],
|
||||
start_inputs: list[dict[str, Any]] | None = None,
|
||||
) -> str:
|
||||
"""Serialize the shared plan compactly so every node call has graph context.
|
||||
|
||||
``start_inputs`` rides along so downstream builders reference the declared
|
||||
``{{#<start-id>.<variable>#}}`` names instead of guessing them from prose
|
||||
— a guessed name gets auto-injected as a spurious form input later.
|
||||
"""
|
||||
payload: dict[str, Any] = {"nodes": plan_nodes, "edges": plan_edges}
|
||||
if start_inputs:
|
||||
payload["start_inputs"] = start_inputs
|
||||
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def format_mode_section(mode: str) -> str:
|
||||
"""Tell each builder which app mode it is configuring for.
|
||||
|
||||
Matters most in advanced-chat, where ``sys.query`` / ``sys.files`` are the
|
||||
sanctioned way to reference the user's message — without this the model
|
||||
invents start-node variables that postprocess then materializes as
|
||||
spurious form inputs.
|
||||
"""
|
||||
if mode == "advanced-chat":
|
||||
return (
|
||||
"# App mode\n\n"
|
||||
"advanced-chat: the user's chat message is available as sys.query and uploaded files "
|
||||
'as sys.files — placeholder {{#sys.query#}}, selector ["sys", "query"]. Reference them '
|
||||
"directly; do NOT invent start-node variables for the chat message.\n\n"
|
||||
)
|
||||
return (
|
||||
"# App mode\n\n"
|
||||
"workflow: there are NO automatic system variables; reference user input only through "
|
||||
"the start node's declared variables.\n\n"
|
||||
)
|
||||
|
||||
|
||||
def format_start_inputs_section(start_inputs: list[dict[str, Any]]) -> str:
|
||||
"""Render planner-declared inputs for the start-node builder only."""
|
||||
if not start_inputs:
|
||||
return ""
|
||||
lines = ["# Start inputs (copy each entry verbatim into start.data.variables)", ""]
|
||||
for input_ in start_inputs:
|
||||
variable = str(input_.get("variable") or "").strip()
|
||||
if not variable:
|
||||
continue
|
||||
label = str(input_.get("label") or "").strip()
|
||||
type_ = str(input_.get("type") or "paragraph").strip()
|
||||
lines.append(f"- variable={variable!r} label={label!r} type={type_!r}")
|
||||
lines.append("")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def format_tool_catalogue_section(catalogue_text: str) -> str:
|
||||
"""Render exact tool identifiers for a tool-node builder only."""
|
||||
if not catalogue_text.strip():
|
||||
return ""
|
||||
return (
|
||||
"# Available tools (use these exact provider/tool identifiers — "
|
||||
"set provider_id and provider_name to the provider portion and "
|
||||
"tool_name to the tool portion)\n\n"
|
||||
f"{catalogue_text}\n\n"
|
||||
)
|
||||
@@ -1,13 +1,14 @@
|
||||
"""
|
||||
Planner prompts.
|
||||
|
||||
The planner is the lightweight first step in the slim planner→builder pipeline.
|
||||
The planner is the lightweight first step in the slim planner→node-builders pipeline.
|
||||
It receives the user's natural-language instruction and emits a high-level
|
||||
node plan in JSON. The builder later turns that plan into the final graph.
|
||||
node and edge plan in JSON. Node builders later produce configs that the runner
|
||||
assembles into the final graph.
|
||||
|
||||
We keep the planner deliberately short — the heavy lifting (config schemas,
|
||||
edge wiring, default values) belongs in the builder. The planner only commits
|
||||
to the *which-node-types* decision so the builder gets a tight scaffold.
|
||||
default values) belongs in the builders. The planner commits to the minimum
|
||||
topology and node types so every builder gets a tight scaffold.
|
||||
"""
|
||||
|
||||
PLANNER_SYSTEM_PROMPT = """You are a Dify workflow planner.
|
||||
@@ -39,6 +40,8 @@ minimum set of Dify workflow nodes needed to fulfil it, in execution order.
|
||||
mutually-exclusive paths before "end" / "answer".
|
||||
- "list-operator" — filter / sort / slice an array variable (e.g. the items
|
||||
fed into or produced by an "iteration").
|
||||
- "assigner" — update an existing conversation or loop variable.
|
||||
- "human-input" — pause for a person to review, approve, or enter data.
|
||||
|
||||
# Rules
|
||||
|
||||
@@ -97,22 +100,45 @@ minimum set of Dify workflow nodes needed to fulfil it, in execution order.
|
||||
variables are automatic — downstream nodes may reference them without
|
||||
a ``start_inputs`` entry. In Workflow mode there is NO automatic
|
||||
variable; everything the user supplies must be in ``start_inputs``.
|
||||
11. Output strictly the JSON object — no prose, no Markdown, no code fences.
|
||||
11. Give every node a unique runtime-safe ``id`` using only letters, digits,
|
||||
and underscores. In create mode use ``node1``, ``node2``, ... in node-list
|
||||
order. In refine mode preserve the existing id for every retained node.
|
||||
12. Emit the target graph's edges in ``edges``. Each edge is
|
||||
``{"source": "<id>", "target": "<id>"}``; add ``source_handle`` only
|
||||
for branch nodes: if-else case id, question-classifier class id, or
|
||||
human-input action id. Container children reference the container id in
|
||||
their ``parent`` field; do not emit the synthetic iteration/loop start node.
|
||||
13. In refine mode add ``action`` to every retained target node:
|
||||
``"keep"`` when its data config is unchanged, ``"update"`` when the user
|
||||
asked to change its config, and ``"add"`` for a new node. Removed nodes are
|
||||
omitted. Edge-only rewiring does not require changing a node's action.
|
||||
14. Output strictly the JSON object — no prose, no Markdown, no code fences.
|
||||
15. Echo the app mode in the ``mode`` output field — exactly "workflow" or
|
||||
"advanced-chat". When the ``# Mode`` section says auto, YOU decide:
|
||||
"workflow" for one-shot automations (run once with form inputs, return a
|
||||
result), "advanced-chat" for conversational multi-turn assistants. The
|
||||
terminal node must match the chosen mode (rule 2): "end" for workflow,
|
||||
"answer" for advanced-chat.
|
||||
|
||||
# Output schema
|
||||
|
||||
{
|
||||
"title": "<≤ 40-char title of the workflow>",
|
||||
"description": "<one-sentence summary>",
|
||||
"mode": "workflow | advanced-chat",
|
||||
"app_name": "<≤ 30-char product-style name, e.g. 'URL Summarizer'>",
|
||||
"icon": "<single emoji that captures the workflow's purpose, e.g. '📰'>",
|
||||
"start_inputs": [
|
||||
{"variable": "url", "label": "URL", "type": "text-input"}
|
||||
],
|
||||
"nodes": [
|
||||
{"label": "Start", "node_type": "start", "purpose": "..."},
|
||||
{"label": "Summarize", "node_type": "llm", "purpose": "..."},
|
||||
{"label": "End", "node_type": "end", "purpose": "..."}
|
||||
{"id": "node1", "label": "Start", "node_type": "start", "purpose": "..."},
|
||||
{"id": "node2", "label": "Summarize", "node_type": "llm", "purpose": "..."},
|
||||
{"id": "node3", "label": "End", "node_type": "end", "purpose": "..."}
|
||||
],
|
||||
"edges": [
|
||||
{"source": "node1", "target": "node2"},
|
||||
{"source": "node2", "target": "node3"}
|
||||
]
|
||||
}
|
||||
"""
|
||||
@@ -140,7 +166,8 @@ def format_existing_graph_section(current_graph: dict | None) -> str:
|
||||
|
||||
We pass only ids / node-types / titles + edge endpoints here — the planner
|
||||
decides *which nodes* exist, so it needs the shape, not the per-node config.
|
||||
The builder gets the full graph JSON to preserve untouched node config.
|
||||
Node builders receive only the config of a node marked ``update``;
|
||||
configs marked ``keep`` are reused directly.
|
||||
"""
|
||||
if not current_graph:
|
||||
return ""
|
||||
@@ -156,7 +183,13 @@ def format_existing_graph_section(current_graph: dict | None) -> str:
|
||||
for edge in edges:
|
||||
if not isinstance(edge, dict):
|
||||
continue
|
||||
edge_lines.append(f"- {edge.get('source', '')} -> {edge.get('target', '')}")
|
||||
# Branch wiring (if-else case ids, classifier class ids, human-input
|
||||
# action ids) lives in ``sourceHandle``. The planner is the only
|
||||
# source of edges for the rebuilt graph, so the real handle must be
|
||||
# surfaced here or refine silently rewires branches.
|
||||
handle = str(edge.get("sourceHandle") or "")
|
||||
handle_suffix = f" (source_handle={handle!r})" if handle and handle != "source" else ""
|
||||
edge_lines.append(f"- {edge.get('source', '')} -> {edge.get('target', '')}{handle_suffix}")
|
||||
nodes_block = "\n".join(node_lines) or "(none)"
|
||||
edges_block = "\n".join(edge_lines) or "(none)"
|
||||
return (
|
||||
@@ -166,7 +199,9 @@ def format_existing_graph_section(current_graph: dict | None) -> str:
|
||||
"node list to reflect that change while keeping everything the "
|
||||
"instruction does not mention — preserve existing nodes, their order, "
|
||||
"and their labels wherever the change leaves them untouched. Only add, "
|
||||
"remove, or rename nodes the requested change actually requires.\n\n"
|
||||
"remove, or rename nodes the requested change actually requires. "
|
||||
"For every retained edge, copy its source_handle verbatim from the "
|
||||
"list below — branch wiring must survive the refine unchanged.\n\n"
|
||||
f"Current nodes:\n{nodes_block}\n\n"
|
||||
f"Current edges:\n{edges_block}\n\n"
|
||||
)
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
"""
|
||||
Workflow generator runner.
|
||||
|
||||
Slim planner→builder pipeline. Pure domain logic; the model instance is
|
||||
Slim planner→parallel-node-builder pipeline. Pure domain logic; the model instance is
|
||||
injected by ``WorkflowGeneratorService`` so this module stays cleanly
|
||||
separated from the infrastructure layer.
|
||||
|
||||
Pipeline:
|
||||
|
||||
1. PLANNER — short LLM call producing a high-level node list.
|
||||
2. BUILDER — structured-output LLM call producing the full graph JSON.
|
||||
2. BUILDERS — bounded concurrent LLM calls producing compact node configs.
|
||||
3. POSTPROC — fill safe defaults, lay nodes out left-to-right, dedupe
|
||||
edge ids, and run a final structural sanity check.
|
||||
|
||||
@@ -20,7 +20,8 @@ Intentionally NOT here (deferred to a future iteration):
|
||||
- Tool / model catalogue filtering
|
||||
|
||||
If quality regresses below product threshold we add those back; for now the
|
||||
single planner+builder pair shipped behind cmd+k `/create` is enough.
|
||||
planner and bounded parallel node builders shipped behind cmd+k `/create` are
|
||||
enough.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -28,17 +29,22 @@ import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from copy import deepcopy
|
||||
from typing import Any, ClassVar, cast
|
||||
|
||||
import json_repair
|
||||
|
||||
from core.workflow.generator.prompts.builder_prompts import (
|
||||
BUILDER_USER_PROMPT,
|
||||
format_builder_existing_graph_section,
|
||||
format_builder_tool_catalogue_section,
|
||||
format_plan_block,
|
||||
from configs import dify_config
|
||||
from core.workflow.generator.prompts.node_builder_prompts import (
|
||||
NODE_BUILDER_USER_PROMPT,
|
||||
format_mode_section,
|
||||
format_parallel_plan,
|
||||
format_start_inputs_section,
|
||||
get_builder_system_prompt,
|
||||
get_node_builder_system_prompt,
|
||||
)
|
||||
from core.workflow.generator.prompts.node_builder_prompts import (
|
||||
format_tool_catalogue_section as format_node_tool_catalogue_section,
|
||||
)
|
||||
from core.workflow.generator.prompts.planner_prompts import (
|
||||
PLANNER_SYSTEM_PROMPT,
|
||||
@@ -55,6 +61,7 @@ from core.workflow.generator.types import (
|
||||
WorkflowGenerateErrorDict,
|
||||
WorkflowGenerateResultDict,
|
||||
WorkflowGenerationMode,
|
||||
WorkflowGenerationModeRequest,
|
||||
)
|
||||
from graphon.enums import BuiltinNodeTypes
|
||||
from graphon.model_runtime.entities.llm_entities import LLMResult
|
||||
@@ -96,11 +103,31 @@ _DEFAULT_FILE_UPLOAD_METHODS = ("local_file", "remote_url")
|
||||
|
||||
# Token ceiling for the planner call when the caller didn't pin one. The plan
|
||||
# is a short JSON node list (a handful of nodes with labels/purposes), so this
|
||||
# is generous headroom while still bounding a runaway response. The builder is
|
||||
# left on the caller's budget — it emits the full graph and genuinely needs it.
|
||||
# is generous headroom while still bounding a runaway response. Builder calls
|
||||
# keep the caller's budget so complex node configs are not truncated.
|
||||
_PLANNER_DEFAULT_MAX_TOKENS = 4096
|
||||
|
||||
|
||||
# Per-node calls trade a larger request count for a shorter critical path.
|
||||
# The cap comes from ``WORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS`` (default 6,
|
||||
# enough to run the planner's recommended 3–6-node plans as a single wave);
|
||||
# provider rate-limit bursts are absorbed by ``_invoke_with_retry``'s bounded
|
||||
# backoff, and operators can dial the env var back down if their provider is
|
||||
# stricter. Read at call time so tests (and live config reloads) can adjust it
|
||||
# without re-importing.
|
||||
def _node_builder_max_workers() -> int:
|
||||
return dify_config.WORKFLOW_GENERATOR_NODE_BUILDER_MAX_WORKERS
|
||||
|
||||
|
||||
_MODEL_NODE_TYPES = frozenset(
|
||||
{
|
||||
BuiltinNodeTypes.LLM,
|
||||
BuiltinNodeTypes.QUESTION_CLASSIFIER,
|
||||
BuiltinNodeTypes.PARAMETER_EXTRACTOR,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# Appended as a trailing user message on the SECOND (and only) attempt when
|
||||
# the first response wasn't parseable as JSON. Keep this terse — the model
|
||||
# already has its full instructions in the original system message; this is
|
||||
@@ -110,6 +137,12 @@ _JSON_RETRY_HINT = (
|
||||
"Do not include any prose, markdown code fences, comments, or trailing commas."
|
||||
)
|
||||
|
||||
_PLANNER_SCHEMA_RETRY_HINT = (
|
||||
"Your plan did not match the required topology schema. Return the complete plan again with "
|
||||
"a unique non-empty id on every node and a non-empty edges array whose source and target "
|
||||
"reference those ids. Return ONLY the JSON object."
|
||||
)
|
||||
|
||||
|
||||
# Provider hiccups we retry: a dropped connection, a 5xx, or a rate-limit are
|
||||
# all transient — the same request usually succeeds moments later. We do NOT
|
||||
@@ -189,14 +222,58 @@ def _result_with_errors(
|
||||
def _with_mode(result: WorkflowGenerateResultDict, mode: WorkflowGenerationMode) -> WorkflowGenerateResultDict:
|
||||
"""Stamp the resolved concrete ``mode`` onto a result envelope.
|
||||
|
||||
``mode="auto"`` requests are resolved to a concrete mode before planning;
|
||||
echoing it back lets the frontend pick the right app type to create. It's
|
||||
present for explicit modes too so the response shape stays uniform.
|
||||
``mode="auto"`` requests are resolved to a concrete mode from the planner
|
||||
output; echoing it back lets the frontend pick the right app type to
|
||||
create. It's present for explicit modes too so the response shape stays
|
||||
uniform.
|
||||
"""
|
||||
result["mode"] = mode
|
||||
return result
|
||||
|
||||
|
||||
def _fallback_mode(mode: WorkflowGenerationModeRequest) -> WorkflowGenerationMode:
|
||||
"""Concrete mode for envelopes emitted before the planner resolved one.
|
||||
|
||||
``auto`` maps to the conversational default — the same never-fail fallback
|
||||
the old standalone classifier used — so ``result.mode`` never leaks the
|
||||
``auto`` sentinel to the frontend.
|
||||
"""
|
||||
return "advanced-chat" if mode == "auto" else mode
|
||||
|
||||
|
||||
def _planner_prompt_mode(mode: WorkflowGenerationModeRequest) -> str:
|
||||
"""Mode string interpolated into the planner user prompt.
|
||||
|
||||
For ``auto`` the value is self-describing so the planner knows the choice
|
||||
is delegated to it (system-prompt rule 15).
|
||||
"""
|
||||
return "auto (choose workflow or advanced-chat)" if mode == "auto" else mode
|
||||
|
||||
|
||||
def _resolve_generation_mode(
|
||||
requested: WorkflowGenerationModeRequest, plan: PlannerResultDict
|
||||
) -> WorkflowGenerationMode:
|
||||
"""Resolve the request mode into the concrete generation mode.
|
||||
|
||||
An explicit request always wins — a contradictory planner ``mode`` field is
|
||||
ignored. For ``auto``: trust the planner's echoed ``mode``, else infer from
|
||||
the plan's terminal node type (the structural source of truth the graph is
|
||||
validated against), else fall back to the conversational default. Lenient
|
||||
on purpose — a bad ``mode`` value must never fail the plan.
|
||||
"""
|
||||
if requested != "auto":
|
||||
return requested
|
||||
planner_mode = str(plan.get("mode") or "").strip().lower()
|
||||
if planner_mode in ("workflow", "advanced-chat"):
|
||||
return cast(WorkflowGenerationMode, planner_mode)
|
||||
node_types = {str(node.get("node_type") or "") for node in plan.get("nodes") or [] if isinstance(node, dict)}
|
||||
if BuiltinNodeTypes.ANSWER in node_types:
|
||||
return "advanced-chat"
|
||||
if BuiltinNodeTypes.END in node_types:
|
||||
return "workflow"
|
||||
return "advanced-chat"
|
||||
|
||||
|
||||
def _build_plan_event(
|
||||
*,
|
||||
plan: PlannerResultDict,
|
||||
@@ -255,7 +332,7 @@ class WorkflowGenerator:
|
||||
provider: str,
|
||||
model_name: str,
|
||||
model_mode: str,
|
||||
mode: WorkflowGenerationMode,
|
||||
mode: WorkflowGenerationModeRequest,
|
||||
instruction: str,
|
||||
ideal_output: str = "",
|
||||
tool_catalogue_text: str = "",
|
||||
@@ -263,20 +340,26 @@ class WorkflowGenerator:
|
||||
current_graph: dict[str, Any] | None = None,
|
||||
) -> WorkflowGenerateResultDict:
|
||||
"""
|
||||
Run planner → builder → postprocess and return a graph payload.
|
||||
Run planner → node builders → postprocess and return a graph payload.
|
||||
|
||||
``mode`` accepts the ``"auto"`` sentinel — the planner then chooses the
|
||||
concrete mode itself (echoed in its ``mode`` output field) so no extra
|
||||
classification call is needed; the resolution is stamped onto the
|
||||
result envelope.
|
||||
|
||||
``current_graph`` switches the pipeline from create mode to REFINE
|
||||
mode: the existing draft graph is injected into both the planner
|
||||
(compact node/edge summary) and the builder (full JSON) so the LLM
|
||||
amends the graph the user is editing instead of inventing a new one.
|
||||
``None`` (the default) is plain create-from-scratch behaviour.
|
||||
mode: the existing draft graph is summarized for the planner. Node
|
||||
builders receive only the config of the node they update, while configs
|
||||
marked ``keep`` are reused without an LLM call. ``None`` (the default)
|
||||
is plain create-from-scratch behaviour.
|
||||
|
||||
``tool_catalogue_text`` is the formatted list of installed tools for
|
||||
the calling tenant (see ``tool_catalogue.build_tool_catalogue`` /
|
||||
``format_tool_catalogue``). It's injected into both the planner and
|
||||
builder prompts so the LLM can pick concrete ``provider/tool``
|
||||
identifiers instead of inventing names; an empty string skips the
|
||||
section entirely (useful for unit tests).
|
||||
identifiers instead of inventing names; node builders receive it
|
||||
only for tool nodes. An empty string skips the section entirely (useful
|
||||
for unit tests).
|
||||
|
||||
``installed_tools`` is the structural sibling — a set of
|
||||
``(provider_name, tool_name)`` pairs the validator consults to reject
|
||||
@@ -316,7 +399,7 @@ class WorkflowGenerator:
|
||||
# The event generator always emits exactly one result envelope; this
|
||||
# fallback only guards against a future refactor that forgets to.
|
||||
if result is None:
|
||||
result = _with_mode(_empty_result(), mode)
|
||||
result = _with_mode(_empty_result(), _fallback_mode(mode))
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
@@ -328,7 +411,7 @@ class WorkflowGenerator:
|
||||
provider: str,
|
||||
model_name: str,
|
||||
model_mode: str,
|
||||
mode: WorkflowGenerationMode,
|
||||
mode: WorkflowGenerationModeRequest,
|
||||
instruction: str,
|
||||
ideal_output: str = "",
|
||||
tool_catalogue_text: str = "",
|
||||
@@ -368,7 +451,7 @@ class WorkflowGenerator:
|
||||
provider: str,
|
||||
model_name: str,
|
||||
model_mode: str,
|
||||
mode: WorkflowGenerationMode,
|
||||
mode: WorkflowGenerationModeRequest,
|
||||
instruction: str,
|
||||
ideal_output: str = "",
|
||||
tool_catalogue_text: str = "",
|
||||
@@ -376,7 +459,7 @@ class WorkflowGenerator:
|
||||
current_graph: dict[str, Any] | None = None,
|
||||
) -> Iterator[tuple[str, dict[str, Any]]]:
|
||||
"""
|
||||
Drive planner → builder → postprocess and yield generation events.
|
||||
Drive planner → node builders → postprocess and yield generation events.
|
||||
|
||||
Shared core for both ``generate_workflow_graph`` (keeps only the final
|
||||
``result``) and ``generate_workflow_graph_stream`` (streams every
|
||||
@@ -402,11 +485,16 @@ class WorkflowGenerator:
|
||||
),
|
||||
)
|
||||
if plan_err is not None:
|
||||
yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [plan_err]), mode))
|
||||
failed = _with_mode(_result_with_errors(_empty_result(), [plan_err]), _fallback_mode(mode))
|
||||
yield "result", cast(dict[str, Any], failed)
|
||||
return
|
||||
|
||||
# The lambda return is non-None when no error fired — narrow it for type-checkers.
|
||||
plan = cast(PlannerResultDict, plan)
|
||||
# ``auto`` requests resolve here — the planner echoed its mode choice
|
||||
# (or we infer it from the plan's terminal node). Explicit modes pass
|
||||
# through unchanged. Everything downstream uses the concrete mode.
|
||||
resolved_mode = _resolve_generation_mode(mode, plan)
|
||||
plan_nodes: list[dict[str, Any]] = cast(list[dict[str, Any]], plan.get("nodes", []))
|
||||
if not plan_nodes:
|
||||
empty_plan = _with_mode(
|
||||
@@ -414,15 +502,12 @@ class WorkflowGenerator:
|
||||
_empty_result(),
|
||||
[_err(WorkflowGenerateErrorCode.EMPTY_PLAN, "Planner returned no nodes")],
|
||||
),
|
||||
mode,
|
||||
resolved_mode,
|
||||
)
|
||||
yield "result", cast(dict[str, Any], empty_plan)
|
||||
return
|
||||
|
||||
# A single LLM cannot select multiple retrieval outputs as context.
|
||||
# Make the required template fan-in explicit in the plan so the
|
||||
# builder receives its schema and assigns stable sequential ids.
|
||||
cls._insert_multi_retrieval_template_plan(plan_nodes)
|
||||
plan_edges = [cast(dict[str, Any], edge) for edge in (plan.get("edges") or []) if isinstance(edge, dict)]
|
||||
|
||||
# Planner-supplied user-input declarations. The builder uses these to
|
||||
# populate ``start.data.variables`` so downstream ``{#start.<var>#}``
|
||||
@@ -436,34 +521,48 @@ class WorkflowGenerator:
|
||||
|
||||
# First event the stream sees: the high-level plan, before the slower
|
||||
# builder call. Non-streaming callers ignore it.
|
||||
yield "plan", _build_plan_event(plan=plan, plan_nodes=plan_nodes, start_inputs=start_inputs, mode=mode)
|
||||
yield "plan", _build_plan_event(plan=plan, plan_nodes=plan_nodes, start_inputs=start_inputs, mode=resolved_mode)
|
||||
|
||||
# ── 2. BUILDER ────────────────────────────────────────────────────
|
||||
graph, build_err = cls._run_stage(
|
||||
stage="Builder",
|
||||
failure_fallback_message="Failed to build workflow graph",
|
||||
run=lambda: cls._run_builder(
|
||||
builder_started_at = time.monotonic()
|
||||
|
||||
def build_graph() -> GraphDict:
|
||||
return cls._run_parallel_node_builders(
|
||||
model_instance=model_instance,
|
||||
model_parameters=model_parameters,
|
||||
provider=provider,
|
||||
model_name=model_name,
|
||||
model_mode=model_mode,
|
||||
mode=mode,
|
||||
mode=resolved_mode,
|
||||
instruction=instruction,
|
||||
ideal_output=ideal_output,
|
||||
plan_nodes=plan_nodes,
|
||||
plan_edges=plan_edges,
|
||||
tool_catalogue_text=tool_catalogue_text,
|
||||
start_inputs=start_inputs,
|
||||
current_graph=current_graph,
|
||||
),
|
||||
)
|
||||
|
||||
graph, build_err = cls._run_stage(
|
||||
stage="Builder",
|
||||
failure_fallback_message="Failed to build workflow graph",
|
||||
run=build_graph,
|
||||
)
|
||||
logger.info(
|
||||
"Workflow generator: node builders completed nodes=%s elapsed_ms=%.1f",
|
||||
len(plan_nodes),
|
||||
(time.monotonic() - builder_started_at) * 1000,
|
||||
)
|
||||
if build_err is not None:
|
||||
yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [build_err]), mode))
|
||||
yield (
|
||||
"result",
|
||||
cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [build_err]), resolved_mode)),
|
||||
)
|
||||
return
|
||||
graph = cast(GraphDict, graph)
|
||||
|
||||
# ── 3. POSTPROC + VALIDATE ────────────────────────────────────────
|
||||
graph = cls._postprocess_graph(graph=graph, mode=mode)
|
||||
graph = cls._postprocess_graph(graph=graph, mode=resolved_mode)
|
||||
|
||||
# ``app_name`` / ``icon`` are planner display metadata; both default
|
||||
# to "" when the LLM omits them — the FE owns the fallback.
|
||||
@@ -475,13 +574,13 @@ class WorkflowGenerator:
|
||||
"error": "",
|
||||
"errors": [],
|
||||
}
|
||||
_with_mode(result, mode)
|
||||
_with_mode(result, resolved_mode)
|
||||
|
||||
# Final structural sanity check — fail closed if start/end shape is
|
||||
# wrong, container topology is broken, a tool was hallucinated, or a
|
||||
# variable reference points at a node that won't expose it. We still
|
||||
# return the partial graph so the caller can debug or salvage it.
|
||||
structural_errors = cls._validate_structure(graph=graph, mode=mode, installed_tools=installed_tools)
|
||||
structural_errors = cls._validate_structure(graph=graph, mode=resolved_mode, installed_tools=installed_tools)
|
||||
if structural_errors:
|
||||
logger.warning("Workflow generator: structural validation failed: %s", structural_errors)
|
||||
yield "result", cast(dict[str, Any], _result_with_errors(result, structural_errors))
|
||||
@@ -622,14 +721,14 @@ class WorkflowGenerator:
|
||||
*,
|
||||
model_instance,
|
||||
model_parameters: dict[str, Any],
|
||||
mode: WorkflowGenerationMode,
|
||||
mode: WorkflowGenerationModeRequest,
|
||||
instruction: str,
|
||||
ideal_output: str,
|
||||
tool_catalogue_text: str,
|
||||
current_graph: dict[str, Any] | None = None,
|
||||
) -> PlannerResultDict:
|
||||
user_prompt = PLANNER_USER_PROMPT.format(
|
||||
mode=mode,
|
||||
mode=_planner_prompt_mode(mode),
|
||||
instruction=instruction.strip(),
|
||||
existing_graph_section=format_existing_graph_section(current_graph),
|
||||
ideal_output_section=format_ideal_output_section(ideal_output),
|
||||
@@ -639,59 +738,65 @@ class WorkflowGenerator:
|
||||
SystemPromptMessage(content=PLANNER_SYSTEM_PROMPT),
|
||||
UserPromptMessage(content=user_prompt),
|
||||
]
|
||||
clamped_parameters = _clamp_for_planner(model_parameters)
|
||||
parsed = cls._invoke_and_parse_json(
|
||||
model_instance=model_instance,
|
||||
messages=messages,
|
||||
model_parameters=_clamp_for_planner(model_parameters),
|
||||
model_parameters=clamped_parameters,
|
||||
stage="Planner",
|
||||
)
|
||||
try:
|
||||
return cls._validate_planner_schema(parsed)
|
||||
except _StageSchemaError:
|
||||
logger.info("Workflow generator: planner schema invalid; retrying once")
|
||||
parsed = cls._invoke_and_parse_json(
|
||||
model_instance=model_instance,
|
||||
messages=[*messages, UserPromptMessage(content=_PLANNER_SCHEMA_RETRY_HINT)],
|
||||
model_parameters=clamped_parameters,
|
||||
stage="Planner",
|
||||
)
|
||||
return cls._validate_planner_schema(parsed)
|
||||
|
||||
@staticmethod
|
||||
def _validate_planner_schema(parsed: dict[str, Any]) -> PlannerResultDict:
|
||||
"""Require the single planner contract consumed by node builders."""
|
||||
nodes = parsed.get("nodes")
|
||||
if not isinstance(nodes, list):
|
||||
raise _StageSchemaError("Planner", "missing 'nodes' array")
|
||||
if not nodes:
|
||||
return cast(PlannerResultDict, parsed)
|
||||
|
||||
node_ids: set[str] = set()
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict) or "node_type" not in node:
|
||||
if not isinstance(node, dict) or not node.get("node_type"):
|
||||
raise _StageSchemaError("Planner", f"malformed node entry: {node!r}")
|
||||
node_id = node.get("id")
|
||||
if not isinstance(node_id, str) or not node_id.strip():
|
||||
raise _StageSchemaError("Planner", f"node missing non-empty id: {node!r}")
|
||||
if node_id in node_ids:
|
||||
raise _StageSchemaError("Planner", f"duplicate node id: {node_id!r}")
|
||||
node_ids.add(node_id)
|
||||
|
||||
edges = parsed.get("edges")
|
||||
if not isinstance(edges, list) or not edges:
|
||||
raise _StageSchemaError("Planner", "missing non-empty 'edges' array")
|
||||
for edge in edges:
|
||||
if not isinstance(edge, dict):
|
||||
raise _StageSchemaError("Planner", f"malformed edge entry: {edge!r}")
|
||||
source = edge.get("source")
|
||||
target = edge.get("target")
|
||||
if not isinstance(source, str) or not isinstance(target, str):
|
||||
raise _StageSchemaError("Planner", f"edge missing source or target: {edge!r}")
|
||||
if source not in node_ids or target not in node_ids:
|
||||
raise _StageSchemaError("Planner", f"edge references unknown node: {edge!r}")
|
||||
|
||||
return cast(PlannerResultDict, parsed)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Plan normalization
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _insert_multi_retrieval_template_plan(plan_nodes: list[dict[str, Any]]) -> None:
|
||||
"""Insert the unambiguous multi-retrieval template step when omitted.
|
||||
|
||||
Multiple LLM nodes make ownership ambiguous, so that case remains in
|
||||
the planner's hands. With exactly one LLM, every independent retrieval
|
||||
result can safely fan into one template immediately before that LLM.
|
||||
"""
|
||||
node_types = [str(node.get("node_type") or "") for node in plan_nodes]
|
||||
if node_types.count(BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL) < 2:
|
||||
return
|
||||
if node_types.count(BuiltinNodeTypes.LLM) != 1:
|
||||
return
|
||||
if BuiltinNodeTypes.TEMPLATE_TRANSFORM in node_types:
|
||||
return
|
||||
|
||||
llm_index = node_types.index(BuiltinNodeTypes.LLM)
|
||||
retrievals_before_llm = node_types[:llm_index].count(BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL)
|
||||
if retrievals_before_llm < 2:
|
||||
return
|
||||
plan_nodes.insert(
|
||||
llm_index,
|
||||
{
|
||||
"label": "Combine Knowledge",
|
||||
"node_type": BuiltinNodeTypes.TEMPLATE_TRANSFORM,
|
||||
"purpose": "Combine every knowledge retrieval result into one labelled context for the LLM.",
|
||||
},
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Builder
|
||||
# ------------------------------------------------------------------
|
||||
@classmethod
|
||||
def _run_builder(
|
||||
def _run_parallel_node_builders(
|
||||
cls,
|
||||
*,
|
||||
model_instance,
|
||||
@@ -703,53 +808,275 @@ class WorkflowGenerator:
|
||||
instruction: str,
|
||||
ideal_output: str,
|
||||
plan_nodes: list[dict[str, Any]],
|
||||
plan_edges: list[dict[str, Any]],
|
||||
tool_catalogue_text: str,
|
||||
start_inputs: list[dict[str, Any]] | None = None,
|
||||
current_graph: dict[str, Any] | None = None,
|
||||
start_inputs: list[dict[str, Any]],
|
||||
current_graph: dict[str, Any] | None,
|
||||
) -> GraphDict:
|
||||
user_prompt = BUILDER_USER_PROMPT.format(
|
||||
"""Build changed node configs concurrently and expand them into a graph.
|
||||
|
||||
Refine plans can mark existing nodes as ``keep``; those nodes bypass
|
||||
the model entirely and retain their full data config. Every other node
|
||||
gets one compact call, with at most ``_node_builder_max_workers()``
|
||||
calls in flight. Any fragment failure aborts the graph, preserving the
|
||||
generator's existing fail-closed contract.
|
||||
"""
|
||||
existing_by_id = {
|
||||
str(node.get("id")): node
|
||||
for node in ((current_graph or {}).get("nodes") or [])
|
||||
if isinstance(node, dict) and node.get("id")
|
||||
}
|
||||
existing_edges = [edge for edge in ((current_graph or {}).get("edges") or []) if isinstance(edge, dict)]
|
||||
nodes_to_build = [
|
||||
node for node in plan_nodes if not (node.get("action") == "keep" and str(node.get("id")) in existing_by_id)
|
||||
]
|
||||
|
||||
# Shared across every builder call in this request — compute once.
|
||||
plan_json = format_parallel_plan(plan_nodes, plan_edges, start_inputs)
|
||||
mode_section = format_mode_section(mode)
|
||||
|
||||
configs_by_id: dict[str, dict[str, Any]] = {}
|
||||
if nodes_to_build:
|
||||
max_workers = min(_node_builder_max_workers(), len(nodes_to_build))
|
||||
with ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix="workflow-node-builder") as executor:
|
||||
futures = {
|
||||
executor.submit(
|
||||
cls._run_node_builder,
|
||||
model_instance=model_instance,
|
||||
model_parameters=model_parameters,
|
||||
provider=provider,
|
||||
model_name=model_name,
|
||||
model_mode=model_mode,
|
||||
mode_section=mode_section,
|
||||
instruction=instruction,
|
||||
ideal_output=ideal_output,
|
||||
target_node=node,
|
||||
plan_json=plan_json,
|
||||
tool_catalogue_text=tool_catalogue_text,
|
||||
start_inputs=start_inputs,
|
||||
existing_node=existing_by_id.get(str(node.get("id"))),
|
||||
): str(node.get("id"))
|
||||
for node in nodes_to_build
|
||||
}
|
||||
try:
|
||||
for future in as_completed(futures):
|
||||
node_id = futures[future]
|
||||
configs_by_id[node_id] = future.result()
|
||||
except BaseException:
|
||||
# Fail fast: one failed fragment aborts the whole graph, so
|
||||
# queued builder calls would only burn quota and delay the
|
||||
# error envelope. In-flight calls cannot be interrupted;
|
||||
# they finish while the pool shuts down.
|
||||
for pending in futures:
|
||||
pending.cancel()
|
||||
raise
|
||||
|
||||
return cls._assemble_parallel_graph(
|
||||
plan_nodes=plan_nodes,
|
||||
plan_edges=plan_edges,
|
||||
configs_by_id=configs_by_id,
|
||||
existing_by_id=existing_by_id,
|
||||
existing_edges=existing_edges,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _run_node_builder(
|
||||
cls,
|
||||
*,
|
||||
model_instance,
|
||||
model_parameters: dict[str, Any],
|
||||
provider: str,
|
||||
model_name: str,
|
||||
model_mode: str,
|
||||
mode_section: str,
|
||||
instruction: str,
|
||||
ideal_output: str,
|
||||
target_node: dict[str, Any],
|
||||
plan_json: str,
|
||||
tool_catalogue_text: str,
|
||||
start_inputs: list[dict[str, Any]],
|
||||
existing_node: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Generate only the semantic config for one normalized plan node."""
|
||||
node_id = str(target_node.get("id") or "")
|
||||
node_type = str(target_node.get("node_type") or "")
|
||||
model_section = ""
|
||||
if node_type in _MODEL_NODE_TYPES:
|
||||
model_section = (
|
||||
f"# Selected model (copy verbatim)\n\nprovider={provider}, name={model_name}, mode={model_mode}\n\n"
|
||||
)
|
||||
existing_config_section = ""
|
||||
if existing_node:
|
||||
existing_data = existing_node.get("data") if isinstance(existing_node.get("data"), dict) else {}
|
||||
existing_config_section = (
|
||||
"# Existing config to preserve unless the instruction changes it\n\n"
|
||||
f"{json.dumps(existing_data, ensure_ascii=False, separators=(',', ':'))}\n\n"
|
||||
)
|
||||
user_prompt = NODE_BUILDER_USER_PROMPT.format(
|
||||
node_id=node_id,
|
||||
node_type=node_type,
|
||||
label=str(target_node.get("label") or ""),
|
||||
purpose=str(target_node.get("purpose") or ""),
|
||||
instruction=instruction.strip(),
|
||||
ideal_output_section=format_ideal_output_section(ideal_output),
|
||||
existing_graph_section=format_builder_existing_graph_section(current_graph),
|
||||
provider=provider,
|
||||
name=model_name,
|
||||
mode_label=model_mode,
|
||||
plan_block=format_plan_block(plan_nodes),
|
||||
tool_catalogue_section=format_builder_tool_catalogue_section(tool_catalogue_text),
|
||||
start_inputs_section=format_start_inputs_section(start_inputs or []),
|
||||
mode_section=mode_section,
|
||||
model_section=model_section,
|
||||
tool_catalogue_section=(
|
||||
format_node_tool_catalogue_section(tool_catalogue_text) if node_type == BuiltinNodeTypes.TOOL else ""
|
||||
),
|
||||
start_inputs_section=(
|
||||
format_start_inputs_section(start_inputs) if node_type == BuiltinNodeTypes.START else ""
|
||||
),
|
||||
existing_config_section=existing_config_section,
|
||||
plan_json=plan_json,
|
||||
)
|
||||
# Scope the builder cheatsheet to exactly the node types the planner
|
||||
# chose, so the prompt carries each type's FULL schema (e.g. a file
|
||||
# start variable's required ``allowed_file_types``) without dragging in
|
||||
# config for unrelated node types.
|
||||
plan_node_types = {
|
||||
str(node.get("node_type") or "").strip() for node in plan_nodes if str(node.get("node_type") or "").strip()
|
||||
}
|
||||
messages = [
|
||||
SystemPromptMessage(content=get_builder_system_prompt(mode, plan_node_types)),
|
||||
UserPromptMessage(content=user_prompt),
|
||||
]
|
||||
parsed = cls._invoke_and_parse_json(
|
||||
model_instance=model_instance,
|
||||
messages=messages,
|
||||
messages=[
|
||||
SystemPromptMessage(content=get_node_builder_system_prompt(node_type)),
|
||||
UserPromptMessage(content=user_prompt),
|
||||
],
|
||||
model_parameters=model_parameters,
|
||||
stage="Builder",
|
||||
stage=f"Builder {node_id}",
|
||||
)
|
||||
config = parsed.get("config")
|
||||
if not isinstance(config, dict):
|
||||
raise _StageSchemaError(f"Builder {node_id}", "missing 'config' object")
|
||||
return cast(dict[str, Any], config)
|
||||
|
||||
nodes = parsed.get("nodes")
|
||||
edges = parsed.get("edges")
|
||||
if not isinstance(nodes, list) or not isinstance(edges, list):
|
||||
raise _StageSchemaError("Builder", "graph missing 'nodes' or 'edges' arrays")
|
||||
@classmethod
|
||||
def _assemble_parallel_graph(
|
||||
cls,
|
||||
*,
|
||||
plan_nodes: list[dict[str, Any]],
|
||||
plan_edges: list[dict[str, Any]],
|
||||
configs_by_id: dict[str, dict[str, Any]],
|
||||
existing_by_id: dict[str, dict[str, Any]],
|
||||
existing_edges: list[dict[str, Any]] | None = None,
|
||||
) -> GraphDict:
|
||||
"""Expand compact node configs and planner topology into graph JSON.
|
||||
|
||||
viewport = parsed.get("viewport") or _DEFAULT_VIEWPORT
|
||||
return cast(
|
||||
GraphDict,
|
||||
{
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"viewport": viewport,
|
||||
},
|
||||
)
|
||||
``existing_edges`` (refine only) preserves wiring the planner cannot
|
||||
express: the synthetic ``<container>start`` entry edge keeps its
|
||||
existing target instead of being re-pointed at whichever child the
|
||||
planner happened to list first.
|
||||
"""
|
||||
label_to_id = {
|
||||
str(node.get("label")): str(node.get("id")) for node in plan_nodes if node.get("label") and node.get("id")
|
||||
}
|
||||
type_by_id = {str(node.get("id")): str(node.get("node_type") or "") for node in plan_nodes}
|
||||
children_by_parent: dict[str, list[str]] = {}
|
||||
nodes: list[dict[str, Any]] = []
|
||||
|
||||
for planned in plan_nodes:
|
||||
node_id = str(planned.get("id") or "")
|
||||
node_type = str(planned.get("node_type") or "")
|
||||
existing = existing_by_id.get(node_id)
|
||||
node: dict[str, Any]
|
||||
if planned.get("action") == "keep" and existing is not None:
|
||||
node = deepcopy(existing)
|
||||
else:
|
||||
config = dict(configs_by_id.get(node_id) or {})
|
||||
for shared_key in ("type", "title", "desc", "selected"):
|
||||
config.pop(shared_key, None)
|
||||
data: dict[str, Any] = {
|
||||
"type": node_type,
|
||||
"title": str(planned.get("label") or node_id),
|
||||
"desc": str(planned.get("purpose") or ""),
|
||||
**config,
|
||||
}
|
||||
node = deepcopy(existing) if existing is not None else {"id": node_id}
|
||||
node["id"] = node_id
|
||||
node["data"] = data
|
||||
|
||||
parent_ref = str(planned.get("parent") or "")
|
||||
parent_id = label_to_id.get(parent_ref, parent_ref)
|
||||
if not parent_id and str(node.get("parentId") or "") in type_by_id:
|
||||
# Kept nodes rarely re-state containment — recover the parent
|
||||
# from the deepcopied wrapper so entry-edge synthesis still
|
||||
# counts this child.
|
||||
parent_id = str(node["parentId"])
|
||||
if parent_id:
|
||||
child_index = len(children_by_parent.get(parent_id, []))
|
||||
node["parentId"] = parent_id
|
||||
node.setdefault("position", {"x": 240 + 260 * child_index, "y": 60})
|
||||
node.setdefault("data", {})
|
||||
parent_type = type_by_id.get(parent_id)
|
||||
if parent_type == BuiltinNodeTypes.ITERATION:
|
||||
node["data"].setdefault("isInIteration", True)
|
||||
node["data"].setdefault("iteration_id", parent_id)
|
||||
elif parent_type == BuiltinNodeTypes.LOOP:
|
||||
node["data"].setdefault("isInLoop", True)
|
||||
node["data"].setdefault("loop_id", parent_id)
|
||||
children_by_parent.setdefault(parent_id, []).append(node_id)
|
||||
elif node.get("parentId"):
|
||||
# The container was dropped from the plan: strip the stale
|
||||
# containment markers so the kept node rejoins the top level
|
||||
# (and its auto-layout) instead of pointing at a deleted parent.
|
||||
for wrapper_key in ("parentId", "extent", "zIndex", "position", "positionAbsolute"):
|
||||
node.pop(wrapper_key, None)
|
||||
if isinstance(node.get("data"), dict):
|
||||
for marker_key in ("isInIteration", "iteration_id", "isInLoop", "loop_id"):
|
||||
node["data"].pop(marker_key, None)
|
||||
|
||||
nodes.append(node)
|
||||
if node_type in cls._CONTAINER_TYPES:
|
||||
start_id = f"{node_id}start"
|
||||
node.setdefault("data", {})["start_node_id"] = start_id
|
||||
node.setdefault("width", 808)
|
||||
node.setdefault("height", 204)
|
||||
node.setdefault("zIndex", 1)
|
||||
is_iteration = node_type == BuiltinNodeTypes.ITERATION
|
||||
nodes.append(
|
||||
{
|
||||
"id": start_id,
|
||||
"type": "custom-iteration-start" if is_iteration else "custom-loop-start",
|
||||
"parentId": node_id,
|
||||
"extent": "parent",
|
||||
"draggable": False,
|
||||
"selectable": False,
|
||||
"zIndex": 1002,
|
||||
"position": {"x": 60, "y": 78},
|
||||
"data": {
|
||||
"type": "iteration-start" if is_iteration else "loop-start",
|
||||
"title": "",
|
||||
"desc": "",
|
||||
"selected": False,
|
||||
"isInIteration" if is_iteration else "isInLoop": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
edges: list[dict[str, Any]] = []
|
||||
for planned_edge in plan_edges:
|
||||
edge: dict[str, Any] = {
|
||||
"source": str(planned_edge.get("source") or ""),
|
||||
"target": str(planned_edge.get("target") or ""),
|
||||
}
|
||||
source_handle = planned_edge.get("source_handle") or planned_edge.get("sourceHandle")
|
||||
target_handle = planned_edge.get("target_handle") or planned_edge.get("targetHandle")
|
||||
if source_handle:
|
||||
edge["sourceHandle"] = str(source_handle)
|
||||
if target_handle:
|
||||
edge["targetHandle"] = str(target_handle)
|
||||
edges.append(edge)
|
||||
|
||||
# Synthesize each container's entry edge. Refine keeps the existing
|
||||
# entry target when it is still a child — the planner's node listing
|
||||
# order says nothing about execution order inside a kept container.
|
||||
planned_sources = {str(edge.get("source") or "") for edge in edges}
|
||||
existing_entry_targets = {
|
||||
str(edge.get("source") or ""): str(edge.get("target") or "") for edge in (existing_edges or [])
|
||||
}
|
||||
for parent_id, child_ids in children_by_parent.items():
|
||||
start_id = f"{parent_id}start"
|
||||
if not child_ids or start_id in planned_sources:
|
||||
continue
|
||||
preferred = existing_entry_targets.get(start_id, "")
|
||||
entry_target = preferred if preferred in child_ids else child_ids[0]
|
||||
edges.append({"source": start_id, "target": entry_target})
|
||||
|
||||
return cast(GraphDict, {"nodes": nodes, "edges": edges, "viewport": _DEFAULT_VIEWPORT})
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Postprocessing
|
||||
@@ -1225,6 +1552,11 @@ class WorkflowGenerator:
|
||||
return var == "output"
|
||||
if node_type == BuiltinNodeTypes.LIST_OPERATOR:
|
||||
return var in {"result", "first_record", "last_record"}
|
||||
if node_type == BuiltinNodeTypes.HUMAN_INPUT:
|
||||
return any(
|
||||
isinstance(item, dict) and item.get("output_variable_name") == var
|
||||
for item in (data.get("inputs") or [])
|
||||
)
|
||||
# Other node types (if-else, iteration-start, loop-start, ...) don't
|
||||
# produce outputs of their own.
|
||||
return False
|
||||
@@ -1247,6 +1579,15 @@ class WorkflowGenerator:
|
||||
if isinstance(parameter, dict) and isinstance(parameter.get("name"), str)
|
||||
]
|
||||
return parameters[0] if len(parameters) == 1 else None
|
||||
if node_type == BuiltinNodeTypes.HUMAN_INPUT:
|
||||
human_outputs: list[str] = []
|
||||
for item in data.get("inputs") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
output_name = item.get("output_variable_name")
|
||||
if isinstance(output_name, str):
|
||||
human_outputs.append(output_name)
|
||||
return human_outputs[0] if len(human_outputs) == 1 else None
|
||||
if not isinstance(node_type, str):
|
||||
return None
|
||||
single_output_by_type: dict[str, str] = {
|
||||
@@ -1533,6 +1874,12 @@ class WorkflowGenerator:
|
||||
for klass in (data.get("classes") or [])
|
||||
if isinstance(klass, dict) and klass.get("id")
|
||||
]
|
||||
elif node_type == BuiltinNodeTypes.HUMAN_INPUT:
|
||||
branch_handles = [
|
||||
str(action["id"])
|
||||
for action in (data.get("user_actions") or [])
|
||||
if isinstance(action, dict) and action.get("id")
|
||||
]
|
||||
else:
|
||||
continue
|
||||
|
||||
@@ -1941,8 +2288,8 @@ class WorkflowGenerator:
|
||||
"""
|
||||
Validate iteration / loop topology:
|
||||
|
||||
* every container has at least one child whose ``parentId``
|
||||
points at it;
|
||||
* every container has at least one executable child whose
|
||||
``parentId`` points at it;
|
||||
* every non-container node with a ``parentId`` points at a real
|
||||
container, not at a non-container node;
|
||||
* no cycles in the parent chain (a node cannot be its own
|
||||
@@ -1959,7 +2306,9 @@ class WorkflowGenerator:
|
||||
if not isinstance(parent, str) or not parent:
|
||||
continue
|
||||
if parent in container_ids:
|
||||
children_by_parent.setdefault(parent, []).append(n.get("id", ""))
|
||||
node_type = (n.get("data") or {}).get("type")
|
||||
if node_type not in {"iteration-start", "loop-start"}:
|
||||
children_by_parent.setdefault(parent, []).append(n.get("id", ""))
|
||||
elif parent in by_id:
|
||||
# Parent exists but isn't a container — that's a topology bug.
|
||||
out.append(
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
Typed payloads for workflow generation.
|
||||
|
||||
These TypedDicts describe the shape that the planner and builder LLM calls are
|
||||
required to return after ``json_repair`` parsing. They mirror the runtime
|
||||
``graph`` shape consumed by ``WorkflowService.sync_draft_workflow`` so the output
|
||||
can be written straight into a draft workflow without further translation.
|
||||
These TypedDicts describe the planner payload and the runtime graph assembled
|
||||
from builder LLM responses after ``json_repair`` parsing. The graph types mirror
|
||||
the shape consumed by ``WorkflowService.sync_draft_workflow`` so the output can
|
||||
be written straight into a draft workflow.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
@@ -12,11 +12,10 @@ from typing import Literal, NotRequired, TypedDict
|
||||
|
||||
WorkflowGenerationMode = Literal["workflow", "advanced-chat"]
|
||||
|
||||
# The mode accepted at the API boundary. ``auto`` is a sentinel that asks the
|
||||
# service to classify the instruction into a concrete ``WorkflowGenerationMode``
|
||||
# (one tiny LLM call) BEFORE planning — see
|
||||
# ``WorkflowGeneratorService._resolve_mode`` and
|
||||
# ``LLMGenerator.classify_workflow_mode``.
|
||||
# The mode accepted at the API boundary. ``auto`` is a sentinel that delegates
|
||||
# the choice to the planner: it echoes a concrete mode in its ``mode`` output
|
||||
# field (falling back to terminal-node inference, then ``advanced-chat``) —
|
||||
# see ``runner._resolve_generation_mode``. No extra LLM call is involved.
|
||||
WorkflowGenerationModeRequest = Literal["workflow", "advanced-chat", "auto"]
|
||||
|
||||
|
||||
@@ -58,9 +57,21 @@ class WorkflowGenerateErrorDict(TypedDict):
|
||||
class PlannerNodeDict(TypedDict):
|
||||
"""One node from the planner's high-level plan."""
|
||||
|
||||
id: NotRequired[str]
|
||||
label: str
|
||||
node_type: str
|
||||
purpose: str
|
||||
parent: NotRequired[str]
|
||||
action: NotRequired[Literal["keep", "update", "add"]]
|
||||
|
||||
|
||||
class PlannerEdgeDict(TypedDict):
|
||||
"""Compact topology emitted by the planner for parallel node building."""
|
||||
|
||||
source: str
|
||||
target: str
|
||||
source_handle: NotRequired[str]
|
||||
target_handle: NotRequired[str]
|
||||
|
||||
|
||||
class PlannerStartInputDict(TypedDict):
|
||||
@@ -82,10 +93,15 @@ class PlannerResultDict(TypedDict):
|
||||
|
||||
title: str
|
||||
description: str
|
||||
# Concrete mode the planner chose ("workflow" / "advanced-chat"). Parsed
|
||||
# leniently — an ``auto`` request infers the mode from the terminal node
|
||||
# when this is missing or invalid, so a bad value never fails the plan.
|
||||
mode: NotRequired[str]
|
||||
app_name: NotRequired[str]
|
||||
icon: NotRequired[str]
|
||||
start_inputs: NotRequired[list[PlannerStartInputDict]]
|
||||
nodes: list[PlannerNodeDict]
|
||||
edges: NotRequired[list[PlannerEdgeDict]]
|
||||
|
||||
|
||||
class GraphNodePositionDict(TypedDict):
|
||||
|
||||
@@ -499,6 +499,9 @@ class DifyNodeFactory(NodeFactory):
|
||||
base_url=dify_config.AGENT_BACKEND_BASE_URL,
|
||||
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
|
||||
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
|
||||
stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
|
||||
stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
|
||||
stream_run_timeout_seconds=dify_config.AGENT_BACKEND_RUN_TIMEOUT_SECONDS,
|
||||
),
|
||||
"event_adapter": AgentBackendRunEventAdapter(),
|
||||
# Agent Files §4.6: reback file outputs from the ToolFile row so
|
||||
|
||||
@@ -5,6 +5,7 @@ from collections.abc import Generator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, override
|
||||
|
||||
from agenton.compositor import CompositorSessionSnapshot
|
||||
from dify_agent.protocol import CancelRunRequest
|
||||
|
||||
from clients.agent_backend import (
|
||||
AgentBackendAgentMessageDeltaInternalEvent,
|
||||
@@ -473,7 +474,10 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
"""
|
||||
stream_event_count = 0
|
||||
try:
|
||||
for public_event in self._agent_backend_client.stream_events(run_id):
|
||||
for public_event in self._agent_backend_client.stream_events(
|
||||
run_id,
|
||||
should_stop=self._is_graph_aborted,
|
||||
):
|
||||
stream_event_count += 1
|
||||
for internal_event in self._event_adapter.adapt(public_event):
|
||||
if internal_event.type == AgentBackendInternalEventType.RUN_STARTED:
|
||||
@@ -501,6 +505,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
| AgentBackendDeferredToolCallInternalEvent,
|
||||
):
|
||||
return internal_event, None
|
||||
self._cancel_backend_run(run_id, reason="unexpected_event")
|
||||
return None, self._failure_event(
|
||||
inputs={},
|
||||
process_data={},
|
||||
@@ -509,6 +514,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
error_type="agent_backend_stream_error",
|
||||
)
|
||||
except AgentBackendError as error:
|
||||
self._cancel_backend_run(run_id, reason=self._stream_stop_reason())
|
||||
return None, self._failure_event(
|
||||
inputs={},
|
||||
process_data={},
|
||||
@@ -517,6 +523,7 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
error_type=self._agent_backend_error_type(error),
|
||||
)
|
||||
except Exception as error:
|
||||
self._cancel_backend_run(run_id, reason=self._stream_stop_reason())
|
||||
return None, self._failure_event(
|
||||
inputs={},
|
||||
process_data={},
|
||||
@@ -525,8 +532,28 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
|
||||
error_type="agent_backend_stream_error",
|
||||
)
|
||||
|
||||
self._cancel_backend_run(run_id, reason="stream_ended_without_terminal_event")
|
||||
return None, None
|
||||
|
||||
def _is_graph_aborted(self) -> bool:
|
||||
"""Let Agent SSE consumption observe GraphEngine's cooperative abort state."""
|
||||
try:
|
||||
return self.graph_runtime_state.graph_execution.aborted
|
||||
except (AttributeError, RuntimeError):
|
||||
return False
|
||||
|
||||
def _stream_stop_reason(self) -> str:
|
||||
return "workflow_graph_aborted" if self._is_graph_aborted() else "event_stream_failed"
|
||||
|
||||
def _cancel_backend_run(self, run_id: str, *, reason: str) -> None:
|
||||
try:
|
||||
self._agent_backend_client.cancel_run(
|
||||
run_id,
|
||||
CancelRunRequest(reason=reason, message="Workflow Agent event consumption stopped"),
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to cancel Workflow Agent backend run: run_id=%s", run_id, exc_info=True)
|
||||
|
||||
@staticmethod
|
||||
def _record_type_check_metadata(metadata: dict[str, Any], outcome: OutputTypeCheckOutcome) -> None:
|
||||
# Surface enough detail in metadata for Inspector / debug logs without
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, Protocol, cast
|
||||
from typing import Any, Final, Literal, Protocol
|
||||
|
||||
from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolProviderType, DifyCoreToolsLayerConfig
|
||||
from dify_agent.layers.dify_plugin import (
|
||||
@@ -29,6 +29,13 @@ from models.provider_ids import ToolProviderID
|
||||
from models.tools import WorkflowToolProvider
|
||||
from services.tools.mcp_tools_manage_service import MCPToolManageService
|
||||
|
||||
_CORE_TOOL_PROVIDER_TYPES: Final[dict[ToolProviderType, DifyCoreToolProviderType]] = {
|
||||
ToolProviderType.BUILT_IN: "builtin",
|
||||
ToolProviderType.API: "api",
|
||||
ToolProviderType.WORKFLOW: "workflow",
|
||||
ToolProviderType.MCP: "mcp",
|
||||
}
|
||||
|
||||
|
||||
class WorkflowAgentDifyToolsBuildError(ValueError):
|
||||
"""Raised when Agent Soul tools cannot be prepared for Agent backend."""
|
||||
@@ -235,7 +242,7 @@ class WorkflowAgentDifyToolsBuilder:
|
||||
if tool_config.tool_name is not None:
|
||||
expanded.append(tool_config)
|
||||
continue
|
||||
provider_type = ToolProviderType.value_of(tool_config.provider_type)
|
||||
provider_type = tool_config.provider_type
|
||||
provider_id = self._provider_id(tool_config)
|
||||
try:
|
||||
tool_names = self._provider_declared_tool_names(
|
||||
@@ -279,7 +286,7 @@ class WorkflowAgentDifyToolsBuilder:
|
||||
tenant_id: str,
|
||||
tool_config: AgentSoulDifyToolConfig,
|
||||
) -> AgentSoulDifyToolConfig:
|
||||
if tool_config.provider_type != ToolProviderType.MCP.value:
|
||||
if tool_config.provider_type is not ToolProviderType.MCP:
|
||||
return tool_config
|
||||
provider_id = self._mcp_provider_id_resolver(tenant_id=tenant_id, provider_id=self._provider_id(tool_config))
|
||||
return tool_config.model_copy(update={"provider_id": provider_id, "plugin_id": None, "provider": None})
|
||||
@@ -326,7 +333,7 @@ class WorkflowAgentDifyToolsBuilder:
|
||||
def _to_agent_tool_entity(tool_config: AgentSoulDifyToolConfig) -> AgentToolEntity:
|
||||
assert tool_config.tool_name is not None
|
||||
return AgentToolEntity(
|
||||
provider_type=ToolProviderType.value_of(tool_config.provider_type),
|
||||
provider_type=tool_config.provider_type,
|
||||
provider_id=WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
|
||||
tool_name=tool_config.tool_name,
|
||||
tool_parameters=dict(tool_config.runtime_parameters),
|
||||
@@ -343,24 +350,16 @@ class WorkflowAgentDifyToolsBuilder:
|
||||
|
||||
@staticmethod
|
||||
def _provider_key(tool_config: AgentSoulDifyToolConfig) -> tuple[ToolProviderType, str]:
|
||||
return (
|
||||
ToolProviderType.value_of(tool_config.provider_type),
|
||||
WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
|
||||
)
|
||||
return (tool_config.provider_type, WorkflowAgentDifyToolsBuilder._provider_id(tool_config))
|
||||
|
||||
@staticmethod
|
||||
def _tool_layer_destination(tool_config: AgentSoulDifyToolConfig) -> Literal["plugin", "core"]:
|
||||
provider_type = ToolProviderType.value_of(tool_config.provider_type)
|
||||
provider_type = tool_config.provider_type
|
||||
if provider_type is ToolProviderType.PLUGIN or (
|
||||
provider_type is ToolProviderType.BUILT_IN and _is_plugin_provider_id(tool_config.provider_id)
|
||||
):
|
||||
return "plugin"
|
||||
if provider_type in {
|
||||
ToolProviderType.BUILT_IN,
|
||||
ToolProviderType.API,
|
||||
ToolProviderType.WORKFLOW,
|
||||
ToolProviderType.MCP,
|
||||
}:
|
||||
if provider_type in _CORE_TOOL_PROVIDER_TYPES:
|
||||
return "core"
|
||||
if provider_type is ToolProviderType.DATASET_RETRIEVAL:
|
||||
raise WorkflowAgentDifyToolsBuildError(
|
||||
@@ -417,7 +416,7 @@ class WorkflowAgentDifyToolsBuilder:
|
||||
) -> DifyCoreToolConfig:
|
||||
parameters = self._prepared_parameters(tool_runtime)
|
||||
return DifyCoreToolConfig(
|
||||
provider_type=cast(DifyCoreToolProviderType, tool_config.provider_type),
|
||||
provider_type=_CORE_TOOL_PROVIDER_TYPES[tool_config.provider_type],
|
||||
provider_id=self._provider_id(tool_config),
|
||||
tool_name=tool_config.tool_name or exposed_name,
|
||||
credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
|
||||
|
||||
@@ -61,16 +61,33 @@ class WorkflowAgentNodeValidator:
|
||||
|
||||
@classmethod
|
||||
def validate_draft_workflow(cls, *, session: Session, workflow: Workflow) -> None:
|
||||
cls._validate_workflow(session=session, workflow=workflow, require_binding=False)
|
||||
cls._validate_workflow(
|
||||
session=session,
|
||||
workflow=workflow,
|
||||
require_binding=False,
|
||||
validate_previous_node_topology=False,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def validate_published_workflow(cls, *, session: Session, workflow: Workflow) -> None:
|
||||
cls._validate_workflow(session=session, workflow=workflow, require_binding=True)
|
||||
cls._validate_workflow(
|
||||
session=session,
|
||||
workflow=workflow,
|
||||
require_binding=True,
|
||||
validate_previous_node_topology=True,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _validate_workflow(cls, *, session: Session, workflow: Workflow, require_binding: bool) -> None:
|
||||
def _validate_workflow(
|
||||
cls,
|
||||
*,
|
||||
session: Session,
|
||||
workflow: Workflow,
|
||||
require_binding: bool,
|
||||
validate_previous_node_topology: bool,
|
||||
) -> None:
|
||||
graph = workflow.graph_dict
|
||||
topology = _WorkflowGraphTopology.from_graph(graph)
|
||||
topology = _WorkflowGraphTopology.from_graph(graph) if validate_previous_node_topology else None
|
||||
for node_id, node_data in cls.iter_agent_v2_nodes(graph):
|
||||
cls._validate_node_schema(node_id=node_id, node_data=node_data)
|
||||
binding = cls._find_binding(
|
||||
@@ -185,12 +202,12 @@ class WorkflowAgentNodeValidator:
|
||||
raise WorkflowAgentNodeValidationError(
|
||||
f"Workflow Agent node {binding.node_id} has invalid previous node output ref."
|
||||
)
|
||||
if topology is None:
|
||||
continue
|
||||
if len(selector) < 2:
|
||||
raise WorkflowAgentNodeValidationError(
|
||||
f"Workflow Agent node {binding.node_id} has incomplete previous node output ref."
|
||||
)
|
||||
if topology is None:
|
||||
continue
|
||||
source_node_id = selector[0]
|
||||
if not topology.has_node(source_node_id):
|
||||
raise WorkflowAgentNodeValidationError(
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
"""Validate Dify Console KnowledgeFS declarations against a pinned OpenAPI document.
|
||||
|
||||
The OpenAPI document is exported only during explicit development validation. Runtime declarations live with Dify
|
||||
product policy; this module validates their transport metadata without generating a complete operation catalog.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypedDict
|
||||
|
||||
API_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(API_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(API_ROOT))
|
||||
|
||||
WORKSPACE_ROOT = API_ROOT.parent
|
||||
LOCK_PATH = API_ROOT / "knowledge-fs-contract.lock.json"
|
||||
DEFAULT_REPOSITORY = WORKSPACE_ROOT.parent / "knowledge-fs"
|
||||
OPENAPI_METHODS = ("delete", "get", "head", "options", "patch", "post", "put", "trace")
|
||||
PROXY_METHODS = frozenset({"delete", "get", "patch", "post", "put"})
|
||||
CONSOLE_PROXY_ERROR_SCHEMA_NAME = "ConsoleProxyError"
|
||||
CONSOLE_PROXY_ERROR_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"required": ["code", "message", "status"],
|
||||
"properties": {
|
||||
"code": {"type": "string"},
|
||||
"message": {"type": "string"},
|
||||
"status": {"type": "integer"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class ContractDeclaration(TypedDict):
|
||||
"""KnowledgeFS transport contract declared by one Dify Console registry entry."""
|
||||
|
||||
operation_id: str
|
||||
method: str
|
||||
path: str
|
||||
required_scope: str | None
|
||||
response_kind: str
|
||||
max_response_bytes: int
|
||||
request_headers: tuple[str, ...]
|
||||
response_headers: tuple[str, ...]
|
||||
response_media_types: tuple[str, ...]
|
||||
error_status_map: tuple[tuple[int, int], ...]
|
||||
|
||||
|
||||
type DeclarationField = Literal[
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
]
|
||||
|
||||
DECLARATION_FIELDS: tuple[DeclarationField, ...] = (
|
||||
"method",
|
||||
"path",
|
||||
"required_scope",
|
||||
"response_kind",
|
||||
"max_response_bytes",
|
||||
"request_headers",
|
||||
"response_headers",
|
||||
"response_media_types",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Update or verify the pin and validate Console declarations against its OpenAPI document."""
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--update-lock", action="store_true")
|
||||
parser.add_argument("--repository", type=Path, default=DEFAULT_REPOSITORY)
|
||||
parser.add_argument("--output-openapi", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
repository = args.repository.resolve()
|
||||
lock = json.loads(LOCK_PATH.read_text())
|
||||
tracked_changes = run("git", "status", "--porcelain", "--untracked-files=no", cwd=repository).strip()
|
||||
if tracked_changes:
|
||||
raise RuntimeError("KnowledgeFS checkout must not contain tracked changes during contract export")
|
||||
|
||||
commit = run("git", "rev-parse", "HEAD", cwd=repository).strip()
|
||||
if not args.update_lock and commit != lock["commit"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS checkout mismatch: expected {lock['commit']}, received {commit}. "
|
||||
"Use the pinned commit or pass --update-lock intentionally."
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="dify-knowledge-fs-contract-") as directory:
|
||||
openapi_path = Path(directory) / "knowledge-fs.openapi.json"
|
||||
subprocess.run(
|
||||
["pnpm", "openapi:export", "--", "--output", str(openapi_path)],
|
||||
cwd=repository,
|
||||
check=True,
|
||||
)
|
||||
openapi_content = openapi_path.read_bytes()
|
||||
|
||||
openapi_sha256 = sha256(openapi_content)
|
||||
if not args.update_lock and openapi_sha256 != lock["openapiSha256"]:
|
||||
raise RuntimeError(
|
||||
f"KnowledgeFS OpenAPI hash mismatch: expected {lock['openapiSha256']}, received {openapi_sha256}"
|
||||
)
|
||||
|
||||
document: dict[str, Any] = json.loads(openapi_content)
|
||||
declarations = console_contract_declarations()
|
||||
validate_declarations(document, declarations)
|
||||
|
||||
if args.output_openapi:
|
||||
filtered_document = filter_openapi_document(document, declarations)
|
||||
filtered_document["x-dify-source-openapi-sha256"] = openapi_sha256
|
||||
filtered_document["x-dify-console-declarations-sha256"] = contract_declarations_sha256(declarations)
|
||||
args.output_openapi.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_openapi.write_text(json.dumps(filtered_document, indent=2) + "\n")
|
||||
|
||||
if args.update_lock:
|
||||
LOCK_PATH.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"commit": commit,
|
||||
"openapiSha256": openapi_sha256,
|
||||
"repository": lock["repository"],
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
|
||||
def validate_declarations(document: dict[str, Any], declarations: tuple[ContractDeclaration, ...]) -> None:
|
||||
"""Validate Dify Console declarations against matching pinned OpenAPI operations."""
|
||||
operations_by_id: dict[str, list[tuple[str, str, dict[str, Any], dict[str, Any]]]] = {}
|
||||
for path, path_item in document.get("paths", {}).items():
|
||||
for method in OPENAPI_METHODS:
|
||||
operation = path_item.get(method)
|
||||
if operation is None:
|
||||
continue
|
||||
operation_id = operation.get("operationId")
|
||||
if isinstance(operation_id, str) and operation_id:
|
||||
operations_by_id.setdefault(operation_id, []).append((method, path, path_item, operation))
|
||||
|
||||
declared_ids: set[str] = set()
|
||||
for declaration in declarations:
|
||||
operation_id = declaration["operation_id"]
|
||||
if operation_id in declared_ids:
|
||||
raise ValueError(f"Dify Console registry has duplicate operationId: {operation_id}")
|
||||
declared_ids.add(operation_id)
|
||||
|
||||
matches = operations_by_id.get(operation_id, [])
|
||||
if not matches:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has no operationId: {operation_id}")
|
||||
if len(matches) > 1:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI has duplicate operationId: {operation_id}")
|
||||
|
||||
method, path, path_item, operation = matches[0]
|
||||
if not path.startswith("/"):
|
||||
raise ValueError(f"KnowledgeFS OpenAPI path must be absolute: {path}")
|
||||
if method not in PROXY_METHODS:
|
||||
raise ValueError(f"KnowledgeFS proxy does not support {method.upper()} {path}")
|
||||
expected: dict[DeclarationField, object] = {
|
||||
"method": method.upper(),
|
||||
"path": path[1:],
|
||||
"required_scope": required_scope(operation),
|
||||
"response_kind": response_kind(operation),
|
||||
"max_response_bytes": required_max_response_bytes(operation),
|
||||
"request_headers": request_header_names(path_item, operation),
|
||||
"response_headers": response_header_names(operation),
|
||||
"response_media_types": response_media_types(operation),
|
||||
}
|
||||
for field in DECLARATION_FIELDS:
|
||||
expected_value = expected[field]
|
||||
received_value = declaration[field]
|
||||
if received_value != expected_value:
|
||||
raise ValueError(
|
||||
f"KnowledgeFS operation {operation_id} field {field} drifted: "
|
||||
f"expected {expected_value!r}, received {received_value!r}"
|
||||
)
|
||||
validate_error_status_map(operation_id, declaration["error_status_map"])
|
||||
|
||||
|
||||
def filter_openapi_document(
|
||||
document: dict[str, Any],
|
||||
declarations: tuple[ContractDeclaration, ...],
|
||||
) -> dict[str, Any]:
|
||||
"""Return a code-generation document containing only Console-allowlisted operations."""
|
||||
filtered_document: dict[str, Any] = {
|
||||
key: value for key, value in document.items() if key not in {"components", "paths"}
|
||||
}
|
||||
source_paths = document.get("paths", {})
|
||||
filtered_paths: dict[str, Any] = {}
|
||||
|
||||
for declaration in declarations:
|
||||
path = f"/{declaration['path']}"
|
||||
method = declaration["method"].lower()
|
||||
source_path_item = source_paths[path]
|
||||
path_metadata = {key: value for key, value in source_path_item.items() if key not in OPENAPI_METHODS}
|
||||
filtered_path_item = filtered_paths.setdefault(path, path_metadata)
|
||||
filtered_operation = deepcopy(source_path_item[method])
|
||||
_rewrite_proxy_error_responses(filtered_operation, declaration["error_status_map"])
|
||||
filtered_path_item[method] = filtered_operation
|
||||
|
||||
filtered_document["paths"] = filtered_paths
|
||||
|
||||
source_components = document.get("components", {})
|
||||
filtered_components = {key: value for key, value in source_components.items() if key != "schemas"}
|
||||
source_schemas = source_components.get("schemas", {})
|
||||
available_schemas = {**source_schemas, CONSOLE_PROXY_ERROR_SCHEMA_NAME: CONSOLE_PROXY_ERROR_SCHEMA}
|
||||
schema_names = _referenced_schema_names(filtered_paths, available_schemas)
|
||||
filtered_components["schemas"] = {
|
||||
name: schema for name, schema in available_schemas.items() if name in schema_names
|
||||
}
|
||||
filtered_document["components"] = filtered_components
|
||||
return filtered_document
|
||||
|
||||
|
||||
def validate_error_status_map(operation_id: str, error_status_map: tuple[tuple[int, int], ...]) -> None:
|
||||
"""Validate the status normalization advertised by one Console operation."""
|
||||
upstream_statuses: set[int] = set()
|
||||
for upstream_status, console_status in error_status_map:
|
||||
if upstream_status in upstream_statuses:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has duplicate error status: {upstream_status}")
|
||||
if not 400 <= upstream_status <= 599 or not 400 <= console_status <= 599:
|
||||
raise ValueError(f"KnowledgeFS operation {operation_id} has invalid error status mapping")
|
||||
upstream_statuses.add(upstream_status)
|
||||
|
||||
|
||||
def _rewrite_proxy_error_responses(
|
||||
operation: dict[str, Any],
|
||||
error_status_map: tuple[tuple[int, int], ...],
|
||||
) -> None:
|
||||
responses = operation.setdefault("responses", {})
|
||||
proxy_error_response = {
|
||||
"description": "Error normalized by the Dify Console KnowledgeFS proxy.",
|
||||
"content": {
|
||||
"application/json": {"schema": {"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"}}
|
||||
},
|
||||
}
|
||||
for upstream_status, console_status in error_status_map:
|
||||
existing_target = responses.get(str(console_status)) if upstream_status != console_status else None
|
||||
responses.pop(str(upstream_status), None)
|
||||
normalized_response: dict[str, Any] = deepcopy(proxy_error_response)
|
||||
existing_schema = (
|
||||
existing_target.get("content", {}).get("application/json", {}).get("schema")
|
||||
if isinstance(existing_target, dict)
|
||||
else None
|
||||
)
|
||||
if existing_schema is not None:
|
||||
normalized_response["content"]["application/json"]["schema"] = {
|
||||
"oneOf": [
|
||||
deepcopy(existing_schema),
|
||||
{"$ref": f"#/components/schemas/{CONSOLE_PROXY_ERROR_SCHEMA_NAME}"},
|
||||
]
|
||||
}
|
||||
responses[str(console_status)] = normalized_response
|
||||
|
||||
|
||||
def _referenced_schema_names(value: Any, schemas: dict[str, Any]) -> set[str]:
|
||||
reference_prefix = "#/components/schemas/"
|
||||
selected: set[str] = set()
|
||||
pending: list[Any] = [value]
|
||||
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if isinstance(current, list):
|
||||
pending.extend(current)
|
||||
continue
|
||||
if not isinstance(current, dict):
|
||||
continue
|
||||
|
||||
reference = current.get("$ref")
|
||||
if isinstance(reference, str) and reference.startswith(reference_prefix):
|
||||
name = reference.removeprefix(reference_prefix)
|
||||
if name not in selected:
|
||||
if name not in schemas:
|
||||
raise ValueError(f"KnowledgeFS OpenAPI references missing schema: {name}")
|
||||
selected.add(name)
|
||||
pending.append(schemas[name])
|
||||
pending.extend(current.values())
|
||||
|
||||
return selected
|
||||
|
||||
|
||||
def console_contract_declarations() -> tuple[ContractDeclaration, ...]:
|
||||
"""Return transport declarations from the runtime Console operation registry."""
|
||||
from services.knowledge_fs_operations import KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
|
||||
return tuple(
|
||||
{
|
||||
"operation_id": operation.operation_id,
|
||||
"method": operation.method,
|
||||
"path": operation.path,
|
||||
"required_scope": operation.required_scope,
|
||||
"response_kind": operation.response_kind,
|
||||
"max_response_bytes": operation.max_response_bytes,
|
||||
"request_headers": operation.request_headers,
|
||||
"response_headers": operation.response_headers,
|
||||
"response_media_types": operation.response_media_types,
|
||||
"error_status_map": operation.error_status_map,
|
||||
}
|
||||
for operation in KNOWLEDGE_FS_CONSOLE_OPERATIONS
|
||||
)
|
||||
|
||||
|
||||
def contract_declarations_sha256(declarations: tuple[ContractDeclaration, ...]) -> str:
|
||||
"""Return a stable digest for the runtime Console operation declarations."""
|
||||
content = json.dumps(declarations, separators=(",", ":"), sort_keys=True).encode()
|
||||
return sha256(content)
|
||||
|
||||
|
||||
def response_kind(operation: dict[str, Any]) -> str:
|
||||
media_types = response_media_types(operation)
|
||||
if "text/event-stream" in media_types:
|
||||
return "stream"
|
||||
if "application/octet-stream" in media_types:
|
||||
return "binary"
|
||||
return "buffered"
|
||||
|
||||
|
||||
def response_media_types(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
media_types: set[str] = set()
|
||||
for status, response in operation.get("responses", {}).items():
|
||||
if status == "2XX" or (len(status) == 3 and status.startswith("2") and status.isdigit()):
|
||||
media_types.update(response.get("content", {}))
|
||||
return tuple(sorted(media_types))
|
||||
|
||||
|
||||
def required_scope(operation: dict[str, Any]) -> str | None:
|
||||
scope = operation.get("x-knowledge-fs-required-scope")
|
||||
if scope in ("knowledge-spaces:read", "knowledge-spaces:write"):
|
||||
return scope
|
||||
if operation.get("security") == []:
|
||||
return None
|
||||
raise ValueError(f"KnowledgeFS operation has no supported required scope: {scope}")
|
||||
|
||||
|
||||
def required_max_response_bytes(operation: dict[str, Any]) -> int:
|
||||
value = operation.get("x-knowledge-fs-max-response-bytes")
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
|
||||
raise ValueError(f"KnowledgeFS operation has no valid response byte limit: {value}")
|
||||
return value
|
||||
|
||||
|
||||
def request_header_names(path_item: dict[str, Any], operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
names: set[str] = set()
|
||||
for parameter in [*path_item.get("parameters", []), *operation.get("parameters", [])]:
|
||||
if "$ref" in parameter:
|
||||
raise ValueError(f"KnowledgeFS request header references are not supported: {parameter['$ref']}")
|
||||
if parameter.get("in") == "header":
|
||||
names.add(parameter["name"].lower())
|
||||
return tuple(sorted(names))
|
||||
|
||||
|
||||
def response_header_names(operation: dict[str, Any]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
{
|
||||
name.lower()
|
||||
for response in operation.get("responses", {}).values()
|
||||
for name in response.get("headers", {})
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def sha256(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
|
||||
def run(*command: str, cwd: Path) -> str:
|
||||
return subprocess.run(command, cwd=cwd, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -7,6 +7,9 @@ from .delete_tool_parameters_cache_when_sync_draft_workflow import (
|
||||
handle as handle_delete_tool_parameters_cache_when_sync_draft_workflow,
|
||||
)
|
||||
from .queue_credential_sync_when_tenant_created import handle as handle_queue_credential_sync_when_tenant_created
|
||||
from .queue_default_plugin_install_when_tenant_created import (
|
||||
handle as handle_queue_default_plugin_install_when_tenant_created,
|
||||
)
|
||||
from .sync_plugin_trigger_when_app_created import handle as handle_sync_plugin_trigger_when_app_created
|
||||
from .sync_webhook_when_app_created import handle as handle_sync_webhook_when_app_created
|
||||
from .sync_workflow_schedule_when_app_published import handle as handle_sync_workflow_schedule_when_app_published
|
||||
@@ -32,6 +35,7 @@ __all__ = [
|
||||
"handle_create_site_record_when_app_created",
|
||||
"handle_delete_tool_parameters_cache_when_sync_draft_workflow",
|
||||
"handle_queue_credential_sync_when_tenant_created",
|
||||
"handle_queue_default_plugin_install_when_tenant_created",
|
||||
"handle_sync_plugin_trigger_when_app_created",
|
||||
"handle_sync_webhook_when_app_created",
|
||||
"handle_sync_workflow_schedule_when_app_published",
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Queue default marketplace plugin installation after tenant creation."""
|
||||
|
||||
import logging
|
||||
|
||||
from configs import dify_config
|
||||
from events.tenant_event import tenant_was_created
|
||||
from tasks.install_default_plugins_task import install_default_plugins_task
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@tenant_was_created.connect
|
||||
def handle(sender, **kwargs) -> None:
|
||||
"""Keep tenant creation non-blocking while installing configured plugins asynchronously."""
|
||||
plugin_ids = dify_config.NEW_USER_DEFAULT_PLUGIN_ID_LIST
|
||||
if not plugin_ids:
|
||||
return
|
||||
|
||||
try:
|
||||
install_default_plugins_task.delay(sender.id, plugin_ids)
|
||||
except Exception:
|
||||
logger.exception("Failed to queue default plugin installation for tenant %s", sender.id)
|
||||
@@ -156,6 +156,8 @@ def init_app(app: DifyApp) -> Celery:
|
||||
"tasks.generate_summary_index_task", # summary index generation
|
||||
"tasks.regenerate_summary_index_task", # summary index regeneration
|
||||
"tasks.initialize_created_app_rbac_access_task", # app access initialization
|
||||
"tasks.install_default_plugins_task", # tenant default plugin installation
|
||||
"tasks.refresh_billing_vector_space_task", # billing vector-space cache refresh
|
||||
"tasks.app_generate.resume_agent_app_task", # ENG-635: Agent v2 chat ask_human resume
|
||||
"tasks.workflow_run_archive_download_tasks", # workflow-run archive download preparation
|
||||
]
|
||||
|
||||
@@ -119,6 +119,19 @@ class Storage:
|
||||
def delete(self, filename: str):
|
||||
return self.storage_runner.delete(filename)
|
||||
|
||||
def generate_presigned_url(
|
||||
self,
|
||||
filename: str,
|
||||
*,
|
||||
expires_in: int,
|
||||
content_type: str | None = None,
|
||||
) -> str:
|
||||
return self.storage_runner.generate_presigned_url(
|
||||
filename,
|
||||
expires_in=expires_in,
|
||||
content_type=content_type,
|
||||
)
|
||||
|
||||
def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
|
||||
return self.storage_runner.scan(path, files=files, directories=directories)
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ from core.repositories.sqlalchemy_workflow_execution_repository import SQLAlchem
|
||||
from extensions.logstore.aliyun_logstore import AliyunLogStore
|
||||
from graphon.entities import WorkflowExecution
|
||||
from graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter
|
||||
from libs.helper import extract_tenant_id
|
||||
from models import (
|
||||
Account,
|
||||
CreatorUserRole,
|
||||
@@ -27,6 +26,7 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowRunTriggeredFrom | None,
|
||||
@@ -36,7 +36,8 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for creating sessions
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (DEBUGGING or APP_RUN)
|
||||
"""
|
||||
@@ -47,10 +48,8 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
# Note: Project/logstore/index initialization is done at app startup via ext_logstore
|
||||
self.logstore_client = AliyunLogStore()
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
@@ -64,7 +63,13 @@ class LogstoreWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||
self._creator_user_role = CreatorUserRole.ACCOUNT if isinstance(user, Account) else CreatorUserRole.END_USER
|
||||
|
||||
# Initialize SQL repository for dual-write support
|
||||
self.sql_repository = SQLAlchemyWorkflowExecutionRepository(session_factory, user, app_id, triggered_from)
|
||||
self.sql_repository = SQLAlchemyWorkflowExecutionRepository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=tenant_id,
|
||||
user=user,
|
||||
app_id=app_id,
|
||||
triggered_from=triggered_from,
|
||||
)
|
||||
|
||||
# Control flag for dual-write (write to both LogStore and SQL database)
|
||||
# Set to True to enable dual-write for safe migration, False to use LogStore only
|
||||
|
||||
+11
-6
@@ -26,7 +26,6 @@ from graphon.entities import WorkflowNodeExecution
|
||||
from graphon.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
|
||||
from graphon.model_runtime.utils.encoders import jsonable_encoder
|
||||
from graphon.workflow_type_encoder import WorkflowRuntimeTypeConverter
|
||||
from libs.helper import extract_tenant_id
|
||||
from models import (
|
||||
Account,
|
||||
CreatorUserRole,
|
||||
@@ -109,6 +108,7 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
def __init__(
|
||||
self,
|
||||
session_factory: sessionmaker | Engine,
|
||||
tenant_id: str,
|
||||
user: Account | EndUser,
|
||||
app_id: str | None,
|
||||
triggered_from: WorkflowNodeExecutionTriggeredFrom | None,
|
||||
@@ -118,7 +118,8 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
|
||||
Args:
|
||||
session_factory: SQLAlchemy sessionmaker or engine for creating sessions
|
||||
user: Account or EndUser object containing tenant_id, user ID, and role information
|
||||
tenant_id: Tenant that owns the workflow node execution
|
||||
user: Account or EndUser used for creator attribution
|
||||
app_id: App ID for filtering by application (can be None)
|
||||
triggered_from: Source of the execution trigger (SINGLE_STEP or WORKFLOW_RUN)
|
||||
"""
|
||||
@@ -128,10 +129,8 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
# Initialize LogStore client
|
||||
self.logstore_client = AliyunLogStore()
|
||||
|
||||
# Extract tenant_id from user
|
||||
tenant_id = extract_tenant_id(user)
|
||||
if not tenant_id:
|
||||
raise ValueError("User must have a tenant_id or current_tenant_id")
|
||||
raise ValueError("tenant_id is required")
|
||||
self._tenant_id = tenant_id
|
||||
|
||||
# Store app context
|
||||
@@ -145,7 +144,13 @@ class LogstoreWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||
self._creator_user_role = CreatorUserRole.ACCOUNT if isinstance(user, Account) else CreatorUserRole.END_USER
|
||||
|
||||
# Initialize SQL repository for dual-write support
|
||||
self.sql_repository = SQLAlchemyWorkflowNodeExecutionRepository(session_factory, user, app_id, triggered_from)
|
||||
self.sql_repository = SQLAlchemyWorkflowNodeExecutionRepository(
|
||||
session_factory=session_factory,
|
||||
tenant_id=tenant_id,
|
||||
user=user,
|
||||
app_id=app_id,
|
||||
triggered_from=triggered_from,
|
||||
)
|
||||
|
||||
# Control flag for dual-write (write to both LogStore and SQL database)
|
||||
# Set to True to enable dual-write for safe migration, False to use LogStore only
|
||||
|
||||
@@ -92,3 +92,21 @@ class AwsS3Storage(BaseStorage):
|
||||
@override
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
@override
|
||||
def generate_presigned_url(
|
||||
self,
|
||||
filename: str,
|
||||
*,
|
||||
expires_in: int,
|
||||
content_type: str | None = None,
|
||||
) -> str:
|
||||
params = {"Bucket": self.bucket_name, "Key": filename}
|
||||
if content_type:
|
||||
params["ResponseContentType"] = content_type
|
||||
|
||||
return self.client.generate_presigned_url(
|
||||
"get_object",
|
||||
Params=params,
|
||||
ExpiresIn=expires_in,
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user