Merge remote-tracking branch 'origin/main' into feat/llm-node-support-tools

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

.claude/settings.json

@@ -0,0 +1,9 @@
+{
+  "enabledPlugins": {
+    "feature-dev@claude-plugins-official": true,
+    "context7@claude-plugins-official": true,
+    "typescript-lsp@claude-plugins-official": true,
+    "pyright-lsp@claude-plugins-official": true,
+    "ralph-wiggum@claude-plugins-official": true
+  }
+}

.claude/settings.json.example

@@ -1,19 +0,0 @@
-{
-    "permissions": {
-      "allow": [],
-      "deny": []
-    },
-    "env": {
-      "__comment": "Environment variables for MCP servers. Override in .claude/settings.local.json with actual values.",
-      "GITHUB_PERSONAL_ACCESS_TOKEN": "ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-    },
-    "enabledMcpjsonServers": [
-      "context7",
-      "sequential-thinking",
-      "github",
-      "fetch",
-      "playwright",
-      "ide"
-    ],
-    "enableAllProjectMcpServers": true
-  }

.claude/skills/component-refactoring/SKILL.md

@@ -0,0 +1,483 @@
+---
+name: component-refactoring
+description: Refactor high-complexity React components in Dify frontend. Use when `pnpm analyze-component --json` shows complexity > 50 or lineCount > 300, when the user asks for code splitting, hook extraction, or complexity reduction, or when `pnpm analyze-component` warns to refactor before testing; avoid for simple/well-structured components, third-party wrappers, or when the user explicitly wants testing without refactoring.
+---
+
+# Dify Component Refactoring Skill
+
+Refactor high-complexity React components in the Dify frontend codebase with the patterns and workflow below.
+
+> **Complexity Threshold**: Components with complexity > 50 (measured by `pnpm analyze-component`) should be refactored before testing.
+
+## Quick Reference
+
+### Commands (run from `web/`)
+
+Use paths relative to `web/` (e.g., `app/components/...`).
+Use `refactor-component` for refactoring prompts and `analyze-component` for testing prompts and metrics.
+
+```bash
+cd web
+
+# Generate refactoring prompt
+pnpm refactor-component <path>
+
+# Output refactoring analysis as JSON
+pnpm refactor-component <path> --json
+
+# Generate testing prompt (after refactoring)
+pnpm analyze-component <path>
+
+# Output testing analysis as JSON
+pnpm analyze-component <path> --json
+```
+
+### Complexity Analysis
+
+```bash
+# Analyze component complexity
+pnpm analyze-component <path> --json
+
+# Key metrics to check:
+# - complexity: normalized score 0-100 (target < 50)
+# - maxComplexity: highest single function complexity
+# - lineCount: total lines (target < 300)
+```
+
+### Complexity Score Interpretation
+
+| Score | Level | Action |
+|-------|-------|--------|
+| 0-25 | 🟢 Simple | Ready for testing |
+| 26-50 | 🟡 Medium | Consider minor refactoring |
+| 51-75 | 🟠 Complex | **Refactor before testing** |
+| 76-100 | 🔴 Very Complex | **Must refactor** |
+
+## Core Refactoring Patterns
+
+### Pattern 1: Extract Custom Hooks
+
+**When**: Component has complex state management, multiple `useState`/`useEffect`, or business logic mixed with UI.
+
+**Dify Convention**: Place hooks in a `hooks/` subdirectory or alongside the component as `use-<feature>.ts`.
+
+```typescript
+// ❌ Before: Complex state logic in component
+const Configuration: FC = () => {
+  const [modelConfig, setModelConfig] = useState<ModelConfig>(...)
+  const [datasetConfigs, setDatasetConfigs] = useState<DatasetConfigs>(...)
+  const [completionParams, setCompletionParams] = useState<FormValue>({})
+  
+  // 50+ lines of state management logic...
+  
+  return <div>...</div>
+}
+
+// ✅ After: Extract to custom hook
+// hooks/use-model-config.ts
+export const useModelConfig = (appId: string) => {
+  const [modelConfig, setModelConfig] = useState<ModelConfig>(...)
+  const [completionParams, setCompletionParams] = useState<FormValue>({})
+  
+  // Related state management logic here
+  
+  return { modelConfig, setModelConfig, completionParams, setCompletionParams }
+}
+
+// Component becomes cleaner
+const Configuration: FC = () => {
+  const { modelConfig, setModelConfig } = useModelConfig(appId)
+  return <div>...</div>
+}
+```
+
+**Dify Examples**:
+- `web/app/components/app/configuration/hooks/use-advanced-prompt-config.ts`
+- `web/app/components/app/configuration/debug/hooks.tsx`
+- `web/app/components/workflow/hooks/use-workflow.ts`
+
+### Pattern 2: Extract Sub-Components
+
+**When**: Single component has multiple UI sections, conditional rendering blocks, or repeated patterns.
+
+**Dify Convention**: Place sub-components in subdirectories or as separate files in the same directory.
+
+```typescript
+// ❌ Before: Monolithic JSX with multiple sections
+const AppInfo = () => {
+  return (
+    <div>
+      {/* 100 lines of header UI */}
+      {/* 100 lines of operations UI */}
+      {/* 100 lines of modals */}
+    </div>
+  )
+}
+
+// ✅ After: Split into focused components
+// app-info/
+//   ├── index.tsx           (orchestration only)
+//   ├── app-header.tsx      (header UI)
+//   ├── app-operations.tsx  (operations UI)
+//   └── app-modals.tsx      (modal management)
+
+const AppInfo = () => {
+  const { showModal, setShowModal } = useAppInfoModals()
+  
+  return (
+    <div>
+      <AppHeader appDetail={appDetail} />
+      <AppOperations onAction={handleAction} />
+      <AppModals show={showModal} onClose={() => setShowModal(null)} />
+    </div>
+  )
+}
+```
+
+**Dify Examples**:
+- `web/app/components/app/configuration/` directory structure
+- `web/app/components/workflow/nodes/` per-node organization
+
+### Pattern 3: Simplify Conditional Logic
+
+**When**: Deep nesting (> 3 levels), complex ternaries, or multiple `if/else` chains.
+
+```typescript
+// ❌ Before: Deeply nested conditionals
+const Template = useMemo(() => {
+  if (appDetail?.mode === AppModeEnum.CHAT) {
+    switch (locale) {
+      case LanguagesSupported[1]:
+        return <TemplateChatZh />
+      case LanguagesSupported[7]:
+        return <TemplateChatJa />
+      default:
+        return <TemplateChatEn />
+    }
+  }
+  if (appDetail?.mode === AppModeEnum.ADVANCED_CHAT) {
+    // Another 15 lines...
+  }
+  // More conditions...
+}, [appDetail, locale])
+
+// ✅ After: Use lookup tables + early returns
+const TEMPLATE_MAP = {
+  [AppModeEnum.CHAT]: {
+    [LanguagesSupported[1]]: TemplateChatZh,
+    [LanguagesSupported[7]]: TemplateChatJa,
+    default: TemplateChatEn,
+  },
+  [AppModeEnum.ADVANCED_CHAT]: {
+    [LanguagesSupported[1]]: TemplateAdvancedChatZh,
+    // ...
+  },
+}
+
+const Template = useMemo(() => {
+  const modeTemplates = TEMPLATE_MAP[appDetail?.mode]
+  if (!modeTemplates) return null
+  
+  const TemplateComponent = modeTemplates[locale] || modeTemplates.default
+  return <TemplateComponent appDetail={appDetail} />
+}, [appDetail, locale])
+```
+
+### Pattern 4: Extract API/Data Logic
+
+**When**: Component directly handles API calls, data transformation, or complex async operations.
+
+**Dify Convention**: Use `@tanstack/react-query` hooks from `web/service/use-*.ts` or create custom data hooks.
+
+```typescript
+// ❌ Before: API logic in component
+const MCPServiceCard = () => {
+  const [basicAppConfig, setBasicAppConfig] = useState({})
+  
+  useEffect(() => {
+    if (isBasicApp && appId) {
+      (async () => {
+        const res = await fetchAppDetail({ url: '/apps', id: appId })
+        setBasicAppConfig(res?.model_config || {})
+      })()
+    }
+  }, [appId, isBasicApp])
+  
+  // More API-related logic...
+}
+
+// ✅ After: Extract to data hook using React Query
+// use-app-config.ts
+import { useQuery } from '@tanstack/react-query'
+import { get } from '@/service/base'
+
+const NAME_SPACE = 'appConfig'
+
+export const useAppConfig = (appId: string, isBasicApp: boolean) => {
+  return useQuery({
+    enabled: isBasicApp && !!appId,
+    queryKey: [NAME_SPACE, 'detail', appId],
+    queryFn: () => get<AppDetailResponse>(`/apps/${appId}`),
+    select: data => data?.model_config || {},
+  })
+}
+
+// Component becomes cleaner
+const MCPServiceCard = () => {
+  const { data: config, isLoading } = useAppConfig(appId, isBasicApp)
+  // UI only
+}
+```
+
+**React Query Best Practices in Dify**:
+- Define `NAME_SPACE` for query key organization
+- Use `enabled` option for conditional fetching
+- Use `select` for data transformation
+- Export invalidation hooks: `useInvalidXxx`
+
+**Dify Examples**:
+- `web/service/use-workflow.ts`
+- `web/service/use-common.ts`
+- `web/service/knowledge/use-dataset.ts`
+- `web/service/knowledge/use-document.ts`
+
+### Pattern 5: Extract Modal/Dialog Management
+
+**When**: Component manages multiple modals with complex open/close states.
+
+**Dify Convention**: Modals should be extracted with their state management.
+
+```typescript
+// ❌ Before: Multiple modal states in component
+const AppInfo = () => {
+  const [showEditModal, setShowEditModal] = useState(false)
+  const [showDuplicateModal, setShowDuplicateModal] = useState(false)
+  const [showConfirmDelete, setShowConfirmDelete] = useState(false)
+  const [showSwitchModal, setShowSwitchModal] = useState(false)
+  const [showImportDSLModal, setShowImportDSLModal] = useState(false)
+  // 5+ more modal states...
+}
+
+// ✅ After: Extract to modal management hook
+type ModalType = 'edit' | 'duplicate' | 'delete' | 'switch' | 'import' | null
+
+const useAppInfoModals = () => {
+  const [activeModal, setActiveModal] = useState<ModalType>(null)
+  
+  const openModal = useCallback((type: ModalType) => setActiveModal(type), [])
+  const closeModal = useCallback(() => setActiveModal(null), [])
+  
+  return {
+    activeModal,
+    openModal,
+    closeModal,
+    isOpen: (type: ModalType) => activeModal === type,
+  }
+}
+```
+
+### Pattern 6: Extract Form Logic
+
+**When**: Complex form validation, submission handling, or field transformation.
+
+**Dify Convention**: Use `@tanstack/react-form` patterns from `web/app/components/base/form/`.
+
+```typescript
+// ✅ Use existing form infrastructure
+import { useAppForm } from '@/app/components/base/form'
+
+const ConfigForm = () => {
+  const form = useAppForm({
+    defaultValues: { name: '', description: '' },
+    onSubmit: handleSubmit,
+  })
+  
+  return <form.Provider>...</form.Provider>
+}
+```
+
+## Dify-Specific Refactoring Guidelines
+
+### 1. Context Provider Extraction
+
+**When**: Component provides complex context values with multiple states.
+
+```typescript
+// ❌ Before: Large context value object
+const value = {
+  appId, isAPIKeySet, isTrailFinished, mode, modelModeType,
+  promptMode, isAdvancedMode, isAgent, isOpenAI, isFunctionCall,
+  // 50+ more properties...
+}
+return <ConfigContext.Provider value={value}>...</ConfigContext.Provider>
+
+// ✅ After: Split into domain-specific contexts
+<ModelConfigProvider value={modelConfigValue}>
+  <DatasetConfigProvider value={datasetConfigValue}>
+    <UIConfigProvider value={uiConfigValue}>
+      {children}
+    </UIConfigProvider>
+  </DatasetConfigProvider>
+</ModelConfigProvider>
+```
+
+**Dify Reference**: `web/context/` directory structure
+
+### 2. Workflow Node Components
+
+**When**: Refactoring workflow node components (`web/app/components/workflow/nodes/`).
+
+**Conventions**:
+- Keep node logic in `use-interactions.ts`
+- Extract panel UI to separate files
+- Use `_base` components for common patterns
+
+```
+nodes/<node-type>/
+  ├── index.tsx              # Node registration
+  ├── node.tsx               # Node visual component
+  ├── panel.tsx              # Configuration panel
+  ├── use-interactions.ts    # Node-specific hooks
+  └── types.ts               # Type definitions
+```
+
+### 3. Configuration Components
+
+**When**: Refactoring app configuration components.
+
+**Conventions**:
+- Separate config sections into subdirectories
+- Use existing patterns from `web/app/components/app/configuration/`
+- Keep feature toggles in dedicated components
+
+### 4. Tool/Plugin Components
+
+**When**: Refactoring tool-related components (`web/app/components/tools/`).
+
+**Conventions**:
+- Follow existing modal patterns
+- Use service hooks from `web/service/use-tools.ts`
+- Keep provider-specific logic isolated
+
+## Refactoring Workflow
+
+### Step 1: Generate Refactoring Prompt
+
+```bash
+pnpm refactor-component <path>
+```
+
+This command will:
+- Analyze component complexity and features
+- Identify specific refactoring actions needed
+- Generate a prompt for AI assistant (auto-copied to clipboard on macOS)
+- Provide detailed requirements based on detected patterns
+
+### Step 2: Analyze Details
+
+```bash
+pnpm analyze-component <path> --json
+```
+
+Identify:
+- Total complexity score
+- Max function complexity
+- Line count
+- Features detected (state, effects, API, etc.)
+
+### Step 3: Plan
+
+Create a refactoring plan based on detected features:
+
+| Detected Feature | Refactoring Action |
+|------------------|-------------------|
+| `hasState: true` + `hasEffects: true` | Extract custom hook |
+| `hasAPI: true` | Extract data/service hook |
+| `hasEvents: true` (many) | Extract event handlers |
+| `lineCount > 300` | Split into sub-components |
+| `maxComplexity > 50` | Simplify conditional logic |
+
+### Step 4: Execute Incrementally
+
+1. **Extract one piece at a time**
+2. **Run lint, type-check, and tests after each extraction**
+3. **Verify functionality before next step**
+
+```
+For each extraction:
+  ┌────────────────────────────────────────┐
+  │ 1. Extract code                        │
+  │ 2. Run: pnpm lint:fix                  │
+  │ 3. Run: pnpm type-check:tsgo           │
+  │ 4. Run: pnpm test                      │
+  │ 5. Test functionality manually         │
+  │ 6. PASS? → Next extraction             │
+  │    FAIL? → Fix before continuing       │
+  └────────────────────────────────────────┘
+```
+
+### Step 5: Verify
+
+After refactoring:
+
+```bash
+# Re-run refactor command to verify improvements
+pnpm refactor-component <path>
+
+# If complexity < 25 and lines < 200, you'll see:
+# ✅ COMPONENT IS WELL-STRUCTURED
+
+# For detailed metrics:
+pnpm analyze-component <path> --json
+
+# Target metrics:
+# - complexity < 50
+# - lineCount < 300
+# - maxComplexity < 30
+```
+
+## Common Mistakes to Avoid
+
+### ❌ Over-Engineering
+
+```typescript
+// ❌ Too many tiny hooks
+const useButtonText = () => useState('Click')
+const useButtonDisabled = () => useState(false)
+const useButtonLoading = () => useState(false)
+
+// ✅ Cohesive hook with related state
+const useButtonState = () => {
+  const [text, setText] = useState('Click')
+  const [disabled, setDisabled] = useState(false)
+  const [loading, setLoading] = useState(false)
+  return { text, setText, disabled, setDisabled, loading, setLoading }
+}
+```
+
+### ❌ Breaking Existing Patterns
+
+- Follow existing directory structures
+- Maintain naming conventions
+- Preserve export patterns for compatibility
+
+### ❌ Premature Abstraction
+
+- Only extract when there's clear complexity benefit
+- Don't create abstractions for single-use code
+- Keep refactored code in the same domain area
+
+## References
+
+### Dify Codebase Examples
+
+- **Hook extraction**: `web/app/components/app/configuration/hooks/`
+- **Component splitting**: `web/app/components/app/configuration/`
+- **Service hooks**: `web/service/use-*.ts`
+- **Workflow patterns**: `web/app/components/workflow/hooks/`
+- **Form patterns**: `web/app/components/base/form/`
+
+### Related Skills
+
+- `frontend-testing` - For testing refactored components
+- `web/testing/testing.md` - Testing specification

.claude/skills/component-refactoring/references/complexity-patterns.md

@@ -0,0 +1,493 @@
+# Complexity Reduction Patterns
+
+This document provides patterns for reducing cognitive complexity in Dify React components.
+
+## Understanding Complexity
+
+### SonarJS Cognitive Complexity
+
+The `pnpm analyze-component` tool uses SonarJS cognitive complexity metrics:
+
+- **Total Complexity**: Sum of all functions' complexity in the file
+- **Max Complexity**: Highest single function complexity
+
+### What Increases Complexity
+
+| Pattern | Complexity Impact |
+|---------|-------------------|
+| `if/else` | +1 per branch |
+| Nested conditions | +1 per nesting level |
+| `switch/case` | +1 per case |
+| `for/while/do` | +1 per loop |
+| `&&`/`||` chains | +1 per operator |
+| Nested callbacks | +1 per nesting level |
+| `try/catch` | +1 per catch |
+| Ternary expressions | +1 per nesting |
+
+## Pattern 1: Replace Conditionals with Lookup Tables
+
+**Before** (complexity: ~15):
+
+```typescript
+const Template = useMemo(() => {
+  if (appDetail?.mode === AppModeEnum.CHAT) {
+    switch (locale) {
+      case LanguagesSupported[1]:
+        return <TemplateChatZh appDetail={appDetail} />
+      case LanguagesSupported[7]:
+        return <TemplateChatJa appDetail={appDetail} />
+      default:
+        return <TemplateChatEn appDetail={appDetail} />
+    }
+  }
+  if (appDetail?.mode === AppModeEnum.ADVANCED_CHAT) {
+    switch (locale) {
+      case LanguagesSupported[1]:
+        return <TemplateAdvancedChatZh appDetail={appDetail} />
+      case LanguagesSupported[7]:
+        return <TemplateAdvancedChatJa appDetail={appDetail} />
+      default:
+        return <TemplateAdvancedChatEn appDetail={appDetail} />
+    }
+  }
+  if (appDetail?.mode === AppModeEnum.WORKFLOW) {
+    // Similar pattern...
+  }
+  return null
+}, [appDetail, locale])
+```
+
+**After** (complexity: ~3):
+
+```typescript
+// Define lookup table outside component
+const TEMPLATE_MAP: Record<AppModeEnum, Record<string, FC<TemplateProps>>> = {
+  [AppModeEnum.CHAT]: {
+    [LanguagesSupported[1]]: TemplateChatZh,
+    [LanguagesSupported[7]]: TemplateChatJa,
+    default: TemplateChatEn,
+  },
+  [AppModeEnum.ADVANCED_CHAT]: {
+    [LanguagesSupported[1]]: TemplateAdvancedChatZh,
+    [LanguagesSupported[7]]: TemplateAdvancedChatJa,
+    default: TemplateAdvancedChatEn,
+  },
+  [AppModeEnum.WORKFLOW]: {
+    [LanguagesSupported[1]]: TemplateWorkflowZh,
+    [LanguagesSupported[7]]: TemplateWorkflowJa,
+    default: TemplateWorkflowEn,
+  },
+  // ...
+}
+
+// Clean component logic
+const Template = useMemo(() => {
+  if (!appDetail?.mode) return null
+  
+  const templates = TEMPLATE_MAP[appDetail.mode]
+  if (!templates) return null
+  
+  const TemplateComponent = templates[locale] ?? templates.default
+  return <TemplateComponent appDetail={appDetail} />
+}, [appDetail, locale])
+```
+
+## Pattern 2: Use Early Returns
+
+**Before** (complexity: ~10):
+
+```typescript
+const handleSubmit = () => {
+  if (isValid) {
+    if (hasChanges) {
+      if (isConnected) {
+        submitData()
+      } else {
+        showConnectionError()
+      }
+    } else {
+      showNoChangesMessage()
+    }
+  } else {
+    showValidationError()
+  }
+}
+```
+
+**After** (complexity: ~4):
+
+```typescript
+const handleSubmit = () => {
+  if (!isValid) {
+    showValidationError()
+    return
+  }
+  
+  if (!hasChanges) {
+    showNoChangesMessage()
+    return
+  }
+  
+  if (!isConnected) {
+    showConnectionError()
+    return
+  }
+  
+  submitData()
+}
+```
+
+## Pattern 3: Extract Complex Conditions
+
+**Before** (complexity: high):
+
+```typescript
+const canPublish = (() => {
+  if (mode !== AppModeEnum.COMPLETION) {
+    if (!isAdvancedMode)
+      return true
+
+    if (modelModeType === ModelModeType.completion) {
+      if (!hasSetBlockStatus.history || !hasSetBlockStatus.query)
+        return false
+      return true
+    }
+    return true
+  }
+  return !promptEmpty
+})()
+```
+
+**After** (complexity: lower):
+
+```typescript
+// Extract to named functions
+const canPublishInCompletionMode = () => !promptEmpty
+
+const canPublishInChatMode = () => {
+  if (!isAdvancedMode) return true
+  if (modelModeType !== ModelModeType.completion) return true
+  return hasSetBlockStatus.history && hasSetBlockStatus.query
+}
+
+// Clean main logic
+const canPublish = mode === AppModeEnum.COMPLETION
+  ? canPublishInCompletionMode()
+  : canPublishInChatMode()
+```
+
+## Pattern 4: Replace Chained Ternaries
+
+**Before** (complexity: ~5):
+
+```typescript
+const statusText = serverActivated
+  ? t('status.running')
+  : serverPublished
+    ? t('status.inactive')
+    : appUnpublished
+      ? t('status.unpublished')
+      : t('status.notConfigured')
+```
+
+**After** (complexity: ~2):
+
+```typescript
+const getStatusText = () => {
+  if (serverActivated) return t('status.running')
+  if (serverPublished) return t('status.inactive')
+  if (appUnpublished) return t('status.unpublished')
+  return t('status.notConfigured')
+}
+
+const statusText = getStatusText()
+```
+
+Or use lookup:
+
+```typescript
+const STATUS_TEXT_MAP = {
+  running: 'status.running',
+  inactive: 'status.inactive',
+  unpublished: 'status.unpublished',
+  notConfigured: 'status.notConfigured',
+} as const
+
+const getStatusKey = (): keyof typeof STATUS_TEXT_MAP => {
+  if (serverActivated) return 'running'
+  if (serverPublished) return 'inactive'
+  if (appUnpublished) return 'unpublished'
+  return 'notConfigured'
+}
+
+const statusText = t(STATUS_TEXT_MAP[getStatusKey()])
+```
+
+## Pattern 5: Flatten Nested Loops
+
+**Before** (complexity: high):
+
+```typescript
+const processData = (items: Item[]) => {
+  const results: ProcessedItem[] = []
+  
+  for (const item of items) {
+    if (item.isValid) {
+      for (const child of item.children) {
+        if (child.isActive) {
+          for (const prop of child.properties) {
+            if (prop.value !== null) {
+              results.push({
+                itemId: item.id,
+                childId: child.id,
+                propValue: prop.value,
+              })
+            }
+          }
+        }
+      }
+    }
+  }
+  
+  return results
+}
+```
+
+**After** (complexity: lower):
+
+```typescript
+// Use functional approach
+const processData = (items: Item[]) => {
+  return items
+    .filter(item => item.isValid)
+    .flatMap(item =>
+      item.children
+        .filter(child => child.isActive)
+        .flatMap(child =>
+          child.properties
+            .filter(prop => prop.value !== null)
+            .map(prop => ({
+              itemId: item.id,
+              childId: child.id,
+              propValue: prop.value,
+            }))
+        )
+    )
+}
+```
+
+## Pattern 6: Extract Event Handler Logic
+
+**Before** (complexity: high in component):
+
+```typescript
+const Component = () => {
+  const handleSelect = (data: DataSet[]) => {
+    if (isEqual(data.map(item => item.id), dataSets.map(item => item.id))) {
+      hideSelectDataSet()
+      return
+    }
+
+    formattingChangedDispatcher()
+    let newDatasets = data
+    if (data.find(item => !item.name)) {
+      const newSelected = produce(data, (draft) => {
+        data.forEach((item, index) => {
+          if (!item.name) {
+            const newItem = dataSets.find(i => i.id === item.id)
+            if (newItem)
+              draft[index] = newItem
+          }
+        })
+      })
+      setDataSets(newSelected)
+      newDatasets = newSelected
+    }
+    else {
+      setDataSets(data)
+    }
+    hideSelectDataSet()
+    
+    // 40 more lines of logic...
+  }
+  
+  return <div>...</div>
+}
+```
+
+**After** (complexity: lower):
+
+```typescript
+// Extract to hook or utility
+const useDatasetSelection = (dataSets: DataSet[], setDataSets: SetState<DataSet[]>) => {
+  const normalizeSelection = (data: DataSet[]) => {
+    const hasUnloadedItem = data.some(item => !item.name)
+    if (!hasUnloadedItem) return data
+    
+    return produce(data, (draft) => {
+      data.forEach((item, index) => {
+        if (!item.name) {
+          const existing = dataSets.find(i => i.id === item.id)
+          if (existing) draft[index] = existing
+        }
+      })
+    })
+  }
+  
+  const hasSelectionChanged = (newData: DataSet[]) => {
+    return !isEqual(
+      newData.map(item => item.id),
+      dataSets.map(item => item.id)
+    )
+  }
+  
+  return { normalizeSelection, hasSelectionChanged }
+}
+
+// Component becomes cleaner
+const Component = () => {
+  const { normalizeSelection, hasSelectionChanged } = useDatasetSelection(dataSets, setDataSets)
+  
+  const handleSelect = (data: DataSet[]) => {
+    if (!hasSelectionChanged(data)) {
+      hideSelectDataSet()
+      return
+    }
+    
+    formattingChangedDispatcher()
+    const normalized = normalizeSelection(data)
+    setDataSets(normalized)
+    hideSelectDataSet()
+  }
+  
+  return <div>...</div>
+}
+```
+
+## Pattern 7: Reduce Boolean Logic Complexity
+
+**Before** (complexity: ~8):
+
+```typescript
+const toggleDisabled = hasInsufficientPermissions
+  || appUnpublished
+  || missingStartNode
+  || triggerModeDisabled
+  || (isAdvancedApp && !currentWorkflow?.graph)
+  || (isBasicApp && !basicAppConfig.updated_at)
+```
+
+**After** (complexity: ~3):
+
+```typescript
+// Extract meaningful boolean functions
+const isAppReady = () => {
+  if (isAdvancedApp) return !!currentWorkflow?.graph
+  return !!basicAppConfig.updated_at
+}
+
+const hasRequiredPermissions = () => {
+  return isCurrentWorkspaceEditor && !hasInsufficientPermissions
+}
+
+const canToggle = () => {
+  if (!hasRequiredPermissions()) return false
+  if (!isAppReady()) return false
+  if (missingStartNode) return false
+  if (triggerModeDisabled) return false
+  return true
+}
+
+const toggleDisabled = !canToggle()
+```
+
+## Pattern 8: Simplify useMemo/useCallback Dependencies
+
+**Before** (complexity: multiple recalculations):
+
+```typescript
+const payload = useMemo(() => {
+  let parameters: Parameter[] = []
+  let outputParameters: OutputParameter[] = []
+
+  if (!published) {
+    parameters = (inputs || []).map((item) => ({
+      name: item.variable,
+      description: '',
+      form: 'llm',
+      required: item.required,
+      type: item.type,
+    }))
+    outputParameters = (outputs || []).map((item) => ({
+      name: item.variable,
+      description: '',
+      type: item.value_type,
+    }))
+  }
+  else if (detail && detail.tool) {
+    parameters = (inputs || []).map((item) => ({
+      // Complex transformation...
+    }))
+    outputParameters = (outputs || []).map((item) => ({
+      // Complex transformation...
+    }))
+  }
+  
+  return {
+    icon: detail?.icon || icon,
+    label: detail?.label || name,
+    // ...more fields
+  }
+}, [detail, published, workflowAppId, icon, name, description, inputs, outputs])
+```
+
+**After** (complexity: separated concerns):
+
+```typescript
+// Separate transformations
+const useParameterTransform = (inputs: InputVar[], detail?: ToolDetail, published?: boolean) => {
+  return useMemo(() => {
+    if (!published) {
+      return inputs.map(item => ({
+        name: item.variable,
+        description: '',
+        form: 'llm',
+        required: item.required,
+        type: item.type,
+      }))
+    }
+    
+    if (!detail?.tool) return []
+    
+    return inputs.map(item => ({
+      name: item.variable,
+      required: item.required,
+      type: item.type === 'paragraph' ? 'string' : item.type,
+      description: detail.tool.parameters.find(p => p.name === item.variable)?.llm_description || '',
+      form: detail.tool.parameters.find(p => p.name === item.variable)?.form || 'llm',
+    }))
+  }, [inputs, detail, published])
+}
+
+// Component uses hook
+const parameters = useParameterTransform(inputs, detail, published)
+const outputParameters = useOutputTransform(outputs, detail, published)
+
+const payload = useMemo(() => ({
+  icon: detail?.icon || icon,
+  label: detail?.label || name,
+  parameters,
+  outputParameters,
+  // ...
+}), [detail, icon, name, parameters, outputParameters])
+```
+
+## Target Metrics After Refactoring
+
+| Metric | Target |
+|--------|--------|
+| Total Complexity | < 50 |
+| Max Function Complexity | < 30 |
+| Function Length | < 30 lines |
+| Nesting Depth | ≤ 3 levels |
+| Conditional Chains | ≤ 3 conditions |

.claude/skills/component-refactoring/references/component-splitting.md

@@ -0,0 +1,477 @@
+# Component Splitting Patterns
+
+This document provides detailed guidance on splitting large components into smaller, focused components in Dify.
+
+## When to Split Components
+
+Split a component when you identify:
+
+1. **Multiple UI sections** - Distinct visual areas with minimal coupling that can be composed independently
+1. **Conditional rendering blocks** - Large `{condition && <JSX />}` blocks
+1. **Repeated patterns** - Similar UI structures used multiple times
+1. **300+ lines** - Component exceeds manageable size
+1. **Modal clusters** - Multiple modals rendered in one component
+
+## Splitting Strategies
+
+### Strategy 1: Section-Based Splitting
+
+Identify visual sections and extract each as a component.
+
+```typescript
+// ❌ Before: Monolithic component (500+ lines)
+const ConfigurationPage = () => {
+  return (
+    <div>
+      {/* Header Section - 50 lines */}
+      <div className="header">
+        <h1>{t('configuration.title')}</h1>
+        <div className="actions">
+          {isAdvancedMode && <Badge>Advanced</Badge>}
+          <ModelParameterModal ... />
+          <AppPublisher ... />
+        </div>
+      </div>
+      
+      {/* Config Section - 200 lines */}
+      <div className="config">
+        <Config />
+      </div>
+      
+      {/* Debug Section - 150 lines */}
+      <div className="debug">
+        <Debug ... />
+      </div>
+      
+      {/* Modals Section - 100 lines */}
+      {showSelectDataSet && <SelectDataSet ... />}
+      {showHistoryModal && <EditHistoryModal ... />}
+      {showUseGPT4Confirm && <Confirm ... />}
+    </div>
+  )
+}
+
+// ✅ After: Split into focused components
+// configuration/
+//   ├── index.tsx              (orchestration)
+//   ├── configuration-header.tsx
+//   ├── configuration-content.tsx
+//   ├── configuration-debug.tsx
+//   └── configuration-modals.tsx
+
+// configuration-header.tsx
+interface ConfigurationHeaderProps {
+  isAdvancedMode: boolean
+  onPublish: () => void
+}
+
+const ConfigurationHeader: FC<ConfigurationHeaderProps> = ({
+  isAdvancedMode,
+  onPublish,
+}) => {
+  const { t } = useTranslation()
+  
+  return (
+    <div className="header">
+      <h1>{t('configuration.title')}</h1>
+      <div className="actions">
+        {isAdvancedMode && <Badge>Advanced</Badge>}
+        <ModelParameterModal ... />
+        <AppPublisher onPublish={onPublish} />
+      </div>
+    </div>
+  )
+}
+
+// index.tsx (orchestration only)
+const ConfigurationPage = () => {
+  const { modelConfig, setModelConfig } = useModelConfig()
+  const { activeModal, openModal, closeModal } = useModalState()
+  
+  return (
+    <div>
+      <ConfigurationHeader
+        isAdvancedMode={isAdvancedMode}
+        onPublish={handlePublish}
+      />
+      <ConfigurationContent
+        modelConfig={modelConfig}
+        onConfigChange={setModelConfig}
+      />
+      {!isMobile && (
+        <ConfigurationDebug
+          inputs={inputs}
+          onSetting={handleSetting}
+        />
+      )}
+      <ConfigurationModals
+        activeModal={activeModal}
+        onClose={closeModal}
+      />
+    </div>
+  )
+}
+```
+
+### Strategy 2: Conditional Block Extraction
+
+Extract large conditional rendering blocks.
+
+```typescript
+// ❌ Before: Large conditional blocks
+const AppInfo = () => {
+  return (
+    <div>
+      {expand ? (
+        <div className="expanded">
+          {/* 100 lines of expanded view */}
+        </div>
+      ) : (
+        <div className="collapsed">
+          {/* 50 lines of collapsed view */}
+        </div>
+      )}
+    </div>
+  )
+}
+
+// ✅ After: Separate view components
+const AppInfoExpanded: FC<AppInfoViewProps> = ({ appDetail, onAction }) => {
+  return (
+    <div className="expanded">
+      {/* Clean, focused expanded view */}
+    </div>
+  )
+}
+
+const AppInfoCollapsed: FC<AppInfoViewProps> = ({ appDetail, onAction }) => {
+  return (
+    <div className="collapsed">
+      {/* Clean, focused collapsed view */}
+    </div>
+  )
+}
+
+const AppInfo = () => {
+  return (
+    <div>
+      {expand
+        ? <AppInfoExpanded appDetail={appDetail} onAction={handleAction} />
+        : <AppInfoCollapsed appDetail={appDetail} onAction={handleAction} />
+      }
+    </div>
+  )
+}
+```
+
+### Strategy 3: Modal Extraction
+
+Extract modals with their trigger logic.
+
+```typescript
+// ❌ Before: Multiple modals in one component
+const AppInfo = () => {
+  const [showEdit, setShowEdit] = useState(false)
+  const [showDuplicate, setShowDuplicate] = useState(false)
+  const [showDelete, setShowDelete] = useState(false)
+  const [showSwitch, setShowSwitch] = useState(false)
+  
+  const onEdit = async (data) => { /* 20 lines */ }
+  const onDuplicate = async (data) => { /* 20 lines */ }
+  const onDelete = async () => { /* 15 lines */ }
+  
+  return (
+    <div>
+      {/* Main content */}
+      
+      {showEdit && <EditModal onConfirm={onEdit} onClose={() => setShowEdit(false)} />}
+      {showDuplicate && <DuplicateModal onConfirm={onDuplicate} onClose={() => setShowDuplicate(false)} />}
+      {showDelete && <DeleteConfirm onConfirm={onDelete} onClose={() => setShowDelete(false)} />}
+      {showSwitch && <SwitchModal ... />}
+    </div>
+  )
+}
+
+// ✅ After: Modal manager component
+// app-info-modals.tsx
+type ModalType = 'edit' | 'duplicate' | 'delete' | 'switch' | null
+
+interface AppInfoModalsProps {
+  appDetail: AppDetail
+  activeModal: ModalType
+  onClose: () => void
+  onSuccess: () => void
+}
+
+const AppInfoModals: FC<AppInfoModalsProps> = ({
+  appDetail,
+  activeModal,
+  onClose,
+  onSuccess,
+}) => {
+  const handleEdit = async (data) => { /* logic */ }
+  const handleDuplicate = async (data) => { /* logic */ }
+  const handleDelete = async () => { /* logic */ }
+
+  return (
+    <>
+      {activeModal === 'edit' && (
+        <EditModal
+          appDetail={appDetail}
+          onConfirm={handleEdit}
+          onClose={onClose}
+        />
+      )}
+      {activeModal === 'duplicate' && (
+        <DuplicateModal
+          appDetail={appDetail}
+          onConfirm={handleDuplicate}
+          onClose={onClose}
+        />
+      )}
+      {activeModal === 'delete' && (
+        <DeleteConfirm
+          onConfirm={handleDelete}
+          onClose={onClose}
+        />
+      )}
+      {activeModal === 'switch' && (
+        <SwitchModal
+          appDetail={appDetail}
+          onClose={onClose}
+        />
+      )}
+    </>
+  )
+}
+
+// Parent component
+const AppInfo = () => {
+  const { activeModal, openModal, closeModal } = useModalState()
+  
+  return (
+    <div>
+      {/* Main content with openModal triggers */}
+      <Button onClick={() => openModal('edit')}>Edit</Button>
+      
+      <AppInfoModals
+        appDetail={appDetail}
+        activeModal={activeModal}
+        onClose={closeModal}
+        onSuccess={handleSuccess}
+      />
+    </div>
+  )
+}
+```
+
+### Strategy 4: List Item Extraction
+
+Extract repeated item rendering.
+
+```typescript
+// ❌ Before: Inline item rendering
+const OperationsList = () => {
+  return (
+    <div>
+      {operations.map(op => (
+        <div key={op.id} className="operation-item">
+          <span className="icon">{op.icon}</span>
+          <span className="title">{op.title}</span>
+          <span className="description">{op.description}</span>
+          <button onClick={() => op.onClick()}>
+            {op.actionLabel}
+          </button>
+          {op.badge && <Badge>{op.badge}</Badge>}
+          {/* More complex rendering... */}
+        </div>
+      ))}
+    </div>
+  )
+}
+
+// ✅ After: Extracted item component
+interface OperationItemProps {
+  operation: Operation
+  onAction: (id: string) => void
+}
+
+const OperationItem: FC<OperationItemProps> = ({ operation, onAction }) => {
+  return (
+    <div className="operation-item">
+      <span className="icon">{operation.icon}</span>
+      <span className="title">{operation.title}</span>
+      <span className="description">{operation.description}</span>
+      <button onClick={() => onAction(operation.id)}>
+        {operation.actionLabel}
+      </button>
+      {operation.badge && <Badge>{operation.badge}</Badge>}
+    </div>
+  )
+}
+
+const OperationsList = () => {
+  const handleAction = useCallback((id: string) => {
+    const op = operations.find(o => o.id === id)
+    op?.onClick()
+  }, [operations])
+
+  return (
+    <div>
+      {operations.map(op => (
+        <OperationItem
+          key={op.id}
+          operation={op}
+          onAction={handleAction}
+        />
+      ))}
+    </div>
+  )
+}
+```
+
+## Directory Structure Patterns
+
+### Pattern A: Flat Structure (Simple Components)
+
+For components with 2-3 sub-components:
+
+```
+component-name/
+  ├── index.tsx           # Main component
+  ├── sub-component-a.tsx
+  ├── sub-component-b.tsx
+  └── types.ts            # Shared types
+```
+
+### Pattern B: Nested Structure (Complex Components)
+
+For components with many sub-components:
+
+```
+component-name/
+  ├── index.tsx           # Main orchestration
+  ├── types.ts            # Shared types
+  ├── hooks/
+  │   ├── use-feature-a.ts
+  │   └── use-feature-b.ts
+  ├── components/
+  │   ├── header/
+  │   │   └── index.tsx
+  │   ├── content/
+  │   │   └── index.tsx
+  │   └── modals/
+  │       └── index.tsx
+  └── utils/
+      └── helpers.ts
+```
+
+### Pattern C: Feature-Based Structure (Dify Standard)
+
+Following Dify's existing patterns:
+
+```
+configuration/
+  ├── index.tsx           # Main page component
+  ├── base/               # Base/shared components
+  │   ├── feature-panel/
+  │   ├── group-name/
+  │   └── operation-btn/
+  ├── config/             # Config section
+  │   ├── index.tsx
+  │   ├── agent/
+  │   └── automatic/
+  ├── dataset-config/     # Dataset section
+  │   ├── index.tsx
+  │   ├── card-item/
+  │   └── params-config/
+  ├── debug/              # Debug section
+  │   ├── index.tsx
+  │   └── hooks.tsx
+  └── hooks/              # Shared hooks
+      └── use-advanced-prompt-config.ts
+```
+
+## Props Design
+
+### Minimal Props Principle
+
+Pass only what's needed:
+
+```typescript
+// ❌ Bad: Passing entire objects when only some fields needed
+<ConfigHeader appDetail={appDetail} modelConfig={modelConfig} />
+
+// ✅ Good: Destructure to minimum required
+<ConfigHeader
+  appName={appDetail.name}
+  isAdvancedMode={modelConfig.isAdvanced}
+  onPublish={handlePublish}
+/>
+```
+
+### Callback Props Pattern
+
+Use callbacks for child-to-parent communication:
+
+```typescript
+// Parent
+const Parent = () => {
+  const [value, setValue] = useState('')
+  
+  return (
+    <Child
+      value={value}
+      onChange={setValue}
+      onSubmit={handleSubmit}
+    />
+  )
+}
+
+// Child
+interface ChildProps {
+  value: string
+  onChange: (value: string) => void
+  onSubmit: () => void
+}
+
+const Child: FC<ChildProps> = ({ value, onChange, onSubmit }) => {
+  return (
+    <div>
+      <input value={value} onChange={e => onChange(e.target.value)} />
+      <button onClick={onSubmit}>Submit</button>
+    </div>
+  )
+}
+```
+
+### Render Props for Flexibility
+
+When sub-components need parent context:
+
+```typescript
+interface ListProps<T> {
+  items: T[]
+  renderItem: (item: T, index: number) => React.ReactNode
+  renderEmpty?: () => React.ReactNode
+}
+
+function List<T>({ items, renderItem, renderEmpty }: ListProps<T>) {
+  if (items.length === 0 && renderEmpty) {
+    return <>{renderEmpty()}</>
+  }
+  
+  return (
+    <div>
+      {items.map((item, index) => renderItem(item, index))}
+    </div>
+  )
+}
+
+// Usage
+<List
+  items={operations}
+  renderItem={(op, i) => <OperationItem key={i} operation={op} />}
+  renderEmpty={() => <EmptyState message="No operations" />}
+/>
+```

.claude/skills/component-refactoring/references/hook-extraction.md

@@ -0,0 +1,317 @@
+# Hook Extraction Patterns
+
+This document provides detailed guidance on extracting custom hooks from complex components in Dify.
+
+## When to Extract Hooks
+
+Extract a custom hook when you identify:
+
+1. **Coupled state groups** - Multiple `useState` hooks that are always used together
+1. **Complex effects** - `useEffect` with multiple dependencies or cleanup logic
+1. **Business logic** - Data transformations, validations, or calculations
+1. **Reusable patterns** - Logic that appears in multiple components
+
+## Extraction Process
+
+### Step 1: Identify State Groups
+
+Look for state variables that are logically related:
+
+```typescript
+// ❌ These belong together - extract to hook
+const [modelConfig, setModelConfig] = useState<ModelConfig>(...)
+const [completionParams, setCompletionParams] = useState<FormValue>({})
+const [modelModeType, setModelModeType] = useState<ModelModeType>(...)
+
+// These are model-related state that should be in useModelConfig()
+```
+
+### Step 2: Identify Related Effects
+
+Find effects that modify the grouped state:
+
+```typescript
+// ❌ These effects belong with the state above
+useEffect(() => {
+  if (hasFetchedDetail && !modelModeType) {
+    const mode = currModel?.model_properties.mode
+    if (mode) {
+      const newModelConfig = produce(modelConfig, (draft) => {
+        draft.mode = mode
+      })
+      setModelConfig(newModelConfig)
+    }
+  }
+}, [textGenerationModelList, hasFetchedDetail, modelModeType, currModel])
+```
+
+### Step 3: Create the Hook
+
+```typescript
+// hooks/use-model-config.ts
+import type { FormValue } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { ModelConfig } from '@/models/debug'
+import { produce } from 'immer'
+import { useEffect, useState } from 'react'
+import { ModelModeType } from '@/types/app'
+
+interface UseModelConfigParams {
+  initialConfig?: Partial<ModelConfig>
+  currModel?: { model_properties?: { mode?: ModelModeType } }
+  hasFetchedDetail: boolean
+}
+
+interface UseModelConfigReturn {
+  modelConfig: ModelConfig
+  setModelConfig: (config: ModelConfig) => void
+  completionParams: FormValue
+  setCompletionParams: (params: FormValue) => void
+  modelModeType: ModelModeType
+}
+
+export const useModelConfig = ({
+  initialConfig,
+  currModel,
+  hasFetchedDetail,
+}: UseModelConfigParams): UseModelConfigReturn => {
+  const [modelConfig, setModelConfig] = useState<ModelConfig>({
+    provider: 'langgenius/openai/openai',
+    model_id: 'gpt-3.5-turbo',
+    mode: ModelModeType.unset,
+    // ... default values
+    ...initialConfig,
+  })
+  
+  const [completionParams, setCompletionParams] = useState<FormValue>({})
+  
+  const modelModeType = modelConfig.mode
+
+  // Fill old app data missing model mode
+  useEffect(() => {
+    if (hasFetchedDetail && !modelModeType) {
+      const mode = currModel?.model_properties?.mode
+      if (mode) {
+        setModelConfig(produce(modelConfig, (draft) => {
+          draft.mode = mode
+        }))
+      }
+    }
+  }, [hasFetchedDetail, modelModeType, currModel])
+
+  return {
+    modelConfig,
+    setModelConfig,
+    completionParams,
+    setCompletionParams,
+    modelModeType,
+  }
+}
+```
+
+### Step 4: Update Component
+
+```typescript
+// Before: 50+ lines of state management
+const Configuration: FC = () => {
+  const [modelConfig, setModelConfig] = useState<ModelConfig>(...)
+  // ... lots of related state and effects
+}
+
+// After: Clean component
+const Configuration: FC = () => {
+  const {
+    modelConfig,
+    setModelConfig,
+    completionParams,
+    setCompletionParams,
+    modelModeType,
+  } = useModelConfig({
+    currModel,
+    hasFetchedDetail,
+  })
+  
+  // Component now focuses on UI
+}
+```
+
+## Naming Conventions
+
+### Hook Names
+
+- Use `use` prefix: `useModelConfig`, `useDatasetConfig`
+- Be specific: `useAdvancedPromptConfig` not `usePrompt`
+- Include domain: `useWorkflowVariables`, `useMCPServer`
+
+### File Names
+
+- Kebab-case: `use-model-config.ts`
+- Place in `hooks/` subdirectory when multiple hooks exist
+- Place alongside component for single-use hooks
+
+### Return Type Names
+
+- Suffix with `Return`: `UseModelConfigReturn`
+- Suffix params with `Params`: `UseModelConfigParams`
+
+## Common Hook Patterns in Dify
+
+### 1. Data Fetching Hook (React Query)
+
+```typescript
+// Pattern: Use @tanstack/react-query for data fetching
+import { useQuery, useQueryClient } from '@tanstack/react-query'
+import { get } from '@/service/base'
+import { useInvalid } from '@/service/use-base'
+
+const NAME_SPACE = 'appConfig'
+
+// Query keys for cache management
+export const appConfigQueryKeys = {
+  detail: (appId: string) => [NAME_SPACE, 'detail', appId] as const,
+}
+
+// Main data hook
+export const useAppConfig = (appId: string) => {
+  return useQuery({
+    enabled: !!appId,
+    queryKey: appConfigQueryKeys.detail(appId),
+    queryFn: () => get<AppDetailResponse>(`/apps/${appId}`),
+    select: data => data?.model_config || null,
+  })
+}
+
+// Invalidation hook for refreshing data
+export const useInvalidAppConfig = () => {
+  return useInvalid([NAME_SPACE])
+}
+
+// Usage in component
+const Component = () => {
+  const { data: config, isLoading, error, refetch } = useAppConfig(appId)
+  const invalidAppConfig = useInvalidAppConfig()
+  
+  const handleRefresh = () => {
+    invalidAppConfig() // Invalidates cache and triggers refetch
+  }
+  
+  return <div>...</div>
+}
+```
+
+### 2. Form State Hook
+
+```typescript
+// Pattern: Form state + validation + submission
+export const useConfigForm = (initialValues: ConfigFormValues) => {
+  const [values, setValues] = useState(initialValues)
+  const [errors, setErrors] = useState<Record<string, string>>({})
+  const [isSubmitting, setIsSubmitting] = useState(false)
+
+  const validate = useCallback(() => {
+    const newErrors: Record<string, string> = {}
+    if (!values.name) newErrors.name = 'Name is required'
+    setErrors(newErrors)
+    return Object.keys(newErrors).length === 0
+  }, [values])
+
+  const handleChange = useCallback((field: string, value: any) => {
+    setValues(prev => ({ ...prev, [field]: value }))
+  }, [])
+
+  const handleSubmit = useCallback(async (onSubmit: (values: ConfigFormValues) => Promise<void>) => {
+    if (!validate()) return
+    setIsSubmitting(true)
+    try {
+      await onSubmit(values)
+    } finally {
+      setIsSubmitting(false)
+    }
+  }, [values, validate])
+
+  return { values, errors, isSubmitting, handleChange, handleSubmit }
+}
+```
+
+### 3. Modal State Hook
+
+```typescript
+// Pattern: Multiple modal management
+type ModalType = 'edit' | 'delete' | 'duplicate' | null
+
+export const useModalState = () => {
+  const [activeModal, setActiveModal] = useState<ModalType>(null)
+  const [modalData, setModalData] = useState<any>(null)
+
+  const openModal = useCallback((type: ModalType, data?: any) => {
+    setActiveModal(type)
+    setModalData(data)
+  }, [])
+
+  const closeModal = useCallback(() => {
+    setActiveModal(null)
+    setModalData(null)
+  }, [])
+
+  return {
+    activeModal,
+    modalData,
+    openModal,
+    closeModal,
+    isOpen: useCallback((type: ModalType) => activeModal === type, [activeModal]),
+  }
+}
+```
+
+### 4. Toggle/Boolean Hook
+
+```typescript
+// Pattern: Boolean state with convenience methods
+export const useToggle = (initialValue = false) => {
+  const [value, setValue] = useState(initialValue)
+
+  const toggle = useCallback(() => setValue(v => !v), [])
+  const setTrue = useCallback(() => setValue(true), [])
+  const setFalse = useCallback(() => setValue(false), [])
+
+  return [value, { toggle, setTrue, setFalse, set: setValue }] as const
+}
+
+// Usage
+const [isExpanded, { toggle, setTrue: expand, setFalse: collapse }] = useToggle()
+```
+
+## Testing Extracted Hooks
+
+After extraction, test hooks in isolation:
+
+```typescript
+// use-model-config.spec.ts
+import { renderHook, act } from '@testing-library/react'
+import { useModelConfig } from './use-model-config'
+
+describe('useModelConfig', () => {
+  it('should initialize with default values', () => {
+    const { result } = renderHook(() => useModelConfig({
+      hasFetchedDetail: false,
+    }))
+
+    expect(result.current.modelConfig.provider).toBe('langgenius/openai/openai')
+    expect(result.current.modelModeType).toBe(ModelModeType.unset)
+  })
+
+  it('should update model config', () => {
+    const { result } = renderHook(() => useModelConfig({
+      hasFetchedDetail: true,
+    }))
+
+    act(() => {
+      result.current.setModelConfig({
+        ...result.current.modelConfig,
+        model_id: 'gpt-4',
+      })
+    })
+
+    expect(result.current.modelConfig.model_id).toBe('gpt-4')
+  })
+})
+```

.claude/skills/frontend-code-review/SKILL.md

@@ -0,0 +1,73 @@
+---
+name: frontend-code-review
+description: "Trigger when the user requests a review of frontend files (e.g., `.tsx`, `.ts`, `.js`). Support both pending-change reviews and focused file reviews while applying the checklist rules."
+---
+
+# Frontend Code Review
+
+## Intent
+Use this skill whenever the user asks to review frontend code (especially `.tsx`, `.ts`, or `.js` files). Support two review modes:
+
+1. **Pending-change review** – inspect staged/working-tree files slated for commit and flag checklist violations before submission.
+2. **File-targeted review** – review the specific file(s) the user names and report the relevant checklist findings.
+
+Stick to the checklist below for every applicable file and mode.
+
+## Checklist
+See [references/code-quality.md](references/code-quality.md), [references/performance.md](references/performance.md), [references/business-logic.md](references/business-logic.md) for the living checklist split by category—treat it as the canonical set of rules to follow.
+
+Flag each rule violation with urgency metadata so future reviewers can prioritize fixes.
+
+## Review Process
+1. Open the relevant component/module. Gather lines that relate to class names, React Flow hooks, prop memoization, and styling.
+2. For each rule in the review point, note where the code deviates and capture a representative snippet.
+3. Compose the review section per the template below. Group violations first by **Urgent** flag, then by category order (Code Quality, Performance, Business Logic).
+
+## Required output
+When invoked, the response must exactly follow one of the two templates:
+
+### Template A (any findings)
+```
+# Code review
+Found <N> urgent issues need to be fixed:
+
+## 1 <brief description of bug>
+FilePath: <path> line <line>
+<relevant code snippet or pointer>
+
+
+### Suggested fix
+<brief description of suggested fix>
+
+---
+... (repeat for each urgent issue) ...
+
+Found <M> suggestions for improvement:
+
+## 1 <brief description of suggestion>
+FilePath: <path> line <line>
+<relevant code snippet or pointer>
+
+
+### Suggested fix
+<brief description of suggested fix>
+
+---
+
+... (repeat for each suggestion) ...
+```
+
+If there are no urgent issues, omit that section. If there are no suggestions, omit that section.
+
+If the issue number is more than 10, summarize as "10+ urgent issues" or "10+ suggestions" and just output the first 10 issues.
+
+Don't compress the blank lines between sections; keep them as-is for readability.
+
+If you use Template A (i.e., there are issues to fix) and at least one issue requires code changes, append a brief follow-up question after the structured output asking whether the user wants you to apply the suggested fix(es). For example: "Would you like me to use the Suggested fix section to address these issues?"
+
+### Template B (no issues)
+```
+## Code review
+No issues found.
+```
+

.claude/skills/frontend-code-review/references/business-logic.md

@@ -0,0 +1,15 @@
+# Rule Catalog — Business Logic
+
+## Can't use workflowStore in Node components
+
+IsUrgent: True
+
+### Description
+
+File path pattern of node components: `web/app/components/workflow/nodes/[nodeName]/node.tsx`
+
+Node components are also used when creating a RAG Pipe from a template, but in that context there is no workflowStore Provider, which results in a blank screen. [This Issue](https://github.com/langgenius/dify/issues/29168) was caused by exactly this reason.
+
+### Suggested Fix
+
+Use `import { useNodes } from 'reactflow'` instead of `import useNodes from '@/app/components/workflow/store/workflow/use-nodes'`.

.claude/skills/frontend-code-review/references/code-quality.md

@@ -0,0 +1,44 @@
+# Rule Catalog — Code Quality
+
+## Conditional class names use utility function
+
+IsUrgent: True
+Category: Code Quality
+
+### Description
+
+Ensure conditional CSS is handled via the shared `classNames` instead of custom ternaries, string concatenation, or template strings. Centralizing class logic keeps components consistent and easier to maintain.
+
+### Suggested Fix
+
+```ts
+import { cn } from '@/utils/classnames'
+const classNames = cn(isActive ? 'text-primary-600' : 'text-gray-500')
+```
+
+## Tailwind-first styling
+
+IsUrgent: True
+Category: Code Quality
+
+### Description
+
+Favor Tailwind CSS utility classes instead of adding new `.module.css` files unless a Tailwind combination cannot achieve the required styling. Keeping styles in Tailwind improves consistency and reduces maintenance overhead.
+
+Update this file when adding, editing, or removing Code Quality rules so the catalog remains accurate.
+
+## Classname ordering for easy overrides
+
+### Description
+
+When writing components, always place the incoming `className` prop after the component’s own class values so that downstream consumers can override or extend the styling. This keeps your component’s defaults but still lets external callers change or remove specific styles.
+
+Example:
+
+```tsx
+import { cn } from '@/utils/classnames'
+
+const Button = ({ className }) => {
+  return <div className={cn('bg-primary-600', className)}></div>
+}
+```

.claude/skills/frontend-code-review/references/performance.md

@@ -0,0 +1,45 @@
+# Rule Catalog — Performance
+
+## React Flow data usage
+
+IsUrgent: True
+Category: Performance
+
+### Description
+
+When rendering React Flow, prefer `useNodes`/`useEdges` for UI consumption and rely on `useStoreApi` inside callbacks that mutate or read node/edge state. Avoid manually pulling Flow data outside of these hooks.
+
+## Complex prop memoization
+
+IsUrgent: True
+Category: Performance
+
+### Description
+
+Wrap complex prop values (objects, arrays, maps) in `useMemo` prior to passing them into child components to guarantee stable references and prevent unnecessary renders.
+
+Update this file when adding, editing, or removing Performance rules so the catalog remains accurate.
+
+Wrong:
+
+```tsx
+<HeavyComp
+    config={{
+        provider: ...,
+        detail: ...
+    }}
+/>
+```
+
+Right:
+
+```tsx
+const config = useMemo(() => ({
+    provider: ...,
+    detail: ...
+}), [provider, detail]);
+
+<HeavyComp
+    config={config}
+/>
+```

.claude/skills/frontend-testing/SKILL.md

@@ -318,5 +318,5 @@ For more detailed information, refer to:
 
 - `web/vitest.config.ts` - Vitest configuration
 - `web/vitest.setup.ts` - Test environment setup
-- `web/testing/analyze-component.js` - Component analysis tool
+- `web/scripts/analyze-component.js` - Component analysis tool
 - Modules are not mocked automatically. Global mocks live in `web/vitest.setup.ts` (for example `react-i18next`, `next/image`); mock other modules like `ky` or `mime` locally in test files.

.claude/skills/frontend-testing/assets/component-test.template.tsx

@@ -28,17 +28,14 @@ import userEvent from '@testing-library/user-event'
 
 // i18n (automatically mocked)
 // WHY: Global mock in web/vitest.setup.ts is auto-loaded by Vitest setup
-// No explicit mock needed - it returns translation keys as-is
+// The global mock provides: useTranslation, Trans, useMixedTranslation, useGetLanguage
+// No explicit mock needed for most tests
+//
 // Override only if custom translations are required:
-// vi.mock('react-i18next', () => ({
-//   useTranslation: () => ({
-//     t: (key: string) => {
-//       const customTranslations: Record<string, string> = {
-//         'my.custom.key': 'Custom Translation',
-//       }
-//       return customTranslations[key] || key
-//     },
-//   }),
+// import { createReactI18nextMock } from '@/test/i18n-mock'
+// vi.mock('react-i18next', () => createReactI18nextMock({
+//   'my.custom.key': 'Custom Translation',
+//   'button.save': 'Save',
 // }))
 
 // Router (if component uses useRouter, usePathname, useSearchParams)

.claude/skills/frontend-testing/references/mocking.md

@@ -52,23 +52,29 @@ Modules are not mocked automatically. Use `vi.mock` in test files, or add global
 ### 1. i18n (Auto-loaded via Global Mock)
 
 A global mock is defined in `web/vitest.setup.ts` and is auto-loaded by Vitest setup.
-**No explicit mock needed** for most tests - it returns translation keys as-is.
 
-For tests requiring custom translations, override the mock:
+The global mock provides:
+
+- `useTranslation` - returns translation keys with namespace prefix
+- `Trans` component - renders i18nKey and components
+- `useMixedTranslation` (from `@/app/components/plugins/marketplace/hooks`)
+- `useGetLanguage` (from `@/context/i18n`) - returns `'en-US'`
+
+**Default behavior**: Most tests should use the global mock (no local override needed).
+
+**For custom translations**: Use the helper function from `@/test/i18n-mock`:
 
 ```typescript
-vi.mock('react-i18next', () => ({
-  useTranslation: () => ({
-    t: (key: string) => {
-      const translations: Record<string, string> = {
-        'my.custom.key': 'Custom translation',
-      }
-      return translations[key] || key
-    },
-  }),
+import { createReactI18nextMock } from '@/test/i18n-mock'
+
+vi.mock('react-i18next', () => createReactI18nextMock({
+  'my.custom.key': 'Custom translation',
+  'button.save': 'Save',
 }))
 ```
 
+**Avoid**: Manually defining `useTranslation` mocks that just return the key - the global mock already does this.
+
 ### 2. Next.js Router
 
 ```typescript

.github/pull_request_template.md

@@ -20,4 +20,4 @@
 - [x] I understand that this PR may be closed in case there was no previous discussion or issues. (This doesn't apply to typos!)
 - [x] I've added a test for each change that was introduced, and I tried as much as possible to make a single atomic change.
 - [x] I've updated the documentation accordingly.
-- [x] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
+- [x] I ran `make lint` and `make type-check` (backend) and `cd web && npx lint-staged` (frontend) to appease the lint gods

.github/workflows/api-tests.yml

@@ -22,12 +22,12 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
@@ -57,7 +57,7 @@ jobs:
         run: sh .github/workflows/expose_service_ports.sh
 
       - name: Set up Sandbox
-        uses: hoverkraft-tech/compose-action@v2.0.2
+        uses: hoverkraft-tech/compose-action@v2
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml

.github/workflows/autofix.yml

@@ -12,7 +12,7 @@ jobs:
     if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
       - name: Check Docker Compose inputs
         id: docker-compose-changes
@@ -27,7 +27,7 @@ jobs:
         with:
           python-version: "3.11"
 
-      - uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@v7
 
       - name: Generate Docker Compose
         if: steps.docker-compose-changes.outputs.any_changed == 'true'

.github/workflows/build-push.yml

@@ -90,7 +90,7 @@ jobs:
           touch "/tmp/digests/${sanitized_digest}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
           path: /tmp/digests/*

.github/workflows/db-migration-test.yml

@@ -13,13 +13,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           python-version: "3.12"
@@ -63,13 +63,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           python-version: "3.12"

.github/workflows/main-ci.yml

@@ -27,7 +27,7 @@ jobs:
       vdb-changed: ${{ steps.changes.outputs.vdb }}
       migration-changed: ${{ steps.changes.outputs.migration }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: dorny/paths-filter@v3
         id: changes
         with:
@@ -38,6 +38,7 @@ jobs:
               - '.github/workflows/api-tests.yml'
             web:
               - 'web/**'
+              - '.github/workflows/web-tests.yml'
             vdb:
               - 'api/core/rag/datasource/**'
               - 'docker/**'

.github/workflows/style.yml

@@ -19,13 +19,13 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v47
         with:
           files: |
             api/**
@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup UV and Python
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: false
           python-version: "3.12"
@@ -68,15 +68,17 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v47
         with:
-          files: web/**
+          files: |
+            web/**
+            .github/workflows/style.yml
 
       - name: Install pnpm
         uses: pnpm/action-setup@v4
@@ -85,7 +87,7 @@ jobs:
           run_install: false
 
       - name: Setup NodeJS
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         if: steps.changed-files.outputs.any_changed == 'true'
         with:
           node-version: 22
@@ -108,20 +110,30 @@ jobs:
         working-directory: ./web
         run: pnpm run type-check:tsgo
 
+      - name: Web dead code check
+        if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
+        run: pnpm run knip
+
+      - name: Web build check
+        if: steps.changed-files.outputs.any_changed == 'true'
+        working-directory: ./web
+        run: pnpm run build
+
   superlinter:
     name: SuperLinter
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           persist-credentials: false
 
       - name: Check changed files
         id: changed-files
-        uses: tj-actions/changed-files@v46
+        uses: tj-actions/changed-files@v47
         with:
           files: |
             **.sh

.github/workflows/tool-test-sdks.yaml

@@ -25,12 +25,12 @@ jobs:
         working-directory: sdks/nodejs-client
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: ${{ matrix.node-version }}
           cache: ''

.github/workflows/translate-i18n-base-on-english.yml

@@ -4,7 +4,8 @@ on:
   push:
     branches: [main]
     paths:
-      - 'web/i18n/en-US/*.ts'
+      - 'web/i18n/en-US/*.json'
+  workflow_dispatch:
 
 permissions:
   contents: write
@@ -18,6 +19,7 @@ jobs:
       run:
         working-directory: web
     steps:
+      # Keep use old checkout action version for https://github.com/peter-evans/create-pull-request/issues/4272
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -26,21 +28,28 @@ jobs:
       - name: Check for file changes in i18n/en-US
         id: check_files
         run: |
-          git fetch origin "${{ github.event.before }}" || true
-          git fetch origin "${{ github.sha }}" || true
-          changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" -- 'i18n/en-US/*.ts')
-          echo "Changed files: $changed_files"
-          if [ -n "$changed_files" ]; then
+          # Skip check for manual trigger, translate all files
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             echo "FILES_CHANGED=true" >> $GITHUB_ENV
-            file_args=""
-            for file in $changed_files; do
-              filename=$(basename "$file" .ts)
-              file_args="$file_args --file $filename"
-            done
-            echo "FILE_ARGS=$file_args" >> $GITHUB_ENV
-            echo "File arguments: $file_args"
+            echo "FILE_ARGS=" >> $GITHUB_ENV
+            echo "Manual trigger: translating all files"
           else
-            echo "FILES_CHANGED=false" >> $GITHUB_ENV
+            git fetch origin "${{ github.event.before }}" || true
+            git fetch origin "${{ github.sha }}" || true
+            changed_files=$(git diff --name-only "${{ github.event.before }}" "${{ github.sha }}" -- 'i18n/en-US/*.json')
+            echo "Changed files: $changed_files"
+            if [ -n "$changed_files" ]; then
+              echo "FILES_CHANGED=true" >> $GITHUB_ENV
+              file_args=""
+              for file in $changed_files; do
+                filename=$(basename "$file" .json)
+                file_args="$file_args --file $filename"
+              done
+              echo "FILE_ARGS=$file_args" >> $GITHUB_ENV
+              echo "File arguments: $file_args"
+            else
+              echo "FILES_CHANGED=false" >> $GITHUB_ENV
+            fi
           fi
 
       - name: Install pnpm
@@ -51,7 +60,7 @@ jobs:
 
       - name: Set up Node.js
         if: env.FILES_CHANGED == 'true'
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: 'lts/*'
           cache: pnpm
@@ -65,7 +74,7 @@ jobs:
       - name: Generate i18n translations
         if: env.FILES_CHANGED == 'true'
         working-directory: ./web
-        run: pnpm run auto-gen-i18n ${{ env.FILE_ARGS }}
+        run: pnpm run i18n:gen ${{ env.FILE_ARGS }}
 
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'

.github/workflows/vdb-tests.yml

@@ -19,19 +19,19 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
       - name: Free Disk Space
-        uses: endersonmenezes/free-disk-space@v2
+        uses: endersonmenezes/free-disk-space@v3
         with:
           remove_dotnet: true
           remove_haskell: true
           remove_tool_cache: true
 
       - name: Setup UV and Python
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}

.github/workflows/web-tests.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           persist-credentials: false
 
@@ -29,7 +29,7 @@ jobs:
           run_install: false
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: 22
           cache: pnpm
@@ -360,7 +360,7 @@ jobs:
 
       - name: Upload Coverage Artifact
         if: steps.coverage-summary.outputs.has_coverage == 'true'
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: web-coverage-report
           path: web/coverage

.gitignore

@@ -235,3 +235,4 @@ scripts/stress-test/reports/
 
 # settings
 *.local.json
+*.local.md

.mcp.json

@@ -1,34 +0,0 @@
-{
-    "mcpServers": {
-      "context7": {
-        "type": "http",
-        "url": "https://mcp.context7.com/mcp"
-      },
-      "sequential-thinking": {
-        "type": "stdio",
-        "command": "npx",
-        "args": ["-y", "@modelcontextprotocol/server-sequential-thinking"],
-        "env": {}
-      },
-      "github": {
-        "type": "stdio",
-        "command": "npx",
-        "args": ["-y", "@modelcontextprotocol/server-github"],
-        "env": {
-          "GITHUB_PERSONAL_ACCESS_TOKEN": "${GITHUB_PERSONAL_ACCESS_TOKEN}"
-        }
-      },
-      "fetch": {
-        "type": "stdio",
-        "command": "uvx",
-        "args": ["mcp-server-fetch"],
-        "env": {}
-      },
-      "playwright": {
-        "type": "stdio",
-        "command": "npx",
-        "args": ["-y", "@playwright/mcp@latest"],
-        "env": {}
-      }
-    }
-  }

Makefile

@@ -60,9 +60,10 @@ check:
 	@echo "✅ Code check complete"
 
 lint:
-	@echo "🔧 Running ruff format, check with fixes, and import linter..."
+	@echo "🔧 Running ruff format, check with fixes, import linter, and dotenv-linter..."
 	@uv run --project api --dev sh -c 'ruff format ./api && ruff check --fix ./api'
 	@uv run --directory api --dev lint-imports
+	@uv run --project api --dev dotenv-linter ./api/.env.example ./web/.env.example
 	@echo "✅ Linting complete"
 
 type-check:
@@ -122,7 +123,7 @@ help:
 	@echo "Backend Code Quality:"
 	@echo "  make format         - Format code with ruff"
 	@echo "  make check          - Check code with ruff"
-	@echo "  make lint           - Format and fix code with ruff"
+	@echo "  make lint           - Format, fix, and lint code (ruff, imports, dotenv)"
 	@echo "  make type-check     - Run type checking with basedpyright"
 	@echo "  make test           - Run backend unit tests"
 	@echo ""

api/.env.example

@@ -101,6 +101,15 @@ S3_ACCESS_KEY=your-access-key
 S3_SECRET_KEY=your-secret-key
 S3_REGION=your-region
 
+# Workflow run and Conversation archive storage (S3-compatible)
+ARCHIVE_STORAGE_ENABLED=false
+ARCHIVE_STORAGE_ENDPOINT=
+ARCHIVE_STORAGE_ARCHIVE_BUCKET=
+ARCHIVE_STORAGE_EXPORT_BUCKET=
+ARCHIVE_STORAGE_ACCESS_KEY=
+ARCHIVE_STORAGE_SECRET_KEY=
+ARCHIVE_STORAGE_REGION=auto
+
 # Azure Blob Storage configuration
 AZURE_BLOB_ACCOUNT_NAME=your-account-name
 AZURE_BLOB_ACCOUNT_KEY=your-account-key
@@ -128,6 +137,7 @@ TENCENT_COS_SECRET_KEY=your-secret-key
 TENCENT_COS_SECRET_ID=your-secret-id
 TENCENT_COS_REGION=your-region
 TENCENT_COS_SCHEME=your-scheme
+TENCENT_COS_CUSTOM_DOMAIN=your-custom-domain
 
 # Huawei OBS Storage Configuration
 HUAWEI_OBS_BUCKET_NAME=your-bucket-name
@@ -492,6 +502,8 @@ LOG_FILE_BACKUP_COUNT=5
 LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
 # Log Timezone
 LOG_TZ=UTC
+# Log output format: text or json
+LOG_OUTPUT_FORMAT=text
 # Log format
 LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s
 
@@ -563,6 +575,10 @@ LOGSTORE_DUAL_WRITE_ENABLED=false
 # Enable dual-read fallback to SQL database when LogStore returns no results (default: true)
 # Useful for migration scenarios where historical data exists only in SQL database
 LOGSTORE_DUAL_READ_ENABLED=true
+# Control flag for whether to write the `graph` field to LogStore.
+# If LOGSTORE_ENABLE_PUT_GRAPH_FIELD is "true", write the full `graph` field;
+# otherwise write an empty {} instead. Defaults to writing the `graph` field.
+LOGSTORE_ENABLE_PUT_GRAPH_FIELD=true
 
 # Celery beat configuration
 CELERY_BEAT_SCHEDULER_TIME=1

api/.importlinter

@@ -3,9 +3,11 @@ root_packages =
     core
     configs
     controllers
+    extensions
     models
     tasks
     services
+include_external_packages = True
 
 [importlinter:contract:workflow]
 name = Workflow
@@ -33,6 +35,29 @@ ignore_imports =
     core.workflow.nodes.loop.loop_node -> core.workflow.graph
     core.workflow.nodes.loop.loop_node -> core.workflow.graph_engine.command_channels
 
+[importlinter:contract:workflow-infrastructure-dependencies]
+name = Workflow Infrastructure Dependencies
+type = forbidden
+source_modules =
+    core.workflow
+forbidden_modules =
+    extensions.ext_database
+    extensions.ext_redis
+allow_indirect_imports = True
+ignore_imports =
+    core.workflow.nodes.agent.agent_node -> extensions.ext_database
+    core.workflow.nodes.datasource.datasource_node -> extensions.ext_database
+    core.workflow.nodes.knowledge_index.knowledge_index_node -> extensions.ext_database
+    core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> extensions.ext_database
+    core.workflow.nodes.llm.file_saver -> extensions.ext_database
+    core.workflow.nodes.llm.llm_utils -> extensions.ext_database
+    core.workflow.nodes.llm.node -> extensions.ext_database
+    core.workflow.nodes.tool.tool_node -> extensions.ext_database
+    core.workflow.nodes.variable_assigner.common.impl -> extensions.ext_database
+    core.workflow.graph_engine.command_channels.redis_channel -> extensions.ext_redis
+    core.workflow.graph_engine.manager -> extensions.ext_redis
+    core.workflow.nodes.knowledge_retrieval.knowledge_retrieval_node -> extensions.ext_redis
+
 [importlinter:contract:rsc]
 name = RSC
 type = layers

api/.ruff.toml

@@ -1,4 +1,8 @@
-exclude = ["migrations/*"]
+exclude = [
+    "migrations/*",
+    ".git",
+    ".git/**",
+]
 line-length = 120
 
 [format]

api/Dockerfile

@@ -79,7 +79,8 @@ COPY --from=packages --chown=dify:dify ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 
 # Download nltk data
-RUN mkdir -p /usr/local/share/nltk_data && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords')" \
+RUN mkdir -p /usr/local/share/nltk_data \
+    && NLTK_DATA=/usr/local/share/nltk_data python -c "import nltk; from unstructured.nlp.tokenize import download_nltk_packages; nltk.download('punkt'); nltk.download('averaged_perceptron_tagger'); nltk.download('stopwords'); download_nltk_packages()" \
     && chmod -R 755 /usr/local/share/nltk_data
 
 ENV TIKTOKEN_CACHE_DIR=/app/api/.tiktoken_cache

api/app_factory.py

@@ -2,9 +2,11 @@ import logging
 import time
 
 from opentelemetry.trace import get_current_span
+from opentelemetry.trace.span import INVALID_SPAN_ID, INVALID_TRACE_ID
 
 from configs import dify_config
 from contexts.wrapper import RecyclableContextVar
+from core.logging.context import init_request_context
 from dify_app import DifyApp
 
 logger = logging.getLogger(__name__)
@@ -25,28 +27,35 @@ def create_flask_app_with_configs() -> DifyApp:
     # add before request hook
     @dify_app.before_request
     def before_request():
-        # add an unique identifier to each request
+        # Initialize logging context for this request
+        init_request_context()
         RecyclableContextVar.increment_thread_recycles()
 
-    # add after request hook for injecting X-Trace-Id header from OpenTelemetry span context
+    # add after request hook for injecting trace headers from OpenTelemetry span context
+    # Only adds headers when OTEL is enabled and has valid context
     @dify_app.after_request
-    def add_trace_id_header(response):
+    def add_trace_headers(response):
         try:
             span = get_current_span()
             ctx = span.get_span_context() if span else None
-            if ctx and ctx.is_valid:
-                trace_id_hex = format(ctx.trace_id, "032x")
-                # Avoid duplicates if some middleware added it
-                if "X-Trace-Id" not in response.headers:
-                    response.headers["X-Trace-Id"] = trace_id_hex
+
+            if not ctx or not ctx.is_valid:
+                return response
+
+            # Inject trace headers from OTEL context
+            if ctx.trace_id != INVALID_TRACE_ID and "X-Trace-Id" not in response.headers:
+                response.headers["X-Trace-Id"] = format(ctx.trace_id, "032x")
+            if ctx.span_id != INVALID_SPAN_ID and "X-Span-Id" not in response.headers:
+                response.headers["X-Span-Id"] = format(ctx.span_id, "016x")
+
         except Exception:
             # Never break the response due to tracing header injection
-            logger.warning("Failed to add trace ID to response header", exc_info=True)
+            logger.warning("Failed to add trace headers to response", exc_info=True)
         return response
 
     # Capture the decorator's return value to avoid pyright reportUnusedFunction
     _ = before_request
-    _ = add_trace_id_header
+    _ = add_trace_headers
 
     return dify_app
 

api/commands.py

@@ -235,7 +235,7 @@ def migrate_annotation_vector_database():
                 if annotations:
                     for annotation in annotations:
                         document = Document(
-                            page_content=annotation.question,
+                            page_content=annotation.question_text,
                             metadata={"annotation_id": annotation.id, "app_id": app.id, "doc_id": annotation.id},
                         )
                         documents.append(document)
@@ -1184,6 +1184,217 @@ def remove_orphaned_files_on_storage(force: bool):
         click.echo(click.style(f"Removed {removed_files} orphaned files, with {error_files} errors.", fg="yellow"))
 
 
+@click.command("file-usage", help="Query file usages and show where files are referenced.")
+@click.option("--file-id", type=str, default=None, help="Filter by file UUID.")
+@click.option("--key", type=str, default=None, help="Filter by storage key.")
+@click.option("--src", type=str, default=None, help="Filter by table.column pattern (e.g., 'documents.%' or '%.icon').")
+@click.option("--limit", type=int, default=100, help="Limit number of results (default: 100).")
+@click.option("--offset", type=int, default=0, help="Offset for pagination (default: 0).")
+@click.option("--json", "output_json", is_flag=True, help="Output results in JSON format.")
+def file_usage(
+    file_id: str | None,
+    key: str | None,
+    src: str | None,
+    limit: int,
+    offset: int,
+    output_json: bool,
+):
+    """
+    Query file usages and show where files are referenced in the database.
+
+    This command reuses the same reference checking logic as clear-orphaned-file-records
+    and displays detailed information about where each file is referenced.
+    """
+    # define tables and columns to process
+    files_tables = [
+        {"table": "upload_files", "id_column": "id", "key_column": "key"},
+        {"table": "tool_files", "id_column": "id", "key_column": "file_key"},
+    ]
+    ids_tables = [
+        {"type": "uuid", "table": "message_files", "column": "upload_file_id", "pk_column": "id"},
+        {"type": "text", "table": "documents", "column": "data_source_info", "pk_column": "id"},
+        {"type": "text", "table": "document_segments", "column": "content", "pk_column": "id"},
+        {"type": "text", "table": "messages", "column": "answer", "pk_column": "id"},
+        {"type": "text", "table": "workflow_node_executions", "column": "inputs", "pk_column": "id"},
+        {"type": "text", "table": "workflow_node_executions", "column": "process_data", "pk_column": "id"},
+        {"type": "text", "table": "workflow_node_executions", "column": "outputs", "pk_column": "id"},
+        {"type": "text", "table": "conversations", "column": "introduction", "pk_column": "id"},
+        {"type": "text", "table": "conversations", "column": "system_instruction", "pk_column": "id"},
+        {"type": "text", "table": "accounts", "column": "avatar", "pk_column": "id"},
+        {"type": "text", "table": "apps", "column": "icon", "pk_column": "id"},
+        {"type": "text", "table": "sites", "column": "icon", "pk_column": "id"},
+        {"type": "json", "table": "messages", "column": "inputs", "pk_column": "id"},
+        {"type": "json", "table": "messages", "column": "message", "pk_column": "id"},
+    ]
+
+    # Stream file usages with pagination to avoid holding all results in memory
+    paginated_usages = []
+    total_count = 0
+
+    # First, build a mapping of file_id -> storage_key from the base tables
+    file_key_map = {}
+    for files_table in files_tables:
+        query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}"
+        with db.engine.begin() as conn:
+            rs = conn.execute(sa.text(query))
+            for row in rs:
+                file_key_map[str(row[0])] = f"{files_table['table']}:{row[1]}"
+
+    # If filtering by key or file_id, verify it exists
+    if file_id and file_id not in file_key_map:
+        if output_json:
+            click.echo(json.dumps({"error": f"File ID {file_id} not found in base tables"}))
+        else:
+            click.echo(click.style(f"File ID {file_id} not found in base tables.", fg="red"))
+        return
+
+    if key:
+        valid_prefixes = {f"upload_files:{key}", f"tool_files:{key}"}
+        matching_file_ids = [fid for fid, fkey in file_key_map.items() if fkey in valid_prefixes]
+        if not matching_file_ids:
+            if output_json:
+                click.echo(json.dumps({"error": f"Key {key} not found in base tables"}))
+            else:
+                click.echo(click.style(f"Key {key} not found in base tables.", fg="red"))
+            return
+
+    guid_regexp = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"
+
+    # For each reference table/column, find matching file IDs and record the references
+    for ids_table in ids_tables:
+        src_filter = f"{ids_table['table']}.{ids_table['column']}"
+
+        # Skip if src filter doesn't match (use fnmatch for wildcard patterns)
+        if src:
+            if "%" in src or "_" in src:
+                import fnmatch
+
+                # Convert SQL LIKE wildcards to fnmatch wildcards (% -> *, _ -> ?)
+                pattern = src.replace("%", "*").replace("_", "?")
+                if not fnmatch.fnmatch(src_filter, pattern):
+                    continue
+            else:
+                if src_filter != src:
+                    continue
+
+        if ids_table["type"] == "uuid":
+            # Direct UUID match
+            query = (
+                f"SELECT {ids_table['pk_column']}, {ids_table['column']} "
+                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+            )
+            with db.engine.begin() as conn:
+                rs = conn.execute(sa.text(query))
+                for row in rs:
+                    record_id = str(row[0])
+                    ref_file_id = str(row[1])
+                    if ref_file_id not in file_key_map:
+                        continue
+                    storage_key = file_key_map[ref_file_id]
+
+                    # Apply filters
+                    if file_id and ref_file_id != file_id:
+                        continue
+                    if key and not storage_key.endswith(key):
+                        continue
+
+                    # Only collect items within the requested page range
+                    if offset <= total_count < offset + limit:
+                        paginated_usages.append(
+                            {
+                                "src": f"{ids_table['table']}.{ids_table['column']}",
+                                "record_id": record_id,
+                                "file_id": ref_file_id,
+                                "key": storage_key,
+                            }
+                        )
+                    total_count += 1
+
+        elif ids_table["type"] in ("text", "json"):
+            # Extract UUIDs from text/json content
+            column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"]
+            query = (
+                f"SELECT {ids_table['pk_column']}, {column_cast} "
+                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+            )
+            with db.engine.begin() as conn:
+                rs = conn.execute(sa.text(query))
+                for row in rs:
+                    record_id = str(row[0])
+                    content = str(row[1])
+
+                    # Find all UUIDs in the content
+                    import re
+
+                    uuid_pattern = re.compile(guid_regexp, re.IGNORECASE)
+                    matches = uuid_pattern.findall(content)
+
+                    for ref_file_id in matches:
+                        if ref_file_id not in file_key_map:
+                            continue
+                        storage_key = file_key_map[ref_file_id]
+
+                        # Apply filters
+                        if file_id and ref_file_id != file_id:
+                            continue
+                        if key and not storage_key.endswith(key):
+                            continue
+
+                        # Only collect items within the requested page range
+                        if offset <= total_count < offset + limit:
+                            paginated_usages.append(
+                                {
+                                    "src": f"{ids_table['table']}.{ids_table['column']}",
+                                    "record_id": record_id,
+                                    "file_id": ref_file_id,
+                                    "key": storage_key,
+                                }
+                            )
+                        total_count += 1
+
+    # Output results
+    if output_json:
+        result = {
+            "total": total_count,
+            "offset": offset,
+            "limit": limit,
+            "usages": paginated_usages,
+        }
+        click.echo(json.dumps(result, indent=2))
+    else:
+        click.echo(
+            click.style(f"Found {total_count} file usages (showing {len(paginated_usages)} results)", fg="white")
+        )
+        click.echo("")
+
+        if not paginated_usages:
+            click.echo(click.style("No file usages found matching the specified criteria.", fg="yellow"))
+            return
+
+        # Print table header
+        click.echo(
+            click.style(
+                f"{'Src (Table.Column)':<50} {'Record ID':<40} {'File ID':<40} {'Storage Key':<60}",
+                fg="cyan",
+            )
+        )
+        click.echo(click.style("-" * 190, fg="white"))
+
+        # Print each usage
+        for usage in paginated_usages:
+            click.echo(f"{usage['src']:<50} {usage['record_id']:<40} {usage['file_id']:<40} {usage['key']:<60}")
+
+        # Show pagination info
+        if offset + limit < total_count:
+            click.echo("")
+            click.echo(
+                click.style(
+                    f"Showing {offset + 1}-{offset + len(paginated_usages)} of {total_count} results", fg="white"
+                )
+            )
+            click.echo(click.style(f"Use --offset {offset + limit} to see next page", fg="white"))
+
+
 @click.command("setup-system-tool-oauth-client", help="Setup system tool oauth client.")
 @click.option("--provider", prompt=True, help="Provider name")
 @click.option("--client-params", prompt=True, help="Client Params")

api/configs/extra/__init__.py

@@ -1,9 +1,11 @@
+from configs.extra.archive_config import ArchiveStorageConfig
 from configs.extra.notion_config import NotionConfig
 from configs.extra.sentry_config import SentryConfig
 
 
 class ExtraServiceConfig(
     # place the configs in alphabet order
+    ArchiveStorageConfig,
     NotionConfig,
     SentryConfig,
 ):

api/configs/extra/archive_config.py

@@ -0,0 +1,43 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ArchiveStorageConfig(BaseSettings):
+    """
+    Configuration settings for workflow run logs archiving storage.
+    """
+
+    ARCHIVE_STORAGE_ENABLED: bool = Field(
+        description="Enable workflow run logs archiving to S3-compatible storage",
+        default=False,
+    )
+
+    ARCHIVE_STORAGE_ENDPOINT: str | None = Field(
+        description="URL of the S3-compatible storage endpoint (e.g., 'https://storage.example.com')",
+        default=None,
+    )
+
+    ARCHIVE_STORAGE_ARCHIVE_BUCKET: str | None = Field(
+        description="Name of the bucket to store archived workflow logs",
+        default=None,
+    )
+
+    ARCHIVE_STORAGE_EXPORT_BUCKET: str | None = Field(
+        description="Name of the bucket to store exported workflow runs",
+        default=None,
+    )
+
+    ARCHIVE_STORAGE_ACCESS_KEY: str | None = Field(
+        description="Access key ID for authenticating with storage",
+        default=None,
+    )
+
+    ARCHIVE_STORAGE_SECRET_KEY: str | None = Field(
+        description="Secret access key for authenticating with storage",
+        default=None,
+    )
+
+    ARCHIVE_STORAGE_REGION: str = Field(
+        description="Region for storage (use 'auto' if the provider supports it)",
+        default="auto",
+    )

api/configs/feature/__init__.py

@@ -587,6 +587,11 @@ class LoggingConfig(BaseSettings):
         default="INFO",
     )
 
+    LOG_OUTPUT_FORMAT: Literal["text", "json"] = Field(
+        description="Log output format: 'text' for human-readable, 'json' for structured JSON logs.",
+        default="text",
+    )
+
     LOG_FILE: str | None = Field(
         description="File path for log output.",
         default=None,

api/configs/middleware/storage/tencent_cos_storage_config.py

@@ -31,3 +31,8 @@ class TencentCloudCOSStorageConfig(BaseSettings):
         description="Protocol scheme for COS requests: 'https' (recommended) or 'http'",
         default=None,
     )
+
+    TENCENT_COS_CUSTOM_DOMAIN: str | None = Field(
+        description="Tencent Cloud COS custom domain setting",
+        default=None,
+    )

api/configs/middleware/vdb/milvus_config.py

@@ -16,7 +16,6 @@ class MilvusConfig(BaseSettings):
         description="Authentication token for Milvus, if token-based authentication is enabled",
         default=None,
     )
-
     MILVUS_USER: str | None = Field(
         description="Username for authenticating with Milvus, if username/password authentication is enabled",
         default=None,

api/controllers/common/fields.py

@@ -1,62 +1,59 @@
-from flask_restx import Api, Namespace, fields
+from __future__ import annotations
 
-from libs.helper import AppIconUrlField
+from typing import Any, TypeAlias
 
-parameters__system_parameters = {
-    "image_file_size_limit": fields.Integer,
-    "video_file_size_limit": fields.Integer,
-    "audio_file_size_limit": fields.Integer,
-    "file_size_limit": fields.Integer,
-    "workflow_file_upload_limit": fields.Integer,
-}
+from pydantic import BaseModel, ConfigDict, computed_field
+
+from core.file import helpers as file_helpers
+from models.model import IconType
+
+JSONValue: TypeAlias = str | int | float | bool | None | dict[str, Any] | list[Any]
+JSONObject: TypeAlias = dict[str, Any]
 
 
-def build_system_parameters_model(api_or_ns: Api | Namespace):
-    """Build the system parameters model for the API or Namespace."""
-    return api_or_ns.model("SystemParameters", parameters__system_parameters)
+class SystemParameters(BaseModel):
+    image_file_size_limit: int
+    video_file_size_limit: int
+    audio_file_size_limit: int
+    file_size_limit: int
+    workflow_file_upload_limit: int
 
 
-parameters_fields = {
-    "opening_statement": fields.String,
-    "suggested_questions": fields.Raw,
-    "suggested_questions_after_answer": fields.Raw,
-    "speech_to_text": fields.Raw,
-    "text_to_speech": fields.Raw,
-    "retriever_resource": fields.Raw,
-    "annotation_reply": fields.Raw,
-    "more_like_this": fields.Raw,
-    "user_input_form": fields.Raw,
-    "sensitive_word_avoidance": fields.Raw,
-    "file_upload": fields.Raw,
-    "system_parameters": fields.Nested(parameters__system_parameters),
-}
+class Parameters(BaseModel):
+    opening_statement: str | None = None
+    suggested_questions: list[str]
+    suggested_questions_after_answer: JSONObject
+    speech_to_text: JSONObject
+    text_to_speech: JSONObject
+    retriever_resource: JSONObject
+    annotation_reply: JSONObject
+    more_like_this: JSONObject
+    user_input_form: list[JSONObject]
+    sensitive_word_avoidance: JSONObject
+    file_upload: JSONObject
+    system_parameters: SystemParameters
 
 
-def build_parameters_model(api_or_ns: Api | Namespace):
-    """Build the parameters model for the API or Namespace."""
-    copied_fields = parameters_fields.copy()
-    copied_fields["system_parameters"] = fields.Nested(build_system_parameters_model(api_or_ns))
-    return api_or_ns.model("Parameters", copied_fields)
+class Site(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
 
+    title: str
+    chat_color_theme: str | None = None
+    chat_color_theme_inverted: bool
+    icon_type: str | None = None
+    icon: str | None = None
+    icon_background: str | None = None
+    description: str | None = None
+    copyright: str | None = None
+    privacy_policy: str | None = None
+    custom_disclaimer: str | None = None
+    default_language: str
+    show_workflow_steps: bool
+    use_icon_as_answer_icon: bool
 
-site_fields = {
-    "title": fields.String,
-    "chat_color_theme": fields.String,
-    "chat_color_theme_inverted": fields.Boolean,
-    "icon_type": fields.String,
-    "icon": fields.String,
-    "icon_background": fields.String,
-    "icon_url": AppIconUrlField,
-    "description": fields.String,
-    "copyright": fields.String,
-    "privacy_policy": fields.String,
-    "custom_disclaimer": fields.String,
-    "default_language": fields.String,
-    "show_workflow_steps": fields.Boolean,
-    "use_icon_as_answer_icon": fields.Boolean,
-}
-
-
-def build_site_model(api_or_ns: Api | Namespace):
-    """Build the site model for the API or Namespace."""
-    return api_or_ns.model("Site", site_fields)
+    @computed_field(return_type=str | None)  # type: ignore
+    @property
+    def icon_url(self) -> str | None:
+        if self.icon and self.icon_type == IconType.IMAGE:
+            return file_helpers.get_signed_file_url(self.icon)
+        return None

api/controllers/console/app/app.py

@@ -1,3 +1,4 @@
+import re
 import uuid
 from typing import Literal
 
@@ -73,6 +74,48 @@ class AppListQuery(BaseModel):
             raise ValueError("Invalid UUID format in tag_ids.") from exc
 
 
+# XSS prevention: patterns that could lead to XSS attacks
+# Includes: script tags, iframe tags, javascript: protocol, SVG with onload, etc.
+_XSS_PATTERNS = [
+    r"<script[^>]*>.*?</script>",  # Script tags
+    r"<iframe\b[^>]*?(?:/>|>.*?</iframe>)",  # Iframe tags (including self-closing)
+    r"javascript:",  # JavaScript protocol
+    r"<svg[^>]*?\s+onload\s*=[^>]*>",  # SVG with onload handler (attribute-aware, flexible whitespace)
+    r"<.*?on\s*\w+\s*=",  # Event handlers like onclick, onerror, etc.
+    r"<object\b[^>]*(?:\s*/>|>.*?</object\s*>)",  # Object tags (opening tag)
+    r"<embed[^>]*>",  # Embed tags (self-closing)
+    r"<link[^>]*>",  # Link tags with javascript
+]
+
+
+def _validate_xss_safe(value: str | None, field_name: str = "Field") -> str | None:
+    """
+    Validate that a string value doesn't contain potential XSS payloads.
+
+    Args:
+        value: The string value to validate
+        field_name: Name of the field for error messages
+
+    Returns:
+        The original value if safe
+
+    Raises:
+        ValueError: If the value contains XSS patterns
+    """
+    if value is None:
+        return None
+
+    value_lower = value.lower()
+    for pattern in _XSS_PATTERNS:
+        if re.search(pattern, value_lower, re.DOTALL | re.IGNORECASE):
+            raise ValueError(
+                f"{field_name} contains invalid characters or patterns. "
+                "HTML tags, JavaScript, and other potentially dangerous content are not allowed."
+            )
+
+    return value
+
+
 class CreateAppPayload(BaseModel):
     name: str = Field(..., min_length=1, description="App name")
     description: str | None = Field(default=None, description="App description (max 400 chars)", max_length=400)
@@ -81,6 +124,11 @@ class CreateAppPayload(BaseModel):
     icon: str | None = Field(default=None, description="Icon")
     icon_background: str | None = Field(default=None, description="Icon background color")
 
+    @field_validator("name", "description", mode="before")
+    @classmethod
+    def validate_xss_safe(cls, value: str | None, info) -> str | None:
+        return _validate_xss_safe(value, info.field_name)
+
 
 class UpdateAppPayload(BaseModel):
     name: str = Field(..., min_length=1, description="App name")
@@ -91,6 +139,11 @@ class UpdateAppPayload(BaseModel):
     use_icon_as_answer_icon: bool | None = Field(default=None, description="Use icon as answer icon")
     max_active_requests: int | None = Field(default=None, description="Maximum active requests")
 
+    @field_validator("name", "description", mode="before")
+    @classmethod
+    def validate_xss_safe(cls, value: str | None, info) -> str | None:
+        return _validate_xss_safe(value, info.field_name)
+
 
 class CopyAppPayload(BaseModel):
     name: str | None = Field(default=None, description="Name for the copied app")
@@ -99,6 +152,11 @@ class CopyAppPayload(BaseModel):
     icon: str | None = Field(default=None, description="Icon")
     icon_background: str | None = Field(default=None, description="Icon background color")
 
+    @field_validator("name", "description", mode="before")
+    @classmethod
+    def validate_xss_safe(cls, value: str | None, info) -> str | None:
+        return _validate_xss_safe(value, info.field_name)
+
 
 class AppExportQuery(BaseModel):
     include_secret: bool = Field(default=False, description="Include secrets in export")

api/controllers/console/app/conversation.py

@@ -13,7 +13,6 @@ from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
-from fields.conversation_fields import MessageTextField
 from fields.raws import FilesContainedField
 from libs.datetime_utils import naive_utc_now, parse_time_range
 from libs.helper import TimestampField
@@ -177,6 +176,12 @@ annotation_hit_history_model = console_ns.model(
     },
 )
 
+
+class MessageTextField(fields.Raw):
+    def format(self, value):
+        return value[0]["text"] if value else ""
+
+
 # Simple message detail model
 simple_message_detail_model = console_ns.model(
     "SimpleMessageDetail",

api/controllers/console/auth/oauth.py

@@ -124,7 +124,7 @@ class OAuthCallback(Resource):
             return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin/invite-settings?invite_token={invite_token}")
 
         try:
-            account = _generate_account(provider, user_info)
+            account, oauth_new_user = _generate_account(provider, user_info)
         except AccountNotFoundError:
             return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Account not found.")
         except (WorkSpaceNotFoundError, WorkSpaceNotAllowedCreateError):
@@ -159,7 +159,10 @@ class OAuthCallback(Resource):
             ip_address=extract_remote_ip(request),
         )
 
-        response = redirect(f"{dify_config.CONSOLE_WEB_URL}")
+        base_url = dify_config.CONSOLE_WEB_URL
+        query_char = "&" if "?" in base_url else "?"
+        target_url = f"{base_url}{query_char}oauth_new_user={str(oauth_new_user).lower()}"
+        response = redirect(target_url)
 
         set_access_token_to_cookie(request, response, token_pair.access_token)
         set_refresh_token_to_cookie(request, response, token_pair.refresh_token)
@@ -177,9 +180,10 @@ def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) ->
     return account
 
 
-def _generate_account(provider: str, user_info: OAuthUserInfo):
+def _generate_account(provider: str, user_info: OAuthUserInfo) -> tuple[Account, bool]:
     # Get account by openid or email.
     account = _get_account_by_openid_or_email(provider, user_info)
+    oauth_new_user = False
 
     if account:
         tenants = TenantService.get_join_tenants(account)
@@ -193,6 +197,7 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
                 tenant_was_created.send(new_tenant)
 
     if not account:
+        oauth_new_user = True
         if not FeatureService.get_system_features().is_allow_register:
             if dify_config.BILLING_ENABLED and BillingService.is_email_in_freeze(user_info.email):
                 raise AccountRegisterError(
@@ -220,4 +225,4 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
     # Link account
     AccountService.link_account_integrate(provider, user_info.id, account)
 
-    return account
+    return account, oauth_new_user

api/controllers/console/billing/billing.py

@@ -1,8 +1,9 @@
 import base64
+from typing import Literal
 
 from flask import request
 from flask_restx import Resource, fields
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field
 from werkzeug.exceptions import BadRequest
 
 from controllers.console import console_ns
@@ -15,22 +16,8 @@ DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
 
 class SubscriptionQuery(BaseModel):
-    plan: str = Field(..., description="Subscription plan")
-    interval: str = Field(..., description="Billing interval")
-
-    @field_validator("plan")
-    @classmethod
-    def validate_plan(cls, value: str) -> str:
-        if value not in [CloudPlan.PROFESSIONAL, CloudPlan.TEAM]:
-            raise ValueError("Invalid plan")
-        return value
-
-    @field_validator("interval")
-    @classmethod
-    def validate_interval(cls, value: str) -> str:
-        if value not in {"month", "year"}:
-            raise ValueError("Invalid interval")
-        return value
+    plan: Literal[CloudPlan.PROFESSIONAL, CloudPlan.TEAM] = Field(..., description="Subscription plan")
+    interval: Literal["month", "year"] = Field(..., description="Billing interval")
 
 
 class PartnerTenantsPayload(BaseModel):

api/controllers/console/datasets/datasets_document.py

@@ -751,12 +751,12 @@ class DocumentApi(DocumentResource):
         elif metadata == "without":
             dataset_process_rules = DatasetService.get_process_rules(dataset_id)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
-            data_source_info = document.data_source_detail_dict
             response = {
                 "id": document.id,
                 "position": document.position,
                 "data_source_type": document.data_source_type,
-                "data_source_info": data_source_info,
+                "data_source_info": document.data_source_info_dict,
+                "data_source_detail_dict": document.data_source_detail_dict,
                 "dataset_process_rule_id": document.dataset_process_rule_id,
                 "dataset_process_rule": dataset_process_rules,
                 "document_process_rule": document_process_rules,
@@ -784,12 +784,12 @@ class DocumentApi(DocumentResource):
         else:
             dataset_process_rules = DatasetService.get_process_rules(dataset_id)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
-            data_source_info = document.data_source_detail_dict
             response = {
                 "id": document.id,
                 "position": document.position,
                 "data_source_type": document.data_source_type,
-                "data_source_info": data_source_info,
+                "data_source_info": document.data_source_info_dict,
+                "data_source_detail_dict": document.data_source_detail_dict,
                 "dataset_process_rule_id": document.dataset_process_rule_id,
                 "dataset_process_rule": dataset_process_rules,
                 "document_process_rule": document_process_rules,

api/controllers/console/datasets/datasets_segments.py

@@ -3,10 +3,12 @@ import uuid
 from flask import request
 from flask_restx import Resource, marshal
 from pydantic import BaseModel, Field
-from sqlalchemy import select
+from sqlalchemy import String, cast, func, or_, select
+from sqlalchemy.dialects.postgresql import JSONB
 from werkzeug.exceptions import Forbidden, NotFound
 
 import services
+from configs import dify_config
 from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.app.error import ProviderNotInitializeError
@@ -143,7 +145,29 @@ class DatasetDocumentSegmentListApi(Resource):
             query = query.where(DocumentSegment.hit_count >= hit_count_gte)
 
         if keyword:
-            query = query.where(DocumentSegment.content.ilike(f"%{keyword}%"))
+            # Search in both content and keywords fields
+            # Use database-specific methods for JSON array search
+            if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql":
+                # PostgreSQL: Use jsonb_array_elements_text to properly handle Unicode/Chinese text
+                keywords_condition = func.array_to_string(
+                    func.array(
+                        select(func.jsonb_array_elements_text(cast(DocumentSegment.keywords, JSONB)))
+                        .correlate(DocumentSegment)
+                        .scalar_subquery()
+                    ),
+                    ",",
+                ).ilike(f"%{keyword}%")
+            else:
+                # MySQL: Cast JSON to string for pattern matching
+                # MySQL stores Chinese text directly in JSON without Unicode escaping
+                keywords_condition = cast(DocumentSegment.keywords, String).ilike(f"%{keyword}%")
+
+            query = query.where(
+                or_(
+                    DocumentSegment.content.ilike(f"%{keyword}%"),
+                    keywords_condition,
+                )
+            )
 
         if args.enabled.lower() != "all":
             if args.enabled.lower() == "true":

api/controllers/console/explore/conversation.py

@@ -1,8 +1,7 @@
 from typing import Any
 
 from flask import request
-from flask_restx import marshal_with
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, TypeAdapter, model_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 
@@ -11,7 +10,11 @@ from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
-from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
+from fields.conversation_fields import (
+    ConversationInfiniteScrollPagination,
+    ResultResponse,
+    SimpleConversation,
+)
 from libs.helper import UUIDStrOrEmpty
 from libs.login import current_user
 from models import Account
@@ -49,7 +52,6 @@ register_schema_models(console_ns, ConversationListQuery, ConversationRenamePayl
     endpoint="installed_app_conversations",
 )
 class ConversationListApi(InstalledAppResource):
-    @marshal_with(conversation_infinite_scroll_pagination_fields)
     @console_ns.expect(console_ns.models[ConversationListQuery.__name__])
     def get(self, installed_app):
         app_model = installed_app.app
@@ -73,7 +75,7 @@ class ConversationListApi(InstalledAppResource):
             if not isinstance(current_user, Account):
                 raise ValueError("current_user must be an Account instance")
             with Session(db.engine) as session:
-                return WebConversationService.pagination_by_last_id(
+                pagination = WebConversationService.pagination_by_last_id(
                     session=session,
                     app_model=app_model,
                     user=current_user,
@@ -82,6 +84,13 @@ class ConversationListApi(InstalledAppResource):
                     invoke_from=InvokeFrom.EXPLORE,
                     pinned=args.pinned,
                 )
+                adapter = TypeAdapter(SimpleConversation)
+                conversations = [adapter.validate_python(item, from_attributes=True) for item in pagination.data]
+                return ConversationInfiniteScrollPagination(
+                    limit=pagination.limit,
+                    has_more=pagination.has_more,
+                    data=conversations,
+                ).model_dump(mode="json")
         except LastConversationNotExistsError:
             raise NotFound("Last Conversation Not Exists.")
 
@@ -105,7 +114,7 @@ class ConversationApi(InstalledAppResource):
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 
-        return {"result": "success"}, 204
+        return ResultResponse(result="success").model_dump(mode="json"), 204
 
 
 @console_ns.route(
@@ -113,7 +122,6 @@ class ConversationApi(InstalledAppResource):
     endpoint="installed_app_conversation_rename",
 )
 class ConversationRenameApi(InstalledAppResource):
-    @marshal_with(simple_conversation_fields)
     @console_ns.expect(console_ns.models[ConversationRenamePayload.__name__])
     def post(self, installed_app, c_id):
         app_model = installed_app.app
@@ -128,9 +136,14 @@ class ConversationRenameApi(InstalledAppResource):
         try:
             if not isinstance(current_user, Account):
                 raise ValueError("current_user must be an Account instance")
-            return ConversationService.rename(
+            conversation = ConversationService.rename(
                 app_model, conversation_id, current_user, payload.name, payload.auto_generate
             )
+            return (
+                TypeAdapter(SimpleConversation)
+                .validate_python(conversation, from_attributes=True)
+                .model_dump(mode="json")
+            )
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 
@@ -155,7 +168,7 @@ class ConversationPinApi(InstalledAppResource):
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @console_ns.route(
@@ -174,4 +187,4 @@ class ConversationUnPinApi(InstalledAppResource):
             raise ValueError("current_user must be an Account instance")
         WebConversationService.unpin(app_model, conversation_id, current_user)
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")

api/controllers/console/explore/message.py

@@ -1,10 +1,8 @@
 import logging
 from typing import Literal
-from uuid import UUID
 
 from flask import request
-from flask_restx import marshal_with
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, TypeAdapter
 from werkzeug.exceptions import InternalServerError, NotFound
 
 from controllers.common.schema import register_schema_models
@@ -24,8 +22,10 @@ from controllers.console.explore.wraps import InstalledAppResource
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from fields.message_fields import message_infinite_scroll_pagination_fields
+from fields.conversation_fields import ResultResponse
+from fields.message_fields import MessageInfiniteScrollPagination, MessageListItem, SuggestedQuestionsResponse
 from libs import helper
+from libs.helper import UUIDStrOrEmpty
 from libs.login import current_account_with_tenant
 from models.model import AppMode
 from services.app_generate_service import AppGenerateService
@@ -44,8 +44,8 @@ logger = logging.getLogger(__name__)
 
 
 class MessageListQuery(BaseModel):
-    conversation_id: UUID
-    first_id: UUID | None = None
+    conversation_id: UUIDStrOrEmpty
+    first_id: UUIDStrOrEmpty | None = None
     limit: int = Field(default=20, ge=1, le=100)
 
 
@@ -66,7 +66,6 @@ register_schema_models(console_ns, MessageListQuery, MessageFeedbackPayload, Mor
     endpoint="installed_app_messages",
 )
 class MessageListApi(InstalledAppResource):
-    @marshal_with(message_infinite_scroll_pagination_fields)
     @console_ns.expect(console_ns.models[MessageListQuery.__name__])
     def get(self, installed_app):
         current_user, _ = current_account_with_tenant()
@@ -78,13 +77,20 @@ class MessageListApi(InstalledAppResource):
         args = MessageListQuery.model_validate(request.args.to_dict())
 
         try:
-            return MessageService.pagination_by_first_id(
+            pagination = MessageService.pagination_by_first_id(
                 app_model,
                 current_user,
                 str(args.conversation_id),
                 str(args.first_id) if args.first_id else None,
                 args.limit,
             )
+            adapter = TypeAdapter(MessageListItem)
+            items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data]
+            return MessageInfiniteScrollPagination(
+                limit=pagination.limit,
+                has_more=pagination.has_more,
+                data=items,
+            ).model_dump(mode="json")
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except FirstMessageNotExistsError:
@@ -116,7 +122,7 @@ class MessageFeedbackApi(InstalledAppResource):
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @console_ns.route(
@@ -201,4 +207,4 @@ class MessageSuggestedQuestionApi(InstalledAppResource):
             logger.exception("internal server error.")
             raise InternalServerError()
 
-        return {"data": questions}
+        return SuggestedQuestionsResponse(data=questions).model_dump(mode="json")

api/controllers/console/explore/parameter.py

@@ -1,5 +1,3 @@
-from flask_restx import marshal_with
-
 from controllers.common import fields
 from controllers.console import console_ns
 from controllers.console.app.error import AppUnavailableError
@@ -13,7 +11,6 @@ from services.app_service import AppService
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""
 
-    @marshal_with(fields.parameters_fields)
     def get(self, installed_app: InstalledApp):
         """Retrieve app parameters."""
         app_model = installed_app.app
@@ -37,7 +34,8 @@ class AppParameterApi(InstalledAppResource):
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        parameters = get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        return fields.Parameters.model_validate(parameters).model_dump(mode="json")
 
 
 @console_ns.route("/installed-apps/<uuid:installed_app_id>/meta", endpoint="installed_app_meta")

api/controllers/console/explore/saved_message.py

@@ -1,55 +1,33 @@
-from uuid import UUID
-
 from flask import request
-from flask_restx import fields, marshal_with
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, TypeAdapter
 from werkzeug.exceptions import NotFound
 
 from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.explore.error import NotCompletionAppError
 from controllers.console.explore.wraps import InstalledAppResource
-from fields.conversation_fields import message_file_fields
-from libs.helper import TimestampField
+from fields.conversation_fields import ResultResponse
+from fields.message_fields import SavedMessageInfiniteScrollPagination, SavedMessageItem
+from libs.helper import UUIDStrOrEmpty
 from libs.login import current_account_with_tenant
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
 
 
 class SavedMessageListQuery(BaseModel):
-    last_id: UUID | None = None
+    last_id: UUIDStrOrEmpty | None = None
     limit: int = Field(default=20, ge=1, le=100)
 
 
 class SavedMessageCreatePayload(BaseModel):
-    message_id: UUID
+    message_id: UUIDStrOrEmpty
 
 
 register_schema_models(console_ns, SavedMessageListQuery, SavedMessageCreatePayload)
 
 
-feedback_fields = {"rating": fields.String}
-
-message_fields = {
-    "id": fields.String,
-    "inputs": fields.Raw,
-    "query": fields.String,
-    "answer": fields.String,
-    "message_files": fields.List(fields.Nested(message_file_fields)),
-    "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True),
-    "created_at": TimestampField,
-}
-
-
 @console_ns.route("/installed-apps/<uuid:installed_app_id>/saved-messages", endpoint="installed_app_saved_messages")
 class SavedMessageListApi(InstalledAppResource):
-    saved_message_infinite_scroll_pagination_fields = {
-        "limit": fields.Integer,
-        "has_more": fields.Boolean,
-        "data": fields.List(fields.Nested(message_fields)),
-    }
-
-    @marshal_with(saved_message_infinite_scroll_pagination_fields)
     @console_ns.expect(console_ns.models[SavedMessageListQuery.__name__])
     def get(self, installed_app):
         current_user, _ = current_account_with_tenant()
@@ -59,12 +37,19 @@ class SavedMessageListApi(InstalledAppResource):
 
         args = SavedMessageListQuery.model_validate(request.args.to_dict())
 
-        return SavedMessageService.pagination_by_last_id(
+        pagination = SavedMessageService.pagination_by_last_id(
             app_model,
             current_user,
             str(args.last_id) if args.last_id else None,
             args.limit,
         )
+        adapter = TypeAdapter(SavedMessageItem)
+        items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data]
+        return SavedMessageInfiniteScrollPagination(
+            limit=pagination.limit,
+            has_more=pagination.has_more,
+            data=items,
+        ).model_dump(mode="json")
 
     @console_ns.expect(console_ns.models[SavedMessageCreatePayload.__name__])
     def post(self, installed_app):
@@ -80,7 +65,7 @@ class SavedMessageListApi(InstalledAppResource):
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @console_ns.route(
@@ -98,4 +83,4 @@ class SavedMessageApi(InstalledAppResource):
 
         SavedMessageService.delete(app_model, current_user, message_id)
 
-        return {"result": "success"}, 204
+        return ResultResponse(result="success").model_dump(mode="json"), 204

api/controllers/console/workspace/load_balancing_config.py

@@ -1,6 +1,8 @@
-from flask_restx import Resource, reqparse
+from flask_restx import Resource
+from pydantic import BaseModel
 from werkzeug.exceptions import Forbidden
 
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, setup_required
 from core.model_runtime.entities.model_entities import ModelType
@@ -10,10 +12,20 @@ from models import TenantAccountRole
 from services.model_load_balancing_service import ModelLoadBalancingService
 
 
+class LoadBalancingCredentialPayload(BaseModel):
+    model: str
+    model_type: ModelType
+    credentials: dict[str, object]
+
+
+register_schema_models(console_ns, LoadBalancingCredentialPayload)
+
+
 @console_ns.route(
     "/workspaces/current/model-providers/<path:provider>/models/load-balancing-configs/credentials-validate"
 )
 class LoadBalancingCredentialsValidateApi(Resource):
+    @console_ns.expect(console_ns.models[LoadBalancingCredentialPayload.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -24,20 +36,7 @@ class LoadBalancingCredentialsValidateApi(Resource):
 
         tenant_id = current_tenant_id
 
-        parser = (
-            reqparse.RequestParser()
-            .add_argument("model", type=str, required=True, nullable=False, location="json")
-            .add_argument(
-                "model_type",
-                type=str,
-                required=True,
-                nullable=False,
-                choices=[mt.value for mt in ModelType],
-                location="json",
-            )
-            .add_argument("credentials", type=dict, required=True, nullable=False, location="json")
-        )
-        args = parser.parse_args()
+        payload = LoadBalancingCredentialPayload.model_validate(console_ns.payload or {})
 
         # validate model load balancing credentials
         model_load_balancing_service = ModelLoadBalancingService()
@@ -49,9 +48,9 @@ class LoadBalancingCredentialsValidateApi(Resource):
             model_load_balancing_service.validate_load_balancing_credentials(
                 tenant_id=tenant_id,
                 provider=provider,
-                model=args["model"],
-                model_type=args["model_type"],
-                credentials=args["credentials"],
+                model=payload.model,
+                model_type=payload.model_type,
+                credentials=payload.credentials,
             )
         except CredentialsValidateFailedError as ex:
             result = False
@@ -69,6 +68,7 @@ class LoadBalancingCredentialsValidateApi(Resource):
     "/workspaces/current/model-providers/<path:provider>/models/load-balancing-configs/<string:config_id>/credentials-validate"
 )
 class LoadBalancingConfigCredentialsValidateApi(Resource):
+    @console_ns.expect(console_ns.models[LoadBalancingCredentialPayload.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -79,20 +79,7 @@ class LoadBalancingConfigCredentialsValidateApi(Resource):
 
         tenant_id = current_tenant_id
 
-        parser = (
-            reqparse.RequestParser()
-            .add_argument("model", type=str, required=True, nullable=False, location="json")
-            .add_argument(
-                "model_type",
-                type=str,
-                required=True,
-                nullable=False,
-                choices=[mt.value for mt in ModelType],
-                location="json",
-            )
-            .add_argument("credentials", type=dict, required=True, nullable=False, location="json")
-        )
-        args = parser.parse_args()
+        payload = LoadBalancingCredentialPayload.model_validate(console_ns.payload or {})
 
         # validate model load balancing config credentials
         model_load_balancing_service = ModelLoadBalancingService()
@@ -104,9 +91,9 @@ class LoadBalancingConfigCredentialsValidateApi(Resource):
             model_load_balancing_service.validate_load_balancing_credentials(
                 tenant_id=tenant_id,
                 provider=provider,
-                model=args["model"],
-                model_type=args["model_type"],
-                credentials=args["credentials"],
+                model=payload.model,
+                model_type=payload.model_type,
+                credentials=payload.credentials,
                 config_id=config_id,
             )
         except CredentialsValidateFailedError as ex:

api/controllers/console/workspace/tool_providers.py

@@ -1,4 +1,5 @@
 import io
+import logging
 from urllib.parse import urlparse
 
 from flask import make_response, redirect, request, send_file
@@ -17,8 +18,8 @@ from controllers.console.wraps import (
     is_admin_or_owner_required,
     setup_required,
 )
+from core.db.session_factory import session_factory
 from core.entities.mcp_provider import MCPAuthentication, MCPConfiguration
-from core.helper.tool_provider_cache import ToolProviderListCache
 from core.mcp.auth.auth_flow import auth, handle_callback
 from core.mcp.error import MCPAuthError, MCPError, MCPRefreshTokenError
 from core.mcp.mcp_client import MCPClient
@@ -40,6 +41,8 @@ from services.tools.tools_manage_service import ToolCommonService
 from services.tools.tools_transform_service import ToolTransformService
 from services.tools.workflow_tools_manage_service import WorkflowToolManageService
 
+logger = logging.getLogger(__name__)
+
 
 def is_valid_url(url: str) -> bool:
     if not url:
@@ -945,8 +948,8 @@ class ToolProviderMCPApi(Resource):
         configuration = MCPConfiguration.model_validate(args["configuration"])
         authentication = MCPAuthentication.model_validate(args["authentication"]) if args["authentication"] else None
 
-        # Create provider in transaction
-        with Session(db.engine) as session, session.begin():
+        # 1) Create provider in a short transaction (no network I/O inside)
+        with session_factory.create_session() as session, session.begin():
             service = MCPToolManageService(session=session)
             result = service.create_provider(
                 tenant_id=tenant_id,
@@ -962,8 +965,26 @@ class ToolProviderMCPApi(Resource):
                 authentication=authentication,
             )
 
-        # Invalidate cache AFTER transaction commits to avoid holding locks during Redis operations
-        ToolProviderListCache.invalidate_cache(tenant_id)
+        # 2) Try to fetch tools immediately after creation so they appear without a second save.
+        #    Perform network I/O outside any DB session to avoid holding locks.
+        try:
+            reconnect = MCPToolManageService.reconnect_with_url(
+                server_url=args["server_url"],
+                headers=args.get("headers") or {},
+                timeout=configuration.timeout,
+                sse_read_timeout=configuration.sse_read_timeout,
+            )
+            # Update just-created provider with authed/tools in a new short transaction
+            with session_factory.create_session() as session, session.begin():
+                service = MCPToolManageService(session=session)
+                db_provider = service.get_provider(provider_id=result.id, tenant_id=tenant_id)
+                db_provider.authed = reconnect.authed
+                db_provider.tools = reconnect.tools
+
+                result = ToolTransformService.mcp_provider_to_user_provider(db_provider, for_list=True)
+        except Exception:
+            # Best-effort: if initial fetch fails (e.g., auth required), return created provider as-is
+            logger.warning("Failed to fetch MCP tools after creation", exc_info=True)
 
         return jsonable_encoder(result)
 
@@ -1011,9 +1032,6 @@ class ToolProviderMCPApi(Resource):
                 validation_result=validation_result,
             )
 
-        # Invalidate cache AFTER transaction commits to avoid holding locks during Redis operations
-        ToolProviderListCache.invalidate_cache(current_tenant_id)
-
         return {"result": "success"}
 
     @console_ns.expect(parser_mcp_delete)
@@ -1028,9 +1046,6 @@ class ToolProviderMCPApi(Resource):
             service = MCPToolManageService(session=session)
             service.delete_provider(tenant_id=current_tenant_id, provider_id=args["provider_id"])
 
-        # Invalidate cache AFTER transaction commits to avoid holding locks during Redis operations
-        ToolProviderListCache.invalidate_cache(current_tenant_id)
-
         return {"result": "success"}
 
 
@@ -1081,8 +1096,6 @@ class ToolMCPAuthApi(Resource):
                         credentials=provider_entity.credentials,
                         authed=True,
                     )
-                # Invalidate cache after updating credentials
-                ToolProviderListCache.invalidate_cache(tenant_id)
                 return {"result": "success"}
         except MCPAuthError as e:
             try:
@@ -1096,22 +1109,16 @@ class ToolMCPAuthApi(Resource):
                 with Session(db.engine) as session, session.begin():
                     service = MCPToolManageService(session=session)
                     response = service.execute_auth_actions(auth_result)
-                    # Invalidate cache after auth actions may have updated provider state
-                    ToolProviderListCache.invalidate_cache(tenant_id)
                     return response
             except MCPRefreshTokenError as e:
                 with Session(db.engine) as session, session.begin():
                     service = MCPToolManageService(session=session)
                     service.clear_provider_credentials(provider_id=provider_id, tenant_id=tenant_id)
-                # Invalidate cache after clearing credentials
-                ToolProviderListCache.invalidate_cache(tenant_id)
                 raise ValueError(f"Failed to refresh token, please try to authorize again: {e}") from e
         except (MCPError, ValueError) as e:
             with Session(db.engine) as session, session.begin():
                 service = MCPToolManageService(session=session)
                 service.clear_provider_credentials(provider_id=provider_id, tenant_id=tenant_id)
-            # Invalidate cache after clearing credentials
-            ToolProviderListCache.invalidate_cache(tenant_id)
             raise ValueError(f"Failed to connect to MCP server: {e}") from e
 
 

api/controllers/console/workspace/trigger_providers.py

@@ -4,12 +4,11 @@ from typing import Any
 
 from flask import make_response, redirect, request
 from flask_restx import Resource, reqparse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, Forbidden
 
 from configs import dify_config
-from constants import HIDDEN_VALUE, UNKNOWN_VALUE
 from controllers.web.error import NotFoundError
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.plugin.entities.plugin_daemon import CredentialType
@@ -44,6 +43,12 @@ class TriggerSubscriptionUpdateRequest(BaseModel):
     parameters: Mapping[str, Any] | None = Field(default=None, description="The parameters for the subscription")
     properties: Mapping[str, Any] | None = Field(default=None, description="The properties for the subscription")
 
+    @model_validator(mode="after")
+    def check_at_least_one_field(self):
+        if all(v is None for v in (self.name, self.credentials, self.parameters, self.properties)):
+            raise ValueError("At least one of name, credentials, parameters, or properties must be provided")
+        return self
+
 
 class TriggerSubscriptionVerifyRequest(BaseModel):
     """Request payload for verifying subscription credentials."""
@@ -333,7 +338,7 @@ class TriggerSubscriptionUpdateApi(Resource):
         user = current_user
         assert user.current_tenant_id is not None
 
-        args = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload)
+        request = TriggerSubscriptionUpdateRequest.model_validate(console_ns.payload)
 
         subscription = TriggerProviderService.get_subscription_by_id(
             tenant_id=user.current_tenant_id,
@@ -345,50 +350,32 @@ class TriggerSubscriptionUpdateApi(Resource):
         provider_id = TriggerProviderID(subscription.provider_id)
 
         try:
-            # rename only
-            if (
-                args.name is not None
-                and args.credentials is None
-                and args.parameters is None
-                and args.properties is None
-            ):
+            # For rename only, just update the name
+            rename = request.name is not None and not any((request.credentials, request.parameters, request.properties))
+            # When credential type is UNAUTHORIZED, it indicates the subscription was manually created
+            # For Manually created subscription, they dont have credentials, parameters
+            # They only have name and properties(which is input by user)
+            manually_created = subscription.credential_type == CredentialType.UNAUTHORIZED
+            if rename or manually_created:
                 TriggerProviderService.update_trigger_subscription(
                     tenant_id=user.current_tenant_id,
                     subscription_id=subscription_id,
-                    name=args.name,
+                    name=request.name,
+                    properties=request.properties,
                 )
                 return 200
 
-            # rebuild for create automatically by the provider
-            match subscription.credential_type:
-                case CredentialType.UNAUTHORIZED:
-                    TriggerProviderService.update_trigger_subscription(
-                        tenant_id=user.current_tenant_id,
-                        subscription_id=subscription_id,
-                        name=args.name,
-                        properties=args.properties,
-                    )
-                    return 200
-                case CredentialType.API_KEY | CredentialType.OAUTH2:
-                    if args.credentials:
-                        new_credentials: dict[str, Any] = {
-                            key: value if value != HIDDEN_VALUE else subscription.credentials.get(key, UNKNOWN_VALUE)
-                            for key, value in args.credentials.items()
-                        }
-                    else:
-                        new_credentials = subscription.credentials
-
-                    TriggerProviderService.rebuild_trigger_subscription(
-                        tenant_id=user.current_tenant_id,
-                        name=args.name,
-                        provider_id=provider_id,
-                        subscription_id=subscription_id,
-                        credentials=new_credentials,
-                        parameters=args.parameters or subscription.parameters,
-                    )
-                    return 200
-                case _:
-                    raise BadRequest("Invalid credential type")
+            # For the rest cases(API_KEY, OAUTH2)
+            # we need to call third party provider(e.g. GitHub) to rebuild the subscription
+            TriggerProviderService.rebuild_trigger_subscription(
+                tenant_id=user.current_tenant_id,
+                name=request.name,
+                provider_id=provider_id,
+                subscription_id=subscription_id,
+                credentials=request.credentials or subscription.credentials,
+                parameters=request.parameters or subscription.parameters,
+            )
+            return 200
         except ValueError as e:
             raise BadRequest(str(e))
         except Exception as e:

api/controllers/service_api/app/annotation.py

@@ -1,7 +1,7 @@
 from typing import Literal
 
 from flask import request
-from flask_restx import Api, Namespace, Resource, fields
+from flask_restx import Namespace, Resource, fields
 from flask_restx.api import HTTPStatus
 from pydantic import BaseModel, Field
 
@@ -92,7 +92,7 @@ annotation_list_fields = {
 }
 
 
-def build_annotation_list_model(api_or_ns: Api | Namespace):
+def build_annotation_list_model(api_or_ns: Namespace):
     """Build the annotation list model for the API or Namespace."""
     copied_annotation_list_fields = annotation_list_fields.copy()
     copied_annotation_list_fields["data"] = fields.List(fields.Nested(build_annotation_model(api_or_ns)))

api/controllers/service_api/app/app.py

@@ -1,6 +1,6 @@
 from flask_restx import Resource
 
-from controllers.common.fields import build_parameters_model
+from controllers.common.fields import Parameters
 from controllers.service_api import service_api_ns
 from controllers.service_api.app.error import AppUnavailableError
 from controllers.service_api.wraps import validate_app_token
@@ -23,7 +23,6 @@ class AppParameterApi(Resource):
         }
     )
     @validate_app_token
-    @service_api_ns.marshal_with(build_parameters_model(service_api_ns))
     def get(self, app_model: App):
         """Retrieve app parameters.
 
@@ -45,7 +44,8 @@ class AppParameterApi(Resource):
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        parameters = get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        return Parameters.model_validate(parameters).model_dump(mode="json")
 
 
 @service_api_ns.route("/meta")

api/controllers/service_api/app/conversation.py

@@ -3,8 +3,7 @@ from uuid import UUID
 
 from flask import request
 from flask_restx import Resource
-from flask_restx._http import HTTPStatus
-from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic import BaseModel, Field, TypeAdapter, field_validator, model_validator
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import BadRequest, NotFound
 
@@ -16,9 +15,9 @@ from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
 from fields.conversation_fields import (
-    build_conversation_delete_model,
-    build_conversation_infinite_scroll_pagination_model,
-    build_simple_conversation_model,
+    ConversationDelete,
+    ConversationInfiniteScrollPagination,
+    SimpleConversation,
 )
 from fields.conversation_variable_fields import (
     build_conversation_variable_infinite_scroll_pagination_model,
@@ -105,7 +104,6 @@ class ConversationApi(Resource):
         }
     )
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.QUERY))
-    @service_api_ns.marshal_with(build_conversation_infinite_scroll_pagination_model(service_api_ns))
     def get(self, app_model: App, end_user: EndUser):
         """List all conversations for the current user.
 
@@ -120,7 +118,7 @@ class ConversationApi(Resource):
 
         try:
             with Session(db.engine) as session:
-                return ConversationService.pagination_by_last_id(
+                pagination = ConversationService.pagination_by_last_id(
                     session=session,
                     app_model=app_model,
                     user=end_user,
@@ -129,6 +127,13 @@ class ConversationApi(Resource):
                     invoke_from=InvokeFrom.SERVICE_API,
                     sort_by=query_args.sort_by,
                 )
+                adapter = TypeAdapter(SimpleConversation)
+                conversations = [adapter.validate_python(item, from_attributes=True) for item in pagination.data]
+                return ConversationInfiniteScrollPagination(
+                    limit=pagination.limit,
+                    has_more=pagination.has_more,
+                    data=conversations,
+                ).model_dump(mode="json")
         except services.errors.conversation.LastConversationNotExistsError:
             raise NotFound("Last Conversation Not Exists.")
 
@@ -146,7 +151,6 @@ class ConversationDetailApi(Resource):
         }
     )
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
-    @service_api_ns.marshal_with(build_conversation_delete_model(service_api_ns), code=HTTPStatus.NO_CONTENT)
     def delete(self, app_model: App, end_user: EndUser, c_id):
         """Delete a specific conversation."""
         app_mode = AppMode.value_of(app_model.mode)
@@ -159,7 +163,7 @@ class ConversationDetailApi(Resource):
             ConversationService.delete(app_model, conversation_id, end_user)
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
-        return {"result": "success"}, 204
+        return ConversationDelete(result="success").model_dump(mode="json"), 204
 
 
 @service_api_ns.route("/conversations/<uuid:c_id>/name")
@@ -176,7 +180,6 @@ class ConversationRenameApi(Resource):
         }
     )
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON))
-    @service_api_ns.marshal_with(build_simple_conversation_model(service_api_ns))
     def post(self, app_model: App, end_user: EndUser, c_id):
         """Rename a conversation or auto-generate a name."""
         app_mode = AppMode.value_of(app_model.mode)
@@ -188,7 +191,14 @@ class ConversationRenameApi(Resource):
         payload = ConversationRenamePayload.model_validate(service_api_ns.payload or {})
 
         try:
-            return ConversationService.rename(app_model, conversation_id, end_user, payload.name, payload.auto_generate)
+            conversation = ConversationService.rename(
+                app_model, conversation_id, end_user, payload.name, payload.auto_generate
+            )
+            return (
+                TypeAdapter(SimpleConversation)
+                .validate_python(conversation, from_attributes=True)
+                .model_dump(mode="json")
+            )
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 

api/controllers/service_api/app/message.py

@@ -1,11 +1,10 @@
-import json
 import logging
 from typing import Literal
 from uuid import UUID
 
 from flask import request
-from flask_restx import Namespace, Resource, fields
-from pydantic import BaseModel, Field
+from flask_restx import Resource
+from pydantic import BaseModel, Field, TypeAdapter
 from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
 
 import services
@@ -14,10 +13,8 @@ from controllers.service_api import service_api_ns
 from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
 from core.app.entities.app_invoke_entities import InvokeFrom
-from fields.conversation_fields import build_message_file_model
-from fields.message_fields import build_agent_thought_model, build_feedback_model
-from fields.raws import FilesContainedField
-from libs.helper import TimestampField
+from fields.conversation_fields import ResultResponse
+from fields.message_fields import MessageInfiniteScrollPagination, MessageListItem
 from models.model import App, AppMode, EndUser
 from services.errors.message import (
     FirstMessageNotExistsError,
@@ -48,49 +45,6 @@ class FeedbackListQuery(BaseModel):
 register_schema_models(service_api_ns, MessageListQuery, MessageFeedbackPayload, FeedbackListQuery)
 
 
-def build_message_model(api_or_ns: Namespace):
-    """Build the message model for the API or Namespace."""
-    # First build the nested models
-    feedback_model = build_feedback_model(api_or_ns)
-    agent_thought_model = build_agent_thought_model(api_or_ns)
-    message_file_model = build_message_file_model(api_or_ns)
-
-    # Then build the message fields with nested models
-    message_fields = {
-        "id": fields.String,
-        "conversation_id": fields.String,
-        "parent_message_id": fields.String,
-        "inputs": FilesContainedField,
-        "query": fields.String,
-        "answer": fields.String(attribute="re_sign_file_url_answer"),
-        "message_files": fields.List(fields.Nested(message_file_model)),
-        "feedback": fields.Nested(feedback_model, attribute="user_feedback", allow_null=True),
-        "retriever_resources": fields.Raw(
-            attribute=lambda obj: json.loads(obj.message_metadata).get("retriever_resources", [])
-            if obj.message_metadata
-            else []
-        ),
-        "created_at": TimestampField,
-        "agent_thoughts": fields.List(fields.Nested(agent_thought_model)),
-        "status": fields.String,
-        "error": fields.String,
-    }
-    return api_or_ns.model("Message", message_fields)
-
-
-def build_message_infinite_scroll_pagination_model(api_or_ns: Namespace):
-    """Build the message infinite scroll pagination model for the API or Namespace."""
-    # Build the nested message model first
-    message_model = build_message_model(api_or_ns)
-
-    message_infinite_scroll_pagination_fields = {
-        "limit": fields.Integer,
-        "has_more": fields.Boolean,
-        "data": fields.List(fields.Nested(message_model)),
-    }
-    return api_or_ns.model("MessageInfiniteScrollPagination", message_infinite_scroll_pagination_fields)
-
-
 @service_api_ns.route("/messages")
 class MessageListApi(Resource):
     @service_api_ns.expect(service_api_ns.models[MessageListQuery.__name__])
@@ -104,7 +58,6 @@ class MessageListApi(Resource):
         }
     )
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.QUERY))
-    @service_api_ns.marshal_with(build_message_infinite_scroll_pagination_model(service_api_ns))
     def get(self, app_model: App, end_user: EndUser):
         """List messages in a conversation.
 
@@ -119,9 +72,16 @@ class MessageListApi(Resource):
         first_id = str(query_args.first_id) if query_args.first_id else None
 
         try:
-            return MessageService.pagination_by_first_id(
+            pagination = MessageService.pagination_by_first_id(
                 app_model, end_user, conversation_id, first_id, query_args.limit
             )
+            adapter = TypeAdapter(MessageListItem)
+            items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data]
+            return MessageInfiniteScrollPagination(
+                limit=pagination.limit,
+                has_more=pagination.has_more,
+                data=items,
+            ).model_dump(mode="json")
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except FirstMessageNotExistsError:
@@ -162,7 +122,7 @@ class MessageFeedbackApi(Resource):
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @service_api_ns.route("/app/feedbacks")

api/controllers/service_api/app/site.py

@@ -1,7 +1,7 @@
 from flask_restx import Resource
 from werkzeug.exceptions import Forbidden
 
-from controllers.common.fields import build_site_model
+from controllers.common.fields import Site as SiteResponse
 from controllers.service_api import service_api_ns
 from controllers.service_api.wraps import validate_app_token
 from extensions.ext_database import db
@@ -23,7 +23,6 @@ class AppSiteApi(Resource):
         }
     )
     @validate_app_token
-    @service_api_ns.marshal_with(build_site_model(service_api_ns))
     def get(self, app_model: App):
         """Retrieve app site info.
 
@@ -38,4 +37,4 @@ class AppSiteApi(Resource):
         if app_model.tenant.status == TenantStatus.ARCHIVE:
             raise Forbidden()
 
-        return site
+        return SiteResponse.model_validate(site).model_dump(mode="json")

api/controllers/service_api/app/workflow.py

@@ -3,7 +3,7 @@ from typing import Any, Literal
 
 from dateutil.parser import isoparse
 from flask import request
-from flask_restx import Api, Namespace, Resource, fields
+from flask_restx import Namespace, Resource, fields
 from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session, sessionmaker
 from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
@@ -78,7 +78,7 @@ workflow_run_fields = {
 }
 
 
-def build_workflow_run_model(api_or_ns: Api | Namespace):
+def build_workflow_run_model(api_or_ns: Namespace):
     """Build the workflow run model for the API or Namespace."""
     return api_or_ns.model("WorkflowRun", workflow_run_fields)
 

api/controllers/service_api/dataset/dataset.py

@@ -13,7 +13,6 @@ from controllers.service_api.dataset.error import DatasetInUseError, DatasetName
 from controllers.service_api.wraps import (
     DatasetApiResource,
     cloud_edition_billing_rate_limit_check,
-    validate_dataset_token,
 )
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
@@ -460,9 +459,8 @@ class DatasetTagsApi(DatasetApiResource):
             401: "Unauthorized - invalid API token",
         }
     )
-    @validate_dataset_token
     @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
-    def get(self, _, dataset_id):
+    def get(self, _):
         """Get all knowledge type tags."""
         assert isinstance(current_user, Account)
         cid = current_user.current_tenant_id
@@ -482,8 +480,7 @@ class DatasetTagsApi(DatasetApiResource):
         }
     )
     @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
-    @validate_dataset_token
-    def post(self, _, dataset_id):
+    def post(self, _):
         """Add a knowledge type tag."""
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
@@ -506,8 +503,7 @@ class DatasetTagsApi(DatasetApiResource):
         }
     )
     @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
-    @validate_dataset_token
-    def patch(self, _, dataset_id):
+    def patch(self, _):
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
             raise Forbidden()
@@ -533,9 +529,8 @@ class DatasetTagsApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
-    @validate_dataset_token
     @edit_permission_required
-    def delete(self, _, dataset_id):
+    def delete(self, _):
         """Delete a knowledge type tag."""
         payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
         TagService.delete_tag(payload.tag_id)
@@ -555,8 +550,7 @@ class DatasetTagBindingApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
-    @validate_dataset_token
-    def post(self, _, dataset_id):
+    def post(self, _):
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
@@ -580,8 +574,7 @@ class DatasetTagUnbindingApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
-    @validate_dataset_token
-    def post(self, _, dataset_id):
+    def post(self, _):
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
@@ -604,7 +597,6 @@ class DatasetTagsBindingStatusApi(DatasetApiResource):
             401: "Unauthorized - invalid API token",
         }
     )
-    @validate_dataset_token
     def get(self, _, *args, **kwargs):
         """Get all knowledge type tags."""
         dataset_id = kwargs.get("dataset_id")

api/controllers/web/app.py

@@ -1,7 +1,7 @@
 import logging
 
 from flask import request
-from flask_restx import Resource, marshal_with
+from flask_restx import Resource
 from pydantic import BaseModel, ConfigDict, Field
 from werkzeug.exceptions import Unauthorized
 
@@ -50,7 +50,6 @@ class AppParameterApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(fields.parameters_fields)
     def get(self, app_model: App, end_user):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
@@ -69,7 +68,8 @@ class AppParameterApi(WebApiResource):
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        parameters = get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+        return fields.Parameters.model_validate(parameters).model_dump(mode="json")
 
 
 @web_ns.route("/meta")

api/controllers/web/conversation.py

@@ -1,5 +1,6 @@
-from flask_restx import fields, marshal_with, reqparse
+from flask_restx import reqparse
 from flask_restx.inputs import int_range
+from pydantic import TypeAdapter
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 
@@ -8,7 +9,11 @@ from controllers.web.error import NotChatAppError
 from controllers.web.wraps import WebApiResource
 from core.app.entities.app_invoke_entities import InvokeFrom
 from extensions.ext_database import db
-from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
+from fields.conversation_fields import (
+    ConversationInfiniteScrollPagination,
+    ResultResponse,
+    SimpleConversation,
+)
 from libs.helper import uuid_value
 from models.model import AppMode
 from services.conversation_service import ConversationService
@@ -54,7 +59,6 @@ class ConversationListApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(conversation_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -82,7 +86,7 @@ class ConversationListApi(WebApiResource):
 
         try:
             with Session(db.engine) as session:
-                return WebConversationService.pagination_by_last_id(
+                pagination = WebConversationService.pagination_by_last_id(
                     session=session,
                     app_model=app_model,
                     user=end_user,
@@ -92,16 +96,19 @@ class ConversationListApi(WebApiResource):
                     pinned=pinned,
                     sort_by=args["sort_by"],
                 )
+                adapter = TypeAdapter(SimpleConversation)
+                conversations = [adapter.validate_python(item, from_attributes=True) for item in pagination.data]
+                return ConversationInfiniteScrollPagination(
+                    limit=pagination.limit,
+                    has_more=pagination.has_more,
+                    data=conversations,
+                ).model_dump(mode="json")
         except LastConversationNotExistsError:
             raise NotFound("Last Conversation Not Exists.")
 
 
 @web_ns.route("/conversations/<uuid:c_id>")
 class ConversationApi(WebApiResource):
-    delete_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Delete Conversation")
     @web_ns.doc(description="Delete a specific conversation.")
     @web_ns.doc(params={"c_id": {"description": "Conversation UUID", "type": "string", "required": True}})
@@ -115,7 +122,6 @@ class ConversationApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(delete_response_fields)
     def delete(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -126,7 +132,7 @@ class ConversationApi(WebApiResource):
             ConversationService.delete(app_model, conversation_id, end_user)
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
-        return {"result": "success"}, 204
+        return ResultResponse(result="success").model_dump(mode="json"), 204
 
 
 @web_ns.route("/conversations/<uuid:c_id>/name")
@@ -155,7 +161,6 @@ class ConversationRenameApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(simple_conversation_fields)
     def post(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -171,17 +176,20 @@ class ConversationRenameApi(WebApiResource):
         args = parser.parse_args()
 
         try:
-            return ConversationService.rename(app_model, conversation_id, end_user, args["name"], args["auto_generate"])
+            conversation = ConversationService.rename(
+                app_model, conversation_id, end_user, args["name"], args["auto_generate"]
+            )
+            return (
+                TypeAdapter(SimpleConversation)
+                .validate_python(conversation, from_attributes=True)
+                .model_dump(mode="json")
+            )
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 
 
 @web_ns.route("/conversations/<uuid:c_id>/pin")
 class ConversationPinApi(WebApiResource):
-    pin_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Pin Conversation")
     @web_ns.doc(description="Pin a specific conversation to keep it at the top of the list.")
     @web_ns.doc(params={"c_id": {"description": "Conversation UUID", "type": "string", "required": True}})
@@ -195,7 +203,6 @@ class ConversationPinApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(pin_response_fields)
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -208,15 +215,11 @@ class ConversationPinApi(WebApiResource):
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @web_ns.route("/conversations/<uuid:c_id>/unpin")
 class ConversationUnPinApi(WebApiResource):
-    unpin_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Unpin Conversation")
     @web_ns.doc(description="Unpin a specific conversation to remove it from the top of the list.")
     @web_ns.doc(params={"c_id": {"description": "Conversation UUID", "type": "string", "required": True}})
@@ -230,7 +233,6 @@ class ConversationUnPinApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(unpin_response_fields)
     def patch(self, app_model, end_user, c_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -239,4 +241,4 @@ class ConversationUnPinApi(WebApiResource):
         conversation_id = str(c_id)
         WebConversationService.unpin(app_model, conversation_id, end_user)
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")

api/controllers/web/message.py

@@ -2,8 +2,7 @@ import logging
 from typing import Literal
 
 from flask import request
-from flask_restx import fields, marshal_with
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
 from werkzeug.exceptions import InternalServerError, NotFound
 
 from controllers.common.schema import register_schema_models
@@ -22,11 +21,10 @@ from controllers.web.wraps import WebApiResource
 from core.app.entities.app_invoke_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from fields.conversation_fields import message_file_fields
-from fields.message_fields import agent_thought_fields, feedback_fields, retriever_resource_fields
-from fields.raws import FilesContainedField
+from fields.conversation_fields import ResultResponse
+from fields.message_fields import SuggestedQuestionsResponse, WebMessageInfiniteScrollPagination, WebMessageListItem
 from libs import helper
-from libs.helper import TimestampField, uuid_value
+from libs.helper import uuid_value
 from models.model import AppMode
 from services.app_generate_service import AppGenerateService
 from services.errors.app import MoreLikeThisDisabledError
@@ -70,29 +68,6 @@ register_schema_models(web_ns, MessageListQuery, MessageFeedbackPayload, Message
 
 @web_ns.route("/messages")
 class MessageListApi(WebApiResource):
-    message_fields = {
-        "id": fields.String,
-        "conversation_id": fields.String,
-        "parent_message_id": fields.String,
-        "inputs": FilesContainedField,
-        "query": fields.String,
-        "answer": fields.String(attribute="re_sign_file_url_answer"),
-        "message_files": fields.List(fields.Nested(message_file_fields)),
-        "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True),
-        "retriever_resources": fields.List(fields.Nested(retriever_resource_fields)),
-        "created_at": TimestampField,
-        "agent_thoughts": fields.List(fields.Nested(agent_thought_fields)),
-        "metadata": fields.Raw(attribute="message_metadata_dict"),
-        "status": fields.String,
-        "error": fields.String,
-    }
-
-    message_infinite_scroll_pagination_fields = {
-        "limit": fields.Integer,
-        "has_more": fields.Boolean,
-        "data": fields.List(fields.Nested(message_fields)),
-    }
-
     @web_ns.doc("Get Message List")
     @web_ns.doc(description="Retrieve paginated list of messages from a conversation in a chat application.")
     @web_ns.doc(
@@ -121,7 +96,6 @@ class MessageListApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -131,9 +105,16 @@ class MessageListApi(WebApiResource):
         query = MessageListQuery.model_validate(raw_args)
 
         try:
-            return MessageService.pagination_by_first_id(
+            pagination = MessageService.pagination_by_first_id(
                 app_model, end_user, query.conversation_id, query.first_id, query.limit
             )
+            adapter = TypeAdapter(WebMessageListItem)
+            items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data]
+            return WebMessageInfiniteScrollPagination(
+                limit=pagination.limit,
+                has_more=pagination.has_more,
+                data=items,
+            ).model_dump(mode="json")
         except ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except FirstMessageNotExistsError:
@@ -142,10 +123,6 @@ class MessageListApi(WebApiResource):
 
 @web_ns.route("/messages/<uuid:message_id>/feedbacks")
 class MessageFeedbackApi(WebApiResource):
-    feedback_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Create Message Feedback")
     @web_ns.doc(description="Submit feedback (like/dislike) for a specific message.")
     @web_ns.doc(params={"message_id": {"description": "Message UUID", "type": "string", "required": True}})
@@ -170,7 +147,6 @@ class MessageFeedbackApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(feedback_response_fields)
     def post(self, app_model, end_user, message_id):
         message_id = str(message_id)
 
@@ -187,7 +163,7 @@ class MessageFeedbackApi(WebApiResource):
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @web_ns.route("/messages/<uuid:message_id>/more-like-this")
@@ -247,10 +223,6 @@ class MessageMoreLikeThisApi(WebApiResource):
 
 @web_ns.route("/messages/<uuid:message_id>/suggested-questions")
 class MessageSuggestedQuestionApi(WebApiResource):
-    suggested_questions_response_fields = {
-        "data": fields.List(fields.String),
-    }
-
     @web_ns.doc("Get Suggested Questions")
     @web_ns.doc(description="Get suggested follow-up questions after a message (chat apps only).")
     @web_ns.doc(params={"message_id": {"description": "Message UUID", "type": "string", "required": True}})
@@ -264,7 +236,6 @@ class MessageSuggestedQuestionApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(suggested_questions_response_fields)
     def get(self, app_model, end_user, message_id):
         app_mode = AppMode.value_of(app_model.mode)
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
@@ -277,7 +248,6 @@ class MessageSuggestedQuestionApi(WebApiResource):
                 app_model=app_model, user=end_user, message_id=message_id, invoke_from=InvokeFrom.WEB_APP
             )
             # questions is a list of strings, not a list of Message objects
-            # so we can directly return it
         except MessageNotExistsError:
             raise NotFound("Message not found")
         except ConversationNotExistsError:
@@ -296,4 +266,4 @@ class MessageSuggestedQuestionApi(WebApiResource):
             logger.exception("internal server error.")
             raise InternalServerError()
 
-        return {"data": questions}
+        return SuggestedQuestionsResponse(data=questions).model_dump(mode="json")

api/controllers/web/saved_message.py

@@ -1,40 +1,20 @@
-from flask_restx import fields, marshal_with, reqparse
+from flask_restx import reqparse
 from flask_restx.inputs import int_range
+from pydantic import TypeAdapter
 from werkzeug.exceptions import NotFound
 
 from controllers.web import web_ns
 from controllers.web.error import NotCompletionAppError
 from controllers.web.wraps import WebApiResource
-from fields.conversation_fields import message_file_fields
-from libs.helper import TimestampField, uuid_value
+from fields.conversation_fields import ResultResponse
+from fields.message_fields import SavedMessageInfiniteScrollPagination, SavedMessageItem
+from libs.helper import uuid_value
 from services.errors.message import MessageNotExistsError
 from services.saved_message_service import SavedMessageService
 
-feedback_fields = {"rating": fields.String}
-
-message_fields = {
-    "id": fields.String,
-    "inputs": fields.Raw,
-    "query": fields.String,
-    "answer": fields.String,
-    "message_files": fields.List(fields.Nested(message_file_fields)),
-    "feedback": fields.Nested(feedback_fields, attribute="user_feedback", allow_null=True),
-    "created_at": TimestampField,
-}
-
 
 @web_ns.route("/saved-messages")
 class SavedMessageListApi(WebApiResource):
-    saved_message_infinite_scroll_pagination_fields = {
-        "limit": fields.Integer,
-        "has_more": fields.Boolean,
-        "data": fields.List(fields.Nested(message_fields)),
-    }
-
-    post_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Get Saved Messages")
     @web_ns.doc(description="Retrieve paginated list of saved messages for a completion application.")
     @web_ns.doc(
@@ -58,7 +38,6 @@ class SavedMessageListApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(saved_message_infinite_scroll_pagination_fields)
     def get(self, app_model, end_user):
         if app_model.mode != "completion":
             raise NotCompletionAppError()
@@ -70,7 +49,14 @@ class SavedMessageListApi(WebApiResource):
         )
         args = parser.parse_args()
 
-        return SavedMessageService.pagination_by_last_id(app_model, end_user, args["last_id"], args["limit"])
+        pagination = SavedMessageService.pagination_by_last_id(app_model, end_user, args["last_id"], args["limit"])
+        adapter = TypeAdapter(SavedMessageItem)
+        items = [adapter.validate_python(message, from_attributes=True) for message in pagination.data]
+        return SavedMessageInfiniteScrollPagination(
+            limit=pagination.limit,
+            has_more=pagination.has_more,
+            data=items,
+        ).model_dump(mode="json")
 
     @web_ns.doc("Save Message")
     @web_ns.doc(description="Save a specific message for later reference.")
@@ -89,7 +75,6 @@ class SavedMessageListApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(post_response_fields)
     def post(self, app_model, end_user):
         if app_model.mode != "completion":
             raise NotCompletionAppError()
@@ -102,15 +87,11 @@ class SavedMessageListApi(WebApiResource):
         except MessageNotExistsError:
             raise NotFound("Message Not Exists.")
 
-        return {"result": "success"}
+        return ResultResponse(result="success").model_dump(mode="json")
 
 
 @web_ns.route("/saved-messages/<uuid:message_id>")
 class SavedMessageApi(WebApiResource):
-    delete_response_fields = {
-        "result": fields.String,
-    }
-
     @web_ns.doc("Delete Saved Message")
     @web_ns.doc(description="Remove a message from saved messages.")
     @web_ns.doc(params={"message_id": {"description": "Message UUID to delete", "type": "string", "required": True}})
@@ -124,7 +105,6 @@ class SavedMessageApi(WebApiResource):
             500: "Internal Server Error",
         }
     )
-    @marshal_with(delete_response_fields)
     def delete(self, app_model, end_user, message_id):
         message_id = str(message_id)
 
@@ -133,4 +113,4 @@ class SavedMessageApi(WebApiResource):
 
         SavedMessageService.delete(app_model, end_user, message_id)
 
-        return {"result": "success"}, 204
+        return ResultResponse(result="success").model_dump(mode="json"), 204

api/core/agent/cot_agent_runner.py

@@ -22,6 +22,7 @@ from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransfo
 from core.tools.__base.tool import Tool
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool_engine import ToolEngine
+from core.workflow.nodes.agent.exc import AgentMaxIterationError
 from models.model import Message
 
 logger = logging.getLogger(__name__)
@@ -165,6 +166,11 @@ class CotAgentRunner(BaseAgentRunner, ABC):
             scratchpad.thought = scratchpad.thought.strip() or "I am thinking about how to help you"
             self._agent_scratchpad.append(scratchpad)
 
+            # Check if max iteration is reached and model still wants to call tools
+            if iteration_step == max_iteration_steps and scratchpad.action:
+                if scratchpad.action.action_name.lower() != "final answer":
+                    raise AgentMaxIterationError(app_config.agent.max_iteration)
+
             # get llm usage
             if "usage" in usage_dict:
                 if usage_dict["usage"] is not None:

api/core/agent/fc_agent_runner.py

@@ -25,6 +25,7 @@ from core.model_runtime.entities.message_entities import ImagePromptMessageConte
 from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransform
 from core.tools.entities.tool_entities import ToolInvokeMeta
 from core.tools.tool_engine import ToolEngine
+from core.workflow.nodes.agent.exc import AgentMaxIterationError
 from models.model import Message
 
 logger = logging.getLogger(__name__)
@@ -222,6 +223,10 @@ class FunctionCallAgentRunner(BaseAgentRunner):
 
             final_answer += response + "\n"
 
+            # Check if max iteration is reached and model still wants to call tools
+            if iteration_step == max_iteration_steps and tool_calls:
+                raise AgentMaxIterationError(app_config.agent.max_iteration)
+
             # call tools
             tool_responses = []
             for tool_call_id, tool_call_name, tool_call_args in tool_calls:

api/core/app/apps/base_app_queue_manager.py

@@ -90,6 +90,7 @@ class AppQueueManager:
         """
         self._clear_task_belong_cache()
         self._q.put(None)
+        self._graph_runtime_state = None  # Release reference to allow GC to reclaim memory
 
     def _clear_task_belong_cache(self) -> None:
         """

api/core/app/features/annotation_reply/annotation_reply.py

@@ -75,7 +75,7 @@ class AnnotationReplyFeature:
                     AppAnnotationService.add_annotation_history(
                         annotation.id,
                         app_record.id,
-                        annotation.question,
+                        annotation.question_text,
                         annotation.content,
                         query,
                         user_id,

api/core/app/layers/pause_state_persist_layer.py

@@ -66,6 +66,7 @@ class PauseStatePersistenceLayer(GraphEngineLayer):
         """
         if isinstance(session_factory, Engine):
             session_factory = sessionmaker(session_factory)
+        super().__init__()
         self._session_maker = session_factory
         self._state_owner_user_id = state_owner_user_id
         self._generate_entity = generate_entity
@@ -98,8 +99,6 @@ class PauseStatePersistenceLayer(GraphEngineLayer):
         if not isinstance(event, GraphRunPausedEvent):
             return
 
-        assert self.graph_runtime_state is not None
-
         entity_wrapper: _GenerateEntityUnion
         if isinstance(self._generate_entity, WorkflowAppGenerateEntity):
             entity_wrapper = _WorkflowGenerateEntityWrapper(entity=self._generate_entity)

api/core/app/layers/trigger_post_layer.py

@@ -33,6 +33,7 @@ class TriggerPostLayer(GraphEngineLayer):
         trigger_log_id: str,
         session_maker: sessionmaker[Session],
     ):
+        super().__init__()
         self.trigger_log_id = trigger_log_id
         self.start_time = start_time
         self.cfs_plan_scheduler_entity = cfs_plan_scheduler_entity
@@ -57,10 +58,6 @@ class TriggerPostLayer(GraphEngineLayer):
                 elapsed_time = (datetime.now(UTC) - self.start_time).total_seconds()
 
                 # Extract relevant data from result
-                if not self.graph_runtime_state:
-                    logger.exception("Graph runtime state is not set")
-                    return
-
                 outputs = self.graph_runtime_state.outputs
 
                 # BASICLY, workflow_execution_id is the same as workflow_run_id

api/core/entities/model_entities.py

@@ -30,7 +30,6 @@ class SimpleModelProviderEntity(BaseModel):
     label: I18nObject
     icon_small: I18nObject | None = None
     icon_small_dark: I18nObject | None = None
-    icon_large: I18nObject | None = None
     supported_model_types: list[ModelType]
 
     def __init__(self, provider_entity: ProviderEntity):
@@ -44,7 +43,6 @@ class SimpleModelProviderEntity(BaseModel):
             label=provider_entity.label,
             icon_small=provider_entity.icon_small,
             icon_small_dark=provider_entity.icon_small_dark,
-            icon_large=provider_entity.icon_large,
             supported_model_types=provider_entity.supported_model_types,
         )
 
@@ -94,7 +92,6 @@ class DefaultModelProviderEntity(BaseModel):
     provider: str
     label: I18nObject
     icon_small: I18nObject | None = None
-    icon_large: I18nObject | None = None
     supported_model_types: Sequence[ModelType] = []
 
 

api/core/helper/code_executor/jinja2/jinja2_transformer.py

@@ -1,9 +1,14 @@
+from collections.abc import Mapping
 from textwrap import dedent
+from typing import Any
 
 from core.helper.code_executor.template_transformer import TemplateTransformer
 
 
 class Jinja2TemplateTransformer(TemplateTransformer):
+    # Use separate placeholder for base64-encoded template to avoid confusion
+    _template_b64_placeholder: str = "{{template_b64}}"
+
     @classmethod
     def transform_response(cls, response: str):
         """
@@ -13,18 +18,35 @@ class Jinja2TemplateTransformer(TemplateTransformer):
         """
         return {"result": cls.extract_result_str_from_response(response)}
 
+    @classmethod
+    def assemble_runner_script(cls, code: str, inputs: Mapping[str, Any]) -> str:
+        """
+        Override base class to use base64 encoding for template code.
+        This prevents issues with special characters (quotes, newlines) in templates
+        breaking the generated Python script. Fixes #26818.
+        """
+        script = cls.get_runner_script()
+        # Encode template as base64 to safely embed any content including quotes
+        code_b64 = cls.serialize_code(code)
+        script = script.replace(cls._template_b64_placeholder, code_b64)
+        inputs_str = cls.serialize_inputs(inputs)
+        script = script.replace(cls._inputs_placeholder, inputs_str)
+        return script
+
     @classmethod
     def get_runner_script(cls) -> str:
         runner_script = dedent(f"""
-            # declare main function
-            def main(**inputs):
-                import jinja2
-                template = jinja2.Template('''{cls._code_placeholder}''')
-                return template.render(**inputs)
-
+            import jinja2
             import json
             from base64 import b64decode
 
+            # declare main function
+            def main(**inputs):
+                # Decode base64-encoded template to handle special characters safely
+                template_code = b64decode('{cls._template_b64_placeholder}').decode('utf-8')
+                template = jinja2.Template(template_code)
+                return template.render(**inputs)
+
             # decode and prepare input dict
             inputs_obj = json.loads(b64decode('{cls._inputs_placeholder}').decode('utf-8'))
 

api/core/helper/code_executor/template_transformer.py

@@ -13,6 +13,15 @@ class TemplateTransformer(ABC):
     _inputs_placeholder: str = "{{inputs}}"
     _result_tag: str = "<<RESULT>>"
 
+    @classmethod
+    def serialize_code(cls, code: str) -> str:
+        """
+        Serialize template code to base64 to safely embed in generated script.
+        This prevents issues with special characters like quotes breaking the script.
+        """
+        code_bytes = code.encode("utf-8")
+        return b64encode(code_bytes).decode("utf-8")
+
     @classmethod
     def transform_caller(cls, code: str, inputs: Mapping[str, Any]) -> tuple[str, str]:
         """

api/core/helper/ssrf_proxy.py

@@ -88,7 +88,41 @@ def _get_user_provided_host_header(headers: dict | None) -> str | None:
     return None
 
 
+def _inject_trace_headers(headers: dict | None) -> dict:
+    """
+    Inject W3C traceparent header for distributed tracing.
+
+    When OTEL is enabled, HTTPXClientInstrumentor handles trace propagation automatically.
+    When OTEL is disabled, we manually inject the traceparent header.
+    """
+    if headers is None:
+        headers = {}
+
+    # Skip if already present (case-insensitive check)
+    for key in headers:
+        if key.lower() == "traceparent":
+            return headers
+
+    # Skip if OTEL is enabled - HTTPXClientInstrumentor handles this automatically
+    if dify_config.ENABLE_OTEL:
+        return headers
+
+    # Generate and inject traceparent for non-OTEL scenarios
+    try:
+        from core.helper.trace_id_helper import generate_traceparent_header
+
+        traceparent = generate_traceparent_header()
+        if traceparent:
+            headers["traceparent"] = traceparent
+    except Exception:
+        # Silently ignore errors to avoid breaking requests
+        logger.debug("Failed to generate traceparent header", exc_info=True)
+
+    return headers
+
+
 def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
+    # Convert requests-style allow_redirects to httpx-style follow_redirects
     if "allow_redirects" in kwargs:
         allow_redirects = kwargs.pop("allow_redirects")
         if "follow_redirects" not in kwargs:
@@ -106,18 +140,21 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
     verify_option = kwargs.pop("ssl_verify", dify_config.HTTP_REQUEST_NODE_SSL_VERIFY)
     client = _get_ssrf_client(verify_option)
 
+    # Inject traceparent header for distributed tracing (when OTEL is not enabled)
+    headers = kwargs.get("headers") or {}
+    headers = _inject_trace_headers(headers)
+    kwargs["headers"] = headers
+
     # Preserve user-provided Host header
     # When using a forward proxy, httpx may override the Host header based on the URL.
     # We extract and preserve any explicitly set Host header to support virtual hosting.
-    headers = kwargs.get("headers", {})
     user_provided_host = _get_user_provided_host_header(headers)
 
     retries = 0
     while retries <= max_retries:
         try:
-            # Build the request manually to preserve the Host header
-            # httpx may override the Host header when using a proxy, so we use
-            # the request API to explicitly set headers before sending
+            # Preserve the user-provided Host header
+            # httpx may override the Host header when using a proxy
             headers = {k: v for k, v in headers.items() if k.lower() != "host"}
             if user_provided_host is not None:
                 headers["host"] = user_provided_host

api/core/helper/tool_provider_cache.py

@@ -1,56 +0,0 @@
-import json
-import logging
-from typing import Any
-
-from core.tools.entities.api_entities import ToolProviderTypeApiLiteral
-from extensions.ext_redis import redis_client, redis_fallback
-
-logger = logging.getLogger(__name__)
-
-
-class ToolProviderListCache:
-    """Cache for tool provider lists"""
-
-    CACHE_TTL = 300  # 5 minutes
-
-    @staticmethod
-    def _generate_cache_key(tenant_id: str, typ: ToolProviderTypeApiLiteral = None) -> str:
-        """Generate cache key for tool providers list"""
-        type_filter = typ or "all"
-        return f"tool_providers:tenant_id:{tenant_id}:type:{type_filter}"
-
-    @staticmethod
-    @redis_fallback(default_return=None)
-    def get_cached_providers(tenant_id: str, typ: ToolProviderTypeApiLiteral = None) -> list[dict[str, Any]] | None:
-        """Get cached tool providers"""
-        cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ)
-        cached_data = redis_client.get(cache_key)
-        if cached_data:
-            try:
-                return json.loads(cached_data.decode("utf-8"))
-            except (json.JSONDecodeError, UnicodeDecodeError):
-                logger.warning("Failed to decode cached tool providers data")
-                return None
-        return None
-
-    @staticmethod
-    @redis_fallback()
-    def set_cached_providers(tenant_id: str, typ: ToolProviderTypeApiLiteral, providers: list[dict[str, Any]]):
-        """Cache tool providers"""
-        cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ)
-        redis_client.setex(cache_key, ToolProviderListCache.CACHE_TTL, json.dumps(providers))
-
-    @staticmethod
-    @redis_fallback()
-    def invalidate_cache(tenant_id: str, typ: ToolProviderTypeApiLiteral = None):
-        """Invalidate cache for tool providers"""
-        if typ:
-            # Invalidate specific type cache
-            cache_key = ToolProviderListCache._generate_cache_key(tenant_id, typ)
-            redis_client.delete(cache_key)
-        else:
-            # Invalidate all caches for this tenant
-            pattern = f"tool_providers:tenant_id:{tenant_id}:*"
-            keys = list(redis_client.scan_iter(pattern))
-            if keys:
-                redis_client.delete(*keys)

api/core/helper/trace_id_helper.py

@@ -103,3 +103,60 @@ def parse_traceparent_header(traceparent: str) -> str | None:
         if len(parts) == 4 and len(parts[1]) == 32:
             return parts[1]
     return None
+
+
+def get_span_id_from_otel_context() -> str | None:
+    """
+    Retrieve the current span ID from the active OpenTelemetry trace context.
+
+    Returns:
+        A 16-character hex string representing the span ID, or None if not available.
+    """
+    try:
+        from opentelemetry.trace import get_current_span
+        from opentelemetry.trace.span import INVALID_SPAN_ID
+
+        span = get_current_span()
+        if not span:
+            return None
+
+        span_context = span.get_span_context()
+        if not span_context or span_context.span_id == INVALID_SPAN_ID:
+            return None
+
+        return f"{span_context.span_id:016x}"
+    except Exception:
+        return None
+
+
+def generate_traceparent_header() -> str | None:
+    """
+    Generate a W3C traceparent header from the current context.
+
+    Uses OpenTelemetry context if available, otherwise uses the
+    ContextVar-based trace_id from the logging context.
+
+    Format: {version}-{trace_id}-{span_id}-{flags}
+    Example: 00-5b8aa5a2d2c872e8321cf37308d69df2-051581bf3bb55c45-01
+
+    Returns:
+        A valid traceparent header string, or None if generation fails.
+    """
+    import uuid
+
+    # Try OTEL context first
+    trace_id = get_trace_id_from_otel_context()
+    span_id = get_span_id_from_otel_context()
+
+    if trace_id and span_id:
+        return f"00-{trace_id}-{span_id}-01"
+
+    # Fallback: use ContextVar-based trace_id or generate new one
+    from core.logging.context import get_trace_id as get_logging_trace_id
+
+    trace_id = get_logging_trace_id() or uuid.uuid4().hex
+
+    # Generate a new span_id (16 hex chars)
+    span_id = uuid.uuid4().hex[:16]
+
+    return f"00-{trace_id}-{span_id}-01"

api/core/logging/__init__.py

@@ -0,0 +1,20 @@
+"""Structured logging components for Dify."""
+
+from core.logging.context import (
+    clear_request_context,
+    get_request_id,
+    get_trace_id,
+    init_request_context,
+)
+from core.logging.filters import IdentityContextFilter, TraceContextFilter
+from core.logging.structured_formatter import StructuredJSONFormatter
+
+__all__ = [
+    "IdentityContextFilter",
+    "StructuredJSONFormatter",
+    "TraceContextFilter",
+    "clear_request_context",
+    "get_request_id",
+    "get_trace_id",
+    "init_request_context",
+]

api/core/logging/context.py

@@ -0,0 +1,35 @@
+"""Request context for logging - framework agnostic.
+
+This module provides request-scoped context variables for logging,
+using Python's contextvars for thread-safe and async-safe storage.
+"""
+
+import uuid
+from contextvars import ContextVar
+
+_request_id: ContextVar[str] = ContextVar("log_request_id", default="")
+_trace_id: ContextVar[str] = ContextVar("log_trace_id", default="")
+
+
+def get_request_id() -> str:
+    """Get current request ID (10 hex chars)."""
+    return _request_id.get()
+
+
+def get_trace_id() -> str:
+    """Get fallback trace ID when OTEL is unavailable (32 hex chars)."""
+    return _trace_id.get()
+
+
+def init_request_context() -> None:
+    """Initialize request context. Call at start of each request."""
+    req_id = uuid.uuid4().hex[:10]
+    trace_id = uuid.uuid5(uuid.NAMESPACE_DNS, req_id).hex
+    _request_id.set(req_id)
+    _trace_id.set(trace_id)
+
+
+def clear_request_context() -> None:
+    """Clear request context. Call at end of request (optional)."""
+    _request_id.set("")
+    _trace_id.set("")

api/core/logging/filters.py

@@ -0,0 +1,94 @@
+"""Logging filters for structured logging."""
+
+import contextlib
+import logging
+
+import flask
+
+from core.logging.context import get_request_id, get_trace_id
+
+
+class TraceContextFilter(logging.Filter):
+    """
+    Filter that adds trace_id and span_id to log records.
+    Integrates with OpenTelemetry when available, falls back to ContextVar-based trace_id.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        # Get trace context from OpenTelemetry
+        trace_id, span_id = self._get_otel_context()
+
+        # Set trace_id (fallback to ContextVar if no OTEL context)
+        if trace_id:
+            record.trace_id = trace_id
+        else:
+            record.trace_id = get_trace_id()
+
+        record.span_id = span_id or ""
+
+        # For backward compatibility, also set req_id
+        record.req_id = get_request_id()
+
+        return True
+
+    def _get_otel_context(self) -> tuple[str, str]:
+        """Extract trace_id and span_id from OpenTelemetry context."""
+        with contextlib.suppress(Exception):
+            from opentelemetry.trace import get_current_span
+            from opentelemetry.trace.span import INVALID_SPAN_ID, INVALID_TRACE_ID
+
+            span = get_current_span()
+            if span and span.get_span_context():
+                ctx = span.get_span_context()
+                if ctx.is_valid and ctx.trace_id != INVALID_TRACE_ID:
+                    trace_id = f"{ctx.trace_id:032x}"
+                    span_id = f"{ctx.span_id:016x}" if ctx.span_id != INVALID_SPAN_ID else ""
+                    return trace_id, span_id
+        return "", ""
+
+
+class IdentityContextFilter(logging.Filter):
+    """
+    Filter that adds user identity context to log records.
+    Extracts tenant_id, user_id, and user_type from Flask-Login current_user.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        identity = self._extract_identity()
+        record.tenant_id = identity.get("tenant_id", "")
+        record.user_id = identity.get("user_id", "")
+        record.user_type = identity.get("user_type", "")
+        return True
+
+    def _extract_identity(self) -> dict[str, str]:
+        """Extract identity from current_user if in request context."""
+        try:
+            if not flask.has_request_context():
+                return {}
+            from flask_login import current_user
+
+            # Check if user is authenticated using the proxy
+            if not current_user.is_authenticated:
+                return {}
+
+            # Access the underlying user object
+            user = current_user
+
+            from models import Account
+            from models.model import EndUser
+
+            identity: dict[str, str] = {}
+
+            if isinstance(user, Account):
+                if user.current_tenant_id:
+                    identity["tenant_id"] = user.current_tenant_id
+                identity["user_id"] = user.id
+                identity["user_type"] = "account"
+            elif isinstance(user, EndUser):
+                identity["tenant_id"] = user.tenant_id
+                identity["user_id"] = user.id
+                identity["user_type"] = user.type or "end_user"
+
+            return identity
+        except Exception:
+            return {}

api/core/logging/structured_formatter.py

@@ -0,0 +1,107 @@
+"""Structured JSON log formatter for Dify."""
+
+import logging
+import traceback
+from datetime import UTC, datetime
+from typing import Any
+
+import orjson
+
+from configs import dify_config
+
+
+class StructuredJSONFormatter(logging.Formatter):
+    """
+    JSON log formatter following the specified schema:
+    {
+      "ts": "ISO 8601 UTC",
+      "severity": "INFO|ERROR|WARN|DEBUG",
+      "service": "service name",
+      "caller": "file:line",
+      "trace_id": "hex 32",
+      "span_id": "hex 16",
+      "identity": { "tenant_id", "user_id", "user_type" },
+      "message": "log message",
+      "attributes": { ... },
+      "stack_trace": "..."
+    }
+    """
+
+    SEVERITY_MAP: dict[int, str] = {
+        logging.DEBUG: "DEBUG",
+        logging.INFO: "INFO",
+        logging.WARNING: "WARN",
+        logging.ERROR: "ERROR",
+        logging.CRITICAL: "ERROR",
+    }
+
+    def __init__(self, service_name: str | None = None):
+        super().__init__()
+        self._service_name = service_name or dify_config.APPLICATION_NAME
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_dict = self._build_log_dict(record)
+        try:
+            return orjson.dumps(log_dict).decode("utf-8")
+        except TypeError:
+            # Fallback: convert non-serializable objects to string
+            import json
+
+            return json.dumps(log_dict, default=str, ensure_ascii=False)
+
+    def _build_log_dict(self, record: logging.LogRecord) -> dict[str, Any]:
+        # Core fields
+        log_dict: dict[str, Any] = {
+            "ts": datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
+            "severity": self.SEVERITY_MAP.get(record.levelno, "INFO"),
+            "service": self._service_name,
+            "caller": f"{record.filename}:{record.lineno}",
+            "message": record.getMessage(),
+        }
+
+        # Trace context (from TraceContextFilter)
+        trace_id = getattr(record, "trace_id", "")
+        span_id = getattr(record, "span_id", "")
+
+        if trace_id:
+            log_dict["trace_id"] = trace_id
+        if span_id:
+            log_dict["span_id"] = span_id
+
+        # Identity context (from IdentityContextFilter)
+        identity = self._extract_identity(record)
+        if identity:
+            log_dict["identity"] = identity
+
+        # Dynamic attributes
+        attributes = getattr(record, "attributes", None)
+        if attributes:
+            log_dict["attributes"] = attributes
+
+        # Stack trace for errors with exceptions
+        if record.exc_info and record.levelno >= logging.ERROR:
+            log_dict["stack_trace"] = self._format_exception(record.exc_info)
+
+        return log_dict
+
+    def _extract_identity(self, record: logging.LogRecord) -> dict[str, str] | None:
+        tenant_id = getattr(record, "tenant_id", None)
+        user_id = getattr(record, "user_id", None)
+        user_type = getattr(record, "user_type", None)
+
+        if not any([tenant_id, user_id, user_type]):
+            return None
+
+        identity: dict[str, str] = {}
+        if tenant_id:
+            identity["tenant_id"] = tenant_id
+        if user_id:
+            identity["user_id"] = user_id
+        if user_type:
+            identity["user_type"] = user_type
+        return identity
+
+    def _format_exception(self, exc_info: tuple[Any, ...]) -> str:
+        if exc_info and exc_info[0] is not None:
+            return "".join(traceback.format_exception(*exc_info))
+        return ""

api/core/mcp/client/streamable_client.py

@@ -313,17 +313,20 @@ class StreamableHTTPTransport:
             if is_initialization:
                 self._maybe_extract_session_id_from_response(response)
 
-            content_type = cast(str, response.headers.get(CONTENT_TYPE, "").lower())
+            # Per https://modelcontextprotocol.io/specification/2025-06-18/basic#notifications:
+            # The server MUST NOT send a response to notifications.
+            if isinstance(message.root, JSONRPCRequest):
+                content_type = cast(str, response.headers.get(CONTENT_TYPE, "").lower())
 
-            if content_type.startswith(JSON):
-                self._handle_json_response(response, ctx.server_to_client_queue)
-            elif content_type.startswith(SSE):
-                self._handle_sse_response(response, ctx)
-            else:
-                self._handle_unexpected_content_type(
-                    content_type,
-                    ctx.server_to_client_queue,
-                )
+                if content_type.startswith(JSON):
+                    self._handle_json_response(response, ctx.server_to_client_queue)
+                elif content_type.startswith(SSE):
+                    self._handle_sse_response(response, ctx)
+                else:
+                    self._handle_unexpected_content_type(
+                        content_type,
+                        ctx.server_to_client_queue,
+                    )
 
     def _handle_json_response(
         self,

api/core/model_runtime/entities/provider_entities.py

@@ -100,7 +100,6 @@ class SimpleProviderEntity(BaseModel):
     label: I18nObject
     icon_small: I18nObject | None = None
     icon_small_dark: I18nObject | None = None
-    icon_large: I18nObject | None = None
     supported_model_types: Sequence[ModelType]
     models: list[AIModelEntity] = []
 
@@ -123,7 +122,6 @@ class ProviderEntity(BaseModel):
     label: I18nObject
     description: I18nObject | None = None
     icon_small: I18nObject | None = None
-    icon_large: I18nObject | None = None
     icon_small_dark: I18nObject | None = None
     background: str | None = None
     help: ProviderHelpEntity | None = None
@@ -157,7 +155,6 @@ class ProviderEntity(BaseModel):
             provider=self.provider,
             label=self.label,
             icon_small=self.icon_small,
-            icon_large=self.icon_large,
             supported_model_types=self.supported_model_types,
             models=self.models,
         )

api/core/model_runtime/model_providers/model_provider_factory.py

@@ -285,7 +285,7 @@ class ModelProviderFactory:
         """
         Get provider icon
         :param provider: provider name
-        :param icon_type: icon type (icon_small or icon_large)
+        :param icon_type: icon type (icon_small or icon_small_dark)
         :param lang: language (zh_Hans or en_US)
         :return: provider icon
         """
@@ -309,13 +309,7 @@ class ModelProviderFactory:
             else:
                 file_name = provider_schema.icon_small_dark.en_US
         else:
-            if not provider_schema.icon_large:
-                raise ValueError(f"Provider {provider} does not have large icon.")
-
-            if lang.lower() == "zh_hans":
-                file_name = provider_schema.icon_large.zh_Hans
-            else:
-                file_name = provider_schema.icon_large.en_US
+            raise ValueError(f"Unsupported icon type: {icon_type}.")
 
         if not file_name:
             raise ValueError(f"Provider {provider} does not have icon.")

api/core/plugin/entities/parameters.py

@@ -76,7 +76,7 @@ class PluginParameter(BaseModel):
     auto_generate: PluginParameterAutoGenerate | None = None
     template: PluginParameterTemplate | None = None
     required: bool = False
-    default: Union[float, int, str, bool] | None = None
+    default: Union[float, int, str, bool, list, dict] | None = None
     min: Union[float, int] | None = None
     max: Union[float, int] | None = None
     precision: int | None = None

api/core/plugin/impl/base.py

@@ -103,6 +103,9 @@ class BasePluginClient:
         prepared_headers["X-Api-Key"] = dify_config.PLUGIN_DAEMON_KEY
         prepared_headers.setdefault("Accept-Encoding", "gzip, deflate, br")
 
+        # Inject traceparent header for distributed tracing
+        self._inject_trace_headers(prepared_headers)
+
         prepared_data: bytes | dict[str, Any] | str | None = (
             data if isinstance(data, (bytes, str, dict)) or data is None else None
         )
@@ -114,6 +117,31 @@ class BasePluginClient:
 
         return str(url), prepared_headers, prepared_data, params, files
 
+    def _inject_trace_headers(self, headers: dict[str, str]) -> None:
+        """
+        Inject W3C traceparent header for distributed tracing.
+
+        This ensures trace context is propagated to plugin daemon even if
+        HTTPXClientInstrumentor doesn't cover module-level httpx functions.
+        """
+        if not dify_config.ENABLE_OTEL:
+            return
+
+        import contextlib
+
+        # Skip if already present (case-insensitive check)
+        for key in headers:
+            if key.lower() == "traceparent":
+                return
+
+        # Inject traceparent - works as fallback when OTEL instrumentation doesn't cover this call
+        with contextlib.suppress(Exception):
+            from core.helper.trace_id_helper import generate_traceparent_header
+
+            traceparent = generate_traceparent_header()
+            if traceparent:
+                headers["traceparent"] = traceparent
+
     def _stream_request(
         self,
         method: str,

api/core/provider_manager.py

@@ -331,7 +331,6 @@ class ProviderManager:
                 provider=provider_schema.provider,
                 label=provider_schema.label,
                 icon_small=provider_schema.icon_small,
-                icon_large=provider_schema.icon_large,
                 supported_model_types=provider_schema.supported_model_types,
             ),
         )

api/core/rag/cleaner/clean_processor.py

@@ -27,26 +27,44 @@ class CleanProcessor:
                     pattern = r"([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)"
                     text = re.sub(pattern, "", text)
 
-                    # Remove URL but keep Markdown image URLs
-                    # First, temporarily replace Markdown image URLs with a placeholder
-                    markdown_image_pattern = r"!\[.*?\]\((https?://[^\s)]+)\)"
-                    placeholders: list[str] = []
+                    # Remove URL but keep Markdown image URLs and link URLs
+                    # Replace the ENTIRE markdown link/image with a single placeholder to protect
+                    # the link text (which might also be a URL) from being removed
+                    markdown_link_pattern = r"\[([^\]]*)\]\((https?://[^)]+)\)"
+                    markdown_image_pattern = r"!\[.*?\]\((https?://[^)]+)\)"
+                    placeholders: list[tuple[str, str, str]] = []  # (type, text, url)
 
-                    def replace_with_placeholder(match, placeholders=placeholders):
+                    def replace_markdown_with_placeholder(match, placeholders=placeholders):
+                        link_type = "link"
+                        link_text = match.group(1)
+                        url = match.group(2)
+                        placeholder = f"__MARKDOWN_PLACEHOLDER_{len(placeholders)}__"
+                        placeholders.append((link_type, link_text, url))
+                        return placeholder
+
+                    def replace_image_with_placeholder(match, placeholders=placeholders):
+                        link_type = "image"
                         url = match.group(1)
-                        placeholder = f"__MARKDOWN_IMAGE_URL_{len(placeholders)}__"
-                        placeholders.append(url)
-                        return f"![image]({placeholder})"
+                        placeholder = f"__MARKDOWN_PLACEHOLDER_{len(placeholders)}__"
+                        placeholders.append((link_type, "image", url))
+                        return placeholder
 
-                    text = re.sub(markdown_image_pattern, replace_with_placeholder, text)
+                    # Protect markdown links first
+                    text = re.sub(markdown_link_pattern, replace_markdown_with_placeholder, text)
+                    # Then protect markdown images
+                    text = re.sub(markdown_image_pattern, replace_image_with_placeholder, text)
 
                     # Now remove all remaining URLs
-                    url_pattern = r"https?://[^\s)]+"
+                    url_pattern = r"https?://\S+"
                     text = re.sub(url_pattern, "", text)
 
-                    # Finally, restore the Markdown image URLs
-                    for i, url in enumerate(placeholders):
-                        text = text.replace(f"__MARKDOWN_IMAGE_URL_{i}__", url)
+                    # Restore the Markdown links and images
+                    for i, (link_type, text_or_alt, url) in enumerate(placeholders):
+                        placeholder = f"__MARKDOWN_PLACEHOLDER_{i}__"
+                        if link_type == "link":
+                            text = text.replace(placeholder, f"[{text_or_alt}]({url})")
+                        else:  # image
+                            text = text.replace(placeholder, f"![{text_or_alt}]({url})")
         return text
 
     def filter_string(self, text):

api/core/rag/datasource/retrieval_service.py

@@ -1,4 +1,5 @@
 import concurrent.futures
+import logging
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any
 
@@ -13,7 +14,7 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.rag.data_post_processor.data_post_processor import DataPostProcessor
 from core.rag.datasource.keyword.keyword_factory import Keyword
 from core.rag.datasource.vdb.vector_factory import Vector
-from core.rag.embedding.retrieval import RetrievalSegments
+from core.rag.embedding.retrieval import RetrievalChildChunk, RetrievalSegments
 from core.rag.entities.metadata_entities import MetadataCondition
 from core.rag.index_processor.constant.doc_type import DocType
 from core.rag.index_processor.constant.index_type import IndexStructureType
@@ -36,6 +37,8 @@ default_retrieval_model = {
     "score_threshold_enabled": False,
 }
 
+logger = logging.getLogger(__name__)
+
 
 class RetrievalService:
     # Cache precompiled regular expressions to avoid repeated compilation
@@ -106,7 +109,12 @@ class RetrievalService:
                         )
                     )
 
-            concurrent.futures.wait(futures, timeout=3600, return_when=concurrent.futures.ALL_COMPLETED)
+            if futures:
+                for future in concurrent.futures.as_completed(futures, timeout=3600):
+                    if exceptions:
+                        for f in futures:
+                            f.cancel()
+                        break
 
         if exceptions:
             raise ValueError(";\n".join(exceptions))
@@ -210,6 +218,7 @@ class RetrievalService:
                 )
                 all_documents.extend(documents)
             except Exception as e:
+                logger.error(e, exc_info=True)
                 exceptions.append(str(e))
 
     @classmethod
@@ -303,6 +312,7 @@ class RetrievalService:
                     else:
                         all_documents.extend(documents)
             except Exception as e:
+                logger.error(e, exc_info=True)
                 exceptions.append(str(e))
 
     @classmethod
@@ -351,6 +361,7 @@ class RetrievalService:
                     else:
                         all_documents.extend(documents)
             except Exception as e:
+                logger.error(e, exc_info=True)
                 exceptions.append(str(e))
 
     @staticmethod
@@ -381,10 +392,9 @@ class RetrievalService:
             records = []
             include_segment_ids = set()
             segment_child_map = {}
-            segment_file_map = {}
 
             valid_dataset_documents = {}
-            image_doc_ids = []
+            image_doc_ids: list[Any] = []
             child_index_node_ids = []
             index_node_ids = []
             doc_to_document_map = {}
@@ -417,28 +427,39 @@ class RetrievalService:
             child_index_node_ids = [i for i in child_index_node_ids if i]
             index_node_ids = [i for i in index_node_ids if i]
 
-            segment_ids = []
+            segment_ids: list[str] = []
             index_node_segments: list[DocumentSegment] = []
             segments: list[DocumentSegment] = []
-            attachment_map = {}
-            child_chunk_map = {}
-            doc_segment_map = {}
+            attachment_map: dict[str, list[dict[str, Any]]] = {}
+            child_chunk_map: dict[str, list[ChildChunk]] = {}
+            doc_segment_map: dict[str, list[str]] = {}
 
             with session_factory.create_session() as session:
                 attachments = cls.get_segment_attachment_infos(image_doc_ids, session)
 
                 for attachment in attachments:
                     segment_ids.append(attachment["segment_id"])
-                    attachment_map[attachment["segment_id"]] = attachment
-                    doc_segment_map[attachment["segment_id"]] = attachment["attachment_id"]
-
+                    if attachment["segment_id"] in attachment_map:
+                        attachment_map[attachment["segment_id"]].append(attachment["attachment_info"])
+                    else:
+                        attachment_map[attachment["segment_id"]] = [attachment["attachment_info"]]
+                    if attachment["segment_id"] in doc_segment_map:
+                        doc_segment_map[attachment["segment_id"]].append(attachment["attachment_id"])
+                    else:
+                        doc_segment_map[attachment["segment_id"]] = [attachment["attachment_id"]]
                 child_chunk_stmt = select(ChildChunk).where(ChildChunk.index_node_id.in_(child_index_node_ids))
                 child_index_nodes = session.execute(child_chunk_stmt).scalars().all()
 
                 for i in child_index_nodes:
                     segment_ids.append(i.segment_id)
-                    child_chunk_map[i.segment_id] = i
-                    doc_segment_map[i.segment_id] = i.index_node_id
+                    if i.segment_id in child_chunk_map:
+                        child_chunk_map[i.segment_id].append(i)
+                    else:
+                        child_chunk_map[i.segment_id] = [i]
+                    if i.segment_id in doc_segment_map:
+                        doc_segment_map[i.segment_id].append(i.index_node_id)
+                    else:
+                        doc_segment_map[i.segment_id] = [i.index_node_id]
 
                 if index_node_ids:
                     document_segment_stmt = select(DocumentSegment).where(
@@ -448,7 +469,7 @@ class RetrievalService:
                     )
                     index_node_segments = session.execute(document_segment_stmt).scalars().all()  # type: ignore
                     for index_node_segment in index_node_segments:
-                        doc_segment_map[index_node_segment.id] = index_node_segment.index_node_id
+                        doc_segment_map[index_node_segment.id] = [index_node_segment.index_node_id]
                 if segment_ids:
                     document_segment_stmt = select(DocumentSegment).where(
                         DocumentSegment.enabled == True,
@@ -461,95 +482,86 @@ class RetrievalService:
                     segments.extend(index_node_segments)
 
             for segment in segments:
-                doc_id = doc_segment_map.get(segment.id)
-                child_chunk = child_chunk_map.get(segment.id)
-                attachment_info = attachment_map.get(segment.id)
+                child_chunks: list[ChildChunk] = child_chunk_map.get(segment.id, [])
+                attachment_infos: list[dict[str, Any]] = attachment_map.get(segment.id, [])
+                ds_dataset_document: DatasetDocument | None = valid_dataset_documents.get(segment.document_id)
 
-                if doc_id:
-                    document = doc_to_document_map[doc_id]
-                    ds_dataset_document: DatasetDocument | None = valid_dataset_documents.get(
-                        document.metadata.get("document_id")
-                    )
-
-                    if ds_dataset_document and ds_dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
-                        if segment.id not in include_segment_ids:
-                            include_segment_ids.add(segment.id)
-                            if child_chunk:
+                if ds_dataset_document and ds_dataset_document.doc_form == IndexStructureType.PARENT_CHILD_INDEX:
+                    if segment.id not in include_segment_ids:
+                        include_segment_ids.add(segment.id)
+                        if child_chunks or attachment_infos:
+                            child_chunk_details = []
+                            max_score = 0.0
+                            for child_chunk in child_chunks:
+                                document = doc_to_document_map[child_chunk.index_node_id]
                                 child_chunk_detail = {
                                     "id": child_chunk.id,
                                     "content": child_chunk.content,
                                     "position": child_chunk.position,
                                     "score": document.metadata.get("score", 0.0) if document else 0.0,
                                 }
-                                map_detail = {
-                                    "max_score": document.metadata.get("score", 0.0) if document else 0.0,
-                                    "child_chunks": [child_chunk_detail],
-                                }
-                                segment_child_map[segment.id] = map_detail
-                            record = {
-                                "segment": segment,
+                                child_chunk_details.append(child_chunk_detail)
+                                max_score = max(max_score, document.metadata.get("score", 0.0) if document else 0.0)
+                            for attachment_info in attachment_infos:
+                                file_document = doc_to_document_map[attachment_info["id"]]
+                                max_score = max(
+                                    max_score, file_document.metadata.get("score", 0.0) if file_document else 0.0
+                                )
+
+                            map_detail = {
+                                "max_score": max_score,
+                                "child_chunks": child_chunk_details,
                             }
-                            if attachment_info:
-                                segment_file_map[segment.id] = [attachment_info]
-                            records.append(record)
-                        else:
-                            if child_chunk:
-                                child_chunk_detail = {
-                                    "id": child_chunk.id,
-                                    "content": child_chunk.content,
-                                    "position": child_chunk.position,
-                                    "score": document.metadata.get("score", 0.0),
-                                }
-                                if segment.id in segment_child_map:
-                                    segment_child_map[segment.id]["child_chunks"].append(child_chunk_detail)  # type: ignore
-                                    segment_child_map[segment.id]["max_score"] = max(
-                                        segment_child_map[segment.id]["max_score"],
-                                        document.metadata.get("score", 0.0) if document else 0.0,
-                                    )
-                                else:
-                                    segment_child_map[segment.id] = {
-                                        "max_score": document.metadata.get("score", 0.0) if document else 0.0,
-                                        "child_chunks": [child_chunk_detail],
-                                    }
-                            if attachment_info:
-                                if segment.id in segment_file_map:
-                                    segment_file_map[segment.id].append(attachment_info)
-                                else:
-                                    segment_file_map[segment.id] = [attachment_info]
-                    else:
-                        if segment.id not in include_segment_ids:
-                            include_segment_ids.add(segment.id)
-                            record = {
-                                "segment": segment,
-                                "score": document.metadata.get("score", 0.0),  # type: ignore
-                            }
-                            if attachment_info:
-                                segment_file_map[segment.id] = [attachment_info]
-                            records.append(record)
-                        else:
-                            if attachment_info:
-                                attachment_infos = segment_file_map.get(segment.id, [])
-                                if attachment_info not in attachment_infos:
-                                    attachment_infos.append(attachment_info)
-                                segment_file_map[segment.id] = attachment_infos
+                            segment_child_map[segment.id] = map_detail
+                        record: dict[str, Any] = {
+                            "segment": segment,
+                        }
+                        records.append(record)
+                else:
+                    if segment.id not in include_segment_ids:
+                        include_segment_ids.add(segment.id)
+                        max_score = 0.0
+                        segment_document = doc_to_document_map.get(segment.index_node_id)
+                        if segment_document:
+                            max_score = max(max_score, segment_document.metadata.get("score", 0.0))
+                        for attachment_info in attachment_infos:
+                            file_doc = doc_to_document_map.get(attachment_info["id"])
+                            if file_doc:
+                                max_score = max(max_score, file_doc.metadata.get("score", 0.0))
+                        record = {
+                            "segment": segment,
+                            "score": max_score,
+                        }
+                        records.append(record)
 
             # Add child chunks information to records
             for record in records:
                 if record["segment"].id in segment_child_map:
                     record["child_chunks"] = segment_child_map[record["segment"].id].get("child_chunks")  # type: ignore
                     record["score"] = segment_child_map[record["segment"].id]["max_score"]  # type: ignore
-                if record["segment"].id in segment_file_map:
-                    record["files"] = segment_file_map[record["segment"].id]  # type: ignore[assignment]
+                if record["segment"].id in attachment_map:
+                    record["files"] = attachment_map[record["segment"].id]  # type: ignore[assignment]
 
-            result = []
+            result: list[RetrievalSegments] = []
             for record in records:
                 # Extract segment
                 segment = record["segment"]
 
                 # Extract child_chunks, ensuring it's a list or None
-                child_chunks = record.get("child_chunks")
-                if not isinstance(child_chunks, list):
-                    child_chunks = None
+                raw_child_chunks = record.get("child_chunks")
+                child_chunks_list: list[RetrievalChildChunk] | None = None
+                if isinstance(raw_child_chunks, list):
+                    # Sort by score descending
+                    sorted_chunks = sorted(raw_child_chunks, key=lambda x: x.get("score", 0.0), reverse=True)
+                    child_chunks_list = [
+                        RetrievalChildChunk(
+                            id=chunk["id"],
+                            content=chunk["content"],
+                            score=chunk.get("score", 0.0),
+                            position=chunk["position"],
+                        )
+                        for chunk in sorted_chunks
+                    ]
 
                 # Extract files, ensuring it's a list or None
                 files = record.get("files")
@@ -566,11 +578,11 @@ class RetrievalService:
 
                 # Create RetrievalSegments object
                 retrieval_segment = RetrievalSegments(
-                    segment=segment, child_chunks=child_chunks, score=score, files=files
+                    segment=segment, child_chunks=child_chunks_list, score=score, files=files
                 )
                 result.append(retrieval_segment)
 
-            return result
+            return sorted(result, key=lambda x: x.score if x.score is not None else 0.0, reverse=True)
         except Exception as e:
             db.session.rollback()
             raise e
@@ -662,7 +674,14 @@ class RetrievalService:
                             document_ids_filter=document_ids_filter,
                         )
                     )
-                concurrent.futures.wait(futures, timeout=300, return_when=concurrent.futures.ALL_COMPLETED)
+                # Use as_completed for early error propagation - cancel remaining futures on first error
+                if futures:
+                    for future in concurrent.futures.as_completed(futures, timeout=300):
+                        if future.exception():
+                            # Cancel remaining futures to avoid unnecessary waiting
+                            for f in futures:
+                                f.cancel()
+                            break
 
             if exceptions:
                 raise ValueError(";\n".join(exceptions))

api/core/rag/datasource/vdb/pgvector/pgvector.py

@@ -255,7 +255,10 @@ class PGVector(BaseVector):
                 return
 
             with self._get_cursor() as cur:
-                cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
+                cur.execute("SELECT 1 FROM pg_extension WHERE extname = 'vector'")
+                if not cur.fetchone():
+                    cur.execute("CREATE EXTENSION vector")
+
                 cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension))
                 # PG hnsw index only support 2000 dimension or less
                 # ref: https://github.com/pgvector/pgvector?tab=readme-ov-file#indexing

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -66,6 +66,8 @@ class WeaviateVector(BaseVector):
     in a Weaviate collection.
     """
 
+    _DOCUMENT_ID_PROPERTY = "document_id"
+
     def __init__(self, collection_name: str, config: WeaviateConfig, attributes: list):
         """
         Initializes the Weaviate vector store.
@@ -353,15 +355,12 @@ class WeaviateVector(BaseVector):
             return []
 
         col = self._client.collections.use(self._collection_name)
-        props = list({*self._attributes, "document_id", Field.TEXT_KEY.value})
+        props = list({*self._attributes, self._DOCUMENT_ID_PROPERTY, Field.TEXT_KEY.value})
 
         where = None
         doc_ids = kwargs.get("document_ids_filter") or []
         if doc_ids:
-            ors = [Filter.by_property("document_id").equal(x) for x in doc_ids]
-            where = ors[0]
-            for f in ors[1:]:
-                where = where | f
+            where = Filter.by_property(self._DOCUMENT_ID_PROPERTY).contains_any(doc_ids)
 
         top_k = int(kwargs.get("top_k", 4))
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
@@ -408,10 +407,7 @@ class WeaviateVector(BaseVector):
         where = None
         doc_ids = kwargs.get("document_ids_filter") or []
         if doc_ids:
-            ors = [Filter.by_property("document_id").equal(x) for x in doc_ids]
-            where = ors[0]
-            for f in ors[1:]:
-                where = where | f
+            where = Filter.by_property(self._DOCUMENT_ID_PROPERTY).contains_any(doc_ids)
 
         top_k = int(kwargs.get("top_k", 4))
 

api/core/rag/extractor/extract_processor.py

@@ -112,7 +112,7 @@ class ExtractProcessor:
                     if file_extension in {".xlsx", ".xls"}:
                         extractor = ExcelExtractor(file_path)
                     elif file_extension == ".pdf":
-                        extractor = PdfExtractor(file_path)
+                        extractor = PdfExtractor(file_path, upload_file.tenant_id, upload_file.created_by)
                     elif file_extension in {".md", ".markdown", ".mdx"}:
                         extractor = (
                             UnstructuredMarkdownExtractor(file_path, unstructured_api_url, unstructured_api_key)
@@ -148,7 +148,7 @@ class ExtractProcessor:
                     if file_extension in {".xlsx", ".xls"}:
                         extractor = ExcelExtractor(file_path)
                     elif file_extension == ".pdf":
-                        extractor = PdfExtractor(file_path)
+                        extractor = PdfExtractor(file_path, upload_file.tenant_id, upload_file.created_by)
                     elif file_extension in {".md", ".markdown", ".mdx"}:
                         extractor = MarkdownExtractor(file_path, autodetect_encoding=True)
                     elif file_extension in {".htm", ".html"}:

api/core/rag/extractor/pdf_extractor.py

@@ -1,25 +1,57 @@
 """Abstract interface for document loader implementations."""
 
 import contextlib
+import io
+import logging
+import uuid
 from collections.abc import Iterator
 
+import pypdfium2
+import pypdfium2.raw as pdfium_c
+
+from configs import dify_config
 from core.rag.extractor.blob.blob import Blob
 from core.rag.extractor.extractor_base import BaseExtractor
 from core.rag.models.document import Document
+from extensions.ext_database import db
 from extensions.ext_storage import storage
+from libs.datetime_utils import naive_utc_now
+from models.enums import CreatorUserRole
+from models.model import UploadFile
+
+logger = logging.getLogger(__name__)
 
 
 class PdfExtractor(BaseExtractor):
-    """Load pdf files.
-
+    """
+    PdfExtractor is used to extract text and images from PDF files.
 
     Args:
-        file_path: Path to the file to load.
+        file_path: Path to the PDF file.
+        tenant_id: Workspace ID.
+        user_id: ID of the user performing the extraction.
+        file_cache_key: Optional cache key for the extracted text.
     """
 
-    def __init__(self, file_path: str, file_cache_key: str | None = None):
-        """Initialize with file path."""
+    # Magic bytes for image format detection: (magic_bytes, extension, mime_type)
+    IMAGE_FORMATS = [
+        (b"\xff\xd8\xff", "jpg", "image/jpeg"),
+        (b"\x89PNG\r\n\x1a\n", "png", "image/png"),
+        (b"\x00\x00\x00\x0c\x6a\x50\x20\x20\x0d\x0a\x87\x0a", "jp2", "image/jp2"),
+        (b"GIF8", "gif", "image/gif"),
+        (b"BM", "bmp", "image/bmp"),
+        (b"II*\x00", "tiff", "image/tiff"),
+        (b"MM\x00*", "tiff", "image/tiff"),
+        (b"II+\x00", "tiff", "image/tiff"),
+        (b"MM\x00+", "tiff", "image/tiff"),
+    ]
+    MAX_MAGIC_LEN = max(len(m) for m, _, _ in IMAGE_FORMATS)
+
+    def __init__(self, file_path: str, tenant_id: str, user_id: str, file_cache_key: str | None = None):
+        """Initialize PdfExtractor."""
         self._file_path = file_path
+        self._tenant_id = tenant_id
+        self._user_id = user_id
         self._file_cache_key = file_cache_key
 
     def extract(self) -> list[Document]:
@@ -50,7 +82,6 @@ class PdfExtractor(BaseExtractor):
 
     def parse(self, blob: Blob) -> Iterator[Document]:
         """Lazily parse the blob."""
-        import pypdfium2  # type: ignore
 
         with blob.as_bytes_io() as file_path:
             pdf_reader = pypdfium2.PdfDocument(file_path, autoclose=True)
@@ -59,8 +90,87 @@ class PdfExtractor(BaseExtractor):
                     text_page = page.get_textpage()
                     content = text_page.get_text_range()
                     text_page.close()
+
+                    image_content = self._extract_images(page)
+                    if image_content:
+                        content += "\n" + image_content
+
                     page.close()
                     metadata = {"source": blob.source, "page": page_number}
                     yield Document(page_content=content, metadata=metadata)
             finally:
                 pdf_reader.close()
+
+    def _extract_images(self, page) -> str:
+        """
+        Extract images from a PDF page, save them to storage and database,
+        and return markdown image links.
+
+        Args:
+            page: pypdfium2 page object.
+
+        Returns:
+            Markdown string containing links to the extracted images.
+        """
+        image_content = []
+        upload_files = []
+        base_url = dify_config.INTERNAL_FILES_URL or dify_config.FILES_URL
+
+        try:
+            image_objects = page.get_objects(filter=(pdfium_c.FPDF_PAGEOBJ_IMAGE,))
+            for obj in image_objects:
+                try:
+                    # Extract image bytes
+                    img_byte_arr = io.BytesIO()
+                    # Extract DCTDecode (JPEG) and JPXDecode (JPEG 2000) images directly
+                    # Fallback to png for other formats
+                    obj.extract(img_byte_arr, fb_format="png")
+                    img_bytes = img_byte_arr.getvalue()
+
+                    if not img_bytes:
+                        continue
+
+                    header = img_bytes[: self.MAX_MAGIC_LEN]
+                    image_ext = None
+                    mime_type = None
+                    for magic, ext, mime in self.IMAGE_FORMATS:
+                        if header.startswith(magic):
+                            image_ext = ext
+                            mime_type = mime
+                            break
+
+                    if not image_ext or not mime_type:
+                        continue
+
+                    file_uuid = str(uuid.uuid4())
+                    file_key = "image_files/" + self._tenant_id + "/" + file_uuid + "." + image_ext
+
+                    storage.save(file_key, img_bytes)
+
+                    # save file to db
+                    upload_file = UploadFile(
+                        tenant_id=self._tenant_id,
+                        storage_type=dify_config.STORAGE_TYPE,
+                        key=file_key,
+                        name=file_key,
+                        size=len(img_bytes),
+                        extension=image_ext,
+                        mime_type=mime_type,
+                        created_by=self._user_id,
+                        created_by_role=CreatorUserRole.ACCOUNT,
+                        created_at=naive_utc_now(),
+                        used=True,
+                        used_by=self._user_id,
+                        used_at=naive_utc_now(),
+                    )
+                    upload_files.append(upload_file)
+                    image_content.append(f"![image]({base_url}/files/{upload_file.id}/file-preview)")
+                except Exception as e:
+                    logger.warning("Failed to extract image from PDF: %s", e)
+                    continue
+        except Exception as e:
+            logger.warning("Failed to get objects from PDF page: %s", e)
+        if upload_files:
+            db.session.add_all(upload_files)
+            db.session.commit()
+        return "\n".join(image_content)

api/core/rag/retrieval/dataset_retrieval.py

@@ -7,7 +7,7 @@ from collections.abc import Generator, Mapping
 from typing import Any, Union, cast
 
 from flask import Flask, current_app
-from sqlalchemy import and_, or_, select
+from sqlalchemy import and_, literal, or_, select
 from sqlalchemy.orm import Session
 
 from core.app.app_config.entities import (
@@ -515,7 +515,11 @@ class DatasetRetrieval:
                         0
                     ].embedding_model_provider
                     weights["vector_setting"]["embedding_model_name"] = available_datasets[0].embedding_model
+        dataset_count = len(available_datasets)
         with measure_time() as timer:
+            cancel_event = threading.Event()
+            thread_exceptions: list[Exception] = []
+
             if query:
                 query_thread = threading.Thread(
                     target=self._multiple_retrieve_thread,
@@ -534,6 +538,9 @@ class DatasetRetrieval:
                         "score_threshold": score_threshold,
                         "query": query,
                         "attachment_id": None,
+                        "dataset_count": dataset_count,
+                        "cancel_event": cancel_event,
+                        "thread_exceptions": thread_exceptions,
                     },
                 )
                 all_threads.append(query_thread)
@@ -557,12 +564,26 @@ class DatasetRetrieval:
                             "score_threshold": score_threshold,
                             "query": None,
                             "attachment_id": attachment_id,
+                            "dataset_count": dataset_count,
+                            "cancel_event": cancel_event,
+                            "thread_exceptions": thread_exceptions,
                         },
                     )
                     all_threads.append(attachment_thread)
                     attachment_thread.start()
-            for thread in all_threads:
-                thread.join()
+
+            # Poll threads with short timeout to detect errors quickly (fail-fast)
+            while any(t.is_alive() for t in all_threads):
+                for thread in all_threads:
+                    thread.join(timeout=0.1)
+                    if thread_exceptions:
+                        cancel_event.set()
+                        break
+                if thread_exceptions:
+                    break
+
+            if thread_exceptions:
+                raise thread_exceptions[0]
         self._on_query(query, attachment_ids, dataset_ids, app_id, user_from, user_id)
 
         if all_documents:
@@ -1036,7 +1057,7 @@ class DatasetRetrieval:
             if automatic_metadata_filters:
                 conditions = []
                 for sequence, filter in enumerate(automatic_metadata_filters):
-                    self._process_metadata_filter_func(
+                    self.process_metadata_filter_func(
                         sequence,
                         filter.get("condition"),  # type: ignore
                         filter.get("metadata_name"),  # type: ignore
@@ -1072,7 +1093,7 @@ class DatasetRetrieval:
                             value=expected_value,
                         )
                     )
-                    filters = self._process_metadata_filter_func(
+                    filters = self.process_metadata_filter_func(
                         sequence,
                         condition.comparison_operator,
                         metadata_name,
@@ -1168,8 +1189,9 @@ class DatasetRetrieval:
             return None
         return automatic_metadata_filters
 
-    def _process_metadata_filter_func(
-        self, sequence: int, condition: str, metadata_name: str, value: Any | None, filters: list
+    @classmethod
+    def process_metadata_filter_func(
+        cls, sequence: int, condition: str, metadata_name: str, value: Any | None, filters: list
     ):
         if value is None and condition not in ("empty", "not empty"):
             return filters
@@ -1218,6 +1240,20 @@ class DatasetRetrieval:
 
             case "≥" | ">=":
                 filters.append(DatasetDocument.doc_metadata[metadata_name].as_float() >= value)
+            case "in" | "not in":
+                if isinstance(value, str):
+                    value_list = [v.strip() for v in value.split(",") if v.strip()]
+                elif isinstance(value, (list, tuple)):
+                    value_list = [str(v) for v in value if v is not None]
+                else:
+                    value_list = [str(value)] if value is not None else []
+
+                if not value_list:
+                    # `field in []` is False, `field not in []` is True
+                    filters.append(literal(condition == "not in"))
+                else:
+                    op = json_field.in_ if condition == "in" else json_field.notin_
+                    filters.append(op(value_list))
             case _:
                 pass
 
@@ -1389,69 +1425,89 @@ class DatasetRetrieval:
         score_threshold: float,
         query: str | None,
         attachment_id: str | None,
+        dataset_count: int,
+        cancel_event: threading.Event | None = None,
+        thread_exceptions: list[Exception] | None = None,
     ):
-        with flask_app.app_context():
-            threads = []
-            all_documents_item: list[Document] = []
-            index_type = None
-            for dataset in available_datasets:
-                index_type = dataset.indexing_technique
-                document_ids_filter = None
-                if dataset.provider != "external":
-                    if metadata_condition and not metadata_filter_document_ids:
-                        continue
-                    if metadata_filter_document_ids:
-                        document_ids = metadata_filter_document_ids.get(dataset.id, [])
-                        if document_ids:
-                            document_ids_filter = document_ids
-                        else:
+        try:
+            with flask_app.app_context():
+                threads = []
+                all_documents_item: list[Document] = []
+                index_type = None
+                for dataset in available_datasets:
+                    # Check for cancellation signal
+                    if cancel_event and cancel_event.is_set():
+                        break
+                    index_type = dataset.indexing_technique
+                    document_ids_filter = None
+                    if dataset.provider != "external":
+                        if metadata_condition and not metadata_filter_document_ids:
                             continue
-                retrieval_thread = threading.Thread(
-                    target=self._retriever,
-                    kwargs={
-                        "flask_app": flask_app,
-                        "dataset_id": dataset.id,
-                        "query": query,
-                        "top_k": top_k,
-                        "all_documents": all_documents_item,
-                        "document_ids_filter": document_ids_filter,
-                        "metadata_condition": metadata_condition,
-                        "attachment_ids": [attachment_id] if attachment_id else None,
-                    },
-                )
-                threads.append(retrieval_thread)
-                retrieval_thread.start()
-            for thread in threads:
-                thread.join()
+                        if metadata_filter_document_ids:
+                            document_ids = metadata_filter_document_ids.get(dataset.id, [])
+                            if document_ids:
+                                document_ids_filter = document_ids
+                            else:
+                                continue
+                    retrieval_thread = threading.Thread(
+                        target=self._retriever,
+                        kwargs={
+                            "flask_app": flask_app,
+                            "dataset_id": dataset.id,
+                            "query": query,
+                            "top_k": top_k,
+                            "all_documents": all_documents_item,
+                            "document_ids_filter": document_ids_filter,
+                            "metadata_condition": metadata_condition,
+                            "attachment_ids": [attachment_id] if attachment_id else None,
+                        },
+                    )
+                    threads.append(retrieval_thread)
+                    retrieval_thread.start()
 
-            if reranking_enable:
-                # do rerank for searched documents
-                data_post_processor = DataPostProcessor(tenant_id, reranking_mode, reranking_model, weights, False)
-                if query:
-                    all_documents_item = data_post_processor.invoke(
-                        query=query,
-                        documents=all_documents_item,
-                        score_threshold=score_threshold,
-                        top_n=top_k,
-                        query_type=QueryType.TEXT_QUERY,
-                    )
-                if attachment_id:
-                    all_documents_item = data_post_processor.invoke(
-                        documents=all_documents_item,
-                        score_threshold=score_threshold,
-                        top_n=top_k,
-                        query_type=QueryType.IMAGE_QUERY,
-                        query=attachment_id,
-                    )
-            else:
-                if index_type == IndexTechniqueType.ECONOMY:
-                    if not query:
-                        all_documents_item = []
-                    else:
-                        all_documents_item = self.calculate_keyword_score(query, all_documents_item, top_k)
-                elif index_type == IndexTechniqueType.HIGH_QUALITY:
-                    all_documents_item = self.calculate_vector_score(all_documents_item, top_k, score_threshold)
+                # Poll threads with short timeout to respond quickly to cancellation
+                while any(t.is_alive() for t in threads):
+                    for thread in threads:
+                        thread.join(timeout=0.1)
+                        if cancel_event and cancel_event.is_set():
+                            break
+                    if cancel_event and cancel_event.is_set():
+                        break
+
+                # Skip second reranking when there is only one dataset
+                if reranking_enable and dataset_count > 1:
+                    # do rerank for searched documents
+                    data_post_processor = DataPostProcessor(tenant_id, reranking_mode, reranking_model, weights, False)
+                    if query:
+                        all_documents_item = data_post_processor.invoke(
+                            query=query,
+                            documents=all_documents_item,
+                            score_threshold=score_threshold,
+                            top_n=top_k,
+                            query_type=QueryType.TEXT_QUERY,
+                        )
+                    if attachment_id:
+                        all_documents_item = data_post_processor.invoke(
+                            documents=all_documents_item,
+                            score_threshold=score_threshold,
+                            top_n=top_k,
+                            query_type=QueryType.IMAGE_QUERY,
+                            query=attachment_id,
+                        )
                 else:
-                    all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item
-            if all_documents_item:
-                all_documents.extend(all_documents_item)
+                    if index_type == IndexTechniqueType.ECONOMY:
+                        if not query:
+                            all_documents_item = []
+                        else:
+                            all_documents_item = self.calculate_keyword_score(query, all_documents_item, top_k)
+                    elif index_type == IndexTechniqueType.HIGH_QUALITY:
+                        all_documents_item = self.calculate_vector_score(all_documents_item, top_k, score_threshold)
+                    else:
+                        all_documents_item = all_documents_item[:top_k] if top_k else all_documents_item
+                if all_documents_item:
+                    all_documents.extend(all_documents_item)
+        except Exception as e:
+            if cancel_event:
+                cancel_event.set()
+            if thread_exceptions is not None:
+                thread_exceptions.append(e)

api/core/tools/mcp_tool/tool.py

@@ -6,7 +6,15 @@ from typing import Any
 
 from core.mcp.auth_client import MCPClientWithAuthRetry
 from core.mcp.error import MCPConnectionError
-from core.mcp.types import AudioContent, CallToolResult, ImageContent, TextContent
+from core.mcp.types import (
+    AudioContent,
+    BlobResourceContents,
+    CallToolResult,
+    EmbeddedResource,
+    ImageContent,
+    TextContent,
+    TextResourceContents,
+)
 from core.tools.__base.tool import Tool
 from core.tools.__base.tool_runtime import ToolRuntime
 from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolProviderType
@@ -53,10 +61,19 @@ class MCPTool(Tool):
         for content in result.content:
             if isinstance(content, TextContent):
                 yield from self._process_text_content(content)
-            elif isinstance(content, ImageContent):
-                yield self._process_image_content(content)
-            elif isinstance(content, AudioContent):
-                yield self._process_audio_content(content)
+            elif isinstance(content, ImageContent | AudioContent):
+                yield self.create_blob_message(
+                    blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType}
+                )
+            elif isinstance(content, EmbeddedResource):
+                resource = content.resource
+                if isinstance(resource, TextResourceContents):
+                    yield self.create_text_message(resource.text)
+                elif isinstance(resource, BlobResourceContents):
+                    mime_type = resource.mimeType or "application/octet-stream"
+                    yield self.create_blob_message(blob=base64.b64decode(resource.blob), meta={"mime_type": mime_type})
+                else:
+                    raise ToolInvokeError(f"Unsupported embedded resource type: {type(resource)}")
             else:
                 logger.warning("Unsupported content type=%s", type(content))
 
@@ -101,14 +118,6 @@ class MCPTool(Tool):
         for item in json_list:
             yield self.create_json_message(item)
 
-    def _process_image_content(self, content: ImageContent) -> ToolInvokeMessage:
-        """Process image content and return a blob message."""
-        return self.create_blob_message(blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType})
-
-    def _process_audio_content(self, content: AudioContent) -> ToolInvokeMessage:
-        """Process audio content and return a blob message."""
-        return self.create_blob_message(blob=base64.b64decode(content.data), meta={"mime_type": content.mimeType})
-
     def fork_tool_runtime(self, runtime: ToolRuntime) -> "MCPTool":
         return MCPTool(
             entity=self.entity,

api/core/tools/utils/parser.py

@@ -378,7 +378,7 @@ class ApiBasedToolSchemaParser:
     @staticmethod
     def auto_parse_to_tool_bundle(
         content: str, extra_info: dict | None = None, warning: dict | None = None
-    ) -> tuple[list[ApiToolBundle], str]:
+    ) -> tuple[list[ApiToolBundle], ApiProviderSchemaType]:
         """
         auto parse to tool bundle
 

api/core/tools/utils/text_processing_utils.py

@@ -4,6 +4,7 @@ import re
 def remove_leading_symbols(text: str) -> str:
     """
     Remove leading punctuation or symbols from the given text.
+    Preserves markdown links like [text](url) at the start.
 
     Args:
         text (str): The input text to process.
@@ -11,6 +12,11 @@ def remove_leading_symbols(text: str) -> str:
     Returns:
         str: The text with leading punctuation or symbols removed.
     """
+    # Check if text starts with a markdown link - preserve it
+    markdown_link_pattern = r"^\[([^\]]+)\]\((https?://[^)]+)\)"
+    if re.match(markdown_link_pattern, text):
+        return text
+
     # Match Unicode ranges for punctuation and symbols
     # FIXME this pattern is confused quick fix for #11868 maybe refactor it later
     pattern = r'^[\[\]\u2000-\u2025\u2027-\u206F\u2E00-\u2E7F\u3000-\u300F\u3011-\u303F"#$%&\'()*+,./:;<=>?@^_`~]+'

api/core/tools/workflow_as_tool/provider.py

@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
 
 from core.app.app_config.entities import VariableEntity, VariableEntityType
 from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
+from core.db.session_factory import session_factory
 from core.plugin.entities.parameters import PluginParameterOption
 from core.tools.__base.tool_provider import ToolProviderController
 from core.tools.__base.tool_runtime import ToolRuntime
@@ -47,33 +48,29 @@ class WorkflowToolProviderController(ToolProviderController):
 
     @classmethod
     def from_db(cls, db_provider: WorkflowToolProvider) -> "WorkflowToolProviderController":
-        with Session(db.engine, expire_on_commit=False) as session, session.begin():
-            provider = session.get(WorkflowToolProvider, db_provider.id) if db_provider.id else None
-            if not provider:
-                raise ValueError("workflow provider not found")
-            app = session.get(App, provider.app_id)
+        with session_factory.create_session() as session, session.begin():
+            app = session.get(App, db_provider.app_id)
             if not app:
                 raise ValueError("app not found")
 
-            user = session.get(Account, provider.user_id) if provider.user_id else None
-
+            user = session.get(Account, db_provider.user_id) if db_provider.user_id else None
             controller = WorkflowToolProviderController(
                 entity=ToolProviderEntity(
                     identity=ToolProviderIdentity(
                         author=user.name if user else "",
-                        name=provider.label,
-                        label=I18nObject(en_US=provider.label, zh_Hans=provider.label),
-                        description=I18nObject(en_US=provider.description, zh_Hans=provider.description),
-                        icon=provider.icon,
+                        name=db_provider.label,
+                        label=I18nObject(en_US=db_provider.label, zh_Hans=db_provider.label),
+                        description=I18nObject(en_US=db_provider.description, zh_Hans=db_provider.description),
+                        icon=db_provider.icon,
                     ),
                     credentials_schema=[],
                     plugin_id=None,
                 ),
-                provider_id=provider.id or "",
+                provider_id=db_provider.id,
             )
 
             controller.tools = [
-                controller._get_db_provider_tool(provider, app, session=session, user=user),
+                controller._get_db_provider_tool(db_provider, app, session=session, user=user),
             ]
 
         return controller

api/core/workflow/README.md

@@ -64,6 +64,9 @@ engine.layer(DebugLoggingLayer(level="INFO"))
 engine.layer(ExecutionLimitsLayer(max_nodes=100))
 ```
 
+`engine.layer()` binds the read-only runtime state before execution, so layer hooks
+can assume `graph_runtime_state` is available.
+
 ### Event-Driven Architecture
 
 All node executions emit events for monitoring and integration:

api/core/workflow/graph_engine/command_channels/redis_channel.py

@@ -9,7 +9,7 @@ Each instance uses a unique key for its command queue.
 import json
 from typing import TYPE_CHECKING, Any, final
 
-from ..entities.commands import AbortCommand, CommandType, GraphEngineCommand, PauseCommand
+from ..entities.commands import AbortCommand, CommandType, GraphEngineCommand, PauseCommand, UpdateVariablesCommand
 
 if TYPE_CHECKING:
     from extensions.ext_redis import RedisClientWrapper
@@ -113,6 +113,8 @@ class RedisChannel:
                 return AbortCommand.model_validate(data)
             if command_type == CommandType.PAUSE:
                 return PauseCommand.model_validate(data)
+            if command_type == CommandType.UPDATE_VARIABLES:
+                return UpdateVariablesCommand.model_validate(data)
 
             # For other command types, use base class
             return GraphEngineCommand.model_validate(data)

api/core/workflow/graph_engine/command_processing/__init__.py

@@ -5,11 +5,12 @@ This package handles external commands sent to the engine
 during execution.
 """
 
-from .command_handlers import AbortCommandHandler, PauseCommandHandler
+from .command_handlers import AbortCommandHandler, PauseCommandHandler, UpdateVariablesCommandHandler
 from .command_processor import CommandProcessor
 
 __all__ = [
     "AbortCommandHandler",
     "CommandProcessor",
     "PauseCommandHandler",
+    "UpdateVariablesCommandHandler",
 ]

api/core/workflow/graph_engine/command_processing/command_handlers.py

@@ -4,9 +4,10 @@ from typing import final
 from typing_extensions import override
 
 from core.workflow.entities.pause_reason import SchedulingPause
+from core.workflow.runtime import VariablePool
 
 from ..domain.graph_execution import GraphExecution
-from ..entities.commands import AbortCommand, GraphEngineCommand, PauseCommand
+from ..entities.commands import AbortCommand, GraphEngineCommand, PauseCommand, UpdateVariablesCommand
 from .command_processor import CommandHandler
 
 logger = logging.getLogger(__name__)
@@ -31,3 +32,25 @@ class PauseCommandHandler(CommandHandler):
         reason = command.reason
         pause_reason = SchedulingPause(message=reason)
         execution.pause(pause_reason)
+
+
+@final
+class UpdateVariablesCommandHandler(CommandHandler):
+    def __init__(self, variable_pool: VariablePool) -> None:
+        self._variable_pool = variable_pool
+
+    @override
+    def handle(self, command: GraphEngineCommand, execution: GraphExecution) -> None:
+        assert isinstance(command, UpdateVariablesCommand)
+        for update in command.updates:
+            try:
+                variable = update.value
+                self._variable_pool.add(variable.selector, variable)
+                logger.debug("Updated variable %s for workflow %s", variable.selector, execution.workflow_id)
+            except ValueError as exc:
+                logger.warning(
+                    "Skipping invalid variable selector %s for workflow %s: %s",
+                    getattr(update.value, "selector", None),
+                    execution.workflow_id,
+                    exc,
+                )