Compare commits

...
9 changed files with 270 additions and 28 deletions
+2 -2
View File
@@ -137,7 +137,7 @@ class CompletionConversationApi(Resource):
.join( # type: ignore
MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
)
.distinct()
.group_by(Conversation.id)
)
elif args.annotation_status == "not_annotated":
query = (
@@ -275,7 +275,7 @@ class ChatConversationApi(Resource):
.join( # type: ignore
MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
)
.distinct()
.group_by(Conversation.id)
)
case "not_annotated":
query = (
@@ -3,7 +3,7 @@ import uuid
from flask import request
from flask_restx import Resource, marshal
from pydantic import BaseModel, Field
from sqlalchemy import String, cast, func, or_, select
from sqlalchemy import String, case, cast, func, literal, or_, select
from sqlalchemy.dialects.postgresql import JSONB
from werkzeug.exceptions import Forbidden, NotFound
@@ -159,9 +159,17 @@ class DatasetDocumentSegmentListApi(Resource):
# Use database-specific methods for JSON array search
if dify_config.SQLALCHEMY_DATABASE_URI_SCHEME == "postgresql":
# PostgreSQL: Use jsonb_array_elements_text to properly handle Unicode/Chinese text
# Feed the set-returning function a JSON array in every row. Filtering in
# the subquery is not enough because PostgreSQL can still evaluate the
# SRF on scalar JSON before applying the predicate.
keywords_jsonb = cast(DocumentSegment.keywords, JSONB)
keywords_array = case(
(func.jsonb_typeof(keywords_jsonb) == "array", keywords_jsonb),
else_=cast(literal("[]"), JSONB),
)
keywords_condition = func.array_to_string(
func.array(
select(func.jsonb_array_elements_text(cast(DocumentSegment.keywords, JSONB)))
select(func.jsonb_array_elements_text(keywords_array))
.correlate(DocumentSegment)
.scalar_subquery()
),
+18
View File
@@ -3,6 +3,7 @@ import json
import logging
from collections.abc import Callable, Generator
from typing import Any, cast
from urllib.parse import unquote
import httpx
from pydantic import BaseModel
@@ -53,6 +54,9 @@ else:
logger = logging.getLogger(__name__)
PLUGIN_DAEMON_MAX_PATH_LENGTH = 4096
PLUGIN_DAEMON_MAX_PATH_DECODE_DEPTH = 8
_httpx_client: httpx.Client = get_pooled_http_client(
"plugin_daemon",
lambda: httpx.Client(limits=httpx.Limits(max_keepalive_connections=50, max_connections=100), trust_env=False),
@@ -103,6 +107,20 @@ class BasePluginClient:
params: dict[str, Any] | None,
files: dict[str, Any] | None,
) -> tuple[str, dict[str, str], bytes | dict[str, Any] | str | None, dict[str, Any] | None, dict[str, Any] | None]:
if len(path) > PLUGIN_DAEMON_MAX_PATH_LENGTH:
raise ValueError(f"Invalid plugin daemon path: path length exceeds {PLUGIN_DAEMON_MAX_PATH_LENGTH}")
decoded_path = path
for _ in range(PLUGIN_DAEMON_MAX_PATH_DECODE_DEPTH):
next_decoded_path = unquote(decoded_path)
if next_decoded_path == decoded_path:
break
decoded_path = next_decoded_path
else:
raise ValueError("Invalid plugin daemon path: path is too deeply encoded")
if any(seg == ".." for seg in decoded_path.split("/")):
raise ValueError(f"Invalid plugin daemon path: traversal sequence detected in {path!r}")
url = plugin_daemon_inner_api_baseurl / path
prepared_headers = dict(headers or {})
prepared_headers["X-Api-Key"] = dify_config.PLUGIN_DAEMON_KEY
+37 -3
View File
@@ -110,6 +110,8 @@ class TokenPair(BaseModel):
REFRESH_TOKEN_PREFIX = "refresh_token:"
ACCOUNT_REFRESH_TOKEN_PREFIX = "account_refresh_token:"
REFRESH_TOKEN_EXPIRY = timedelta(days=dify_config.REFRESH_TOKEN_EXPIRE_DAYS)
ACCOUNT_LAST_ACTIVE_REFRESH_PREFIX = "account_last_active_refresh:"
ACCOUNT_LAST_ACTIVE_REFRESH_INTERVAL = timedelta(minutes=10)
class AccountService:
@@ -146,6 +148,40 @@ class AccountService:
def _get_account_refresh_token_key(account_id: str) -> str:
return f"{ACCOUNT_REFRESH_TOKEN_PREFIX}{account_id}"
@staticmethod
def _get_account_last_active_refresh_key(account_id: str) -> str:
return f"{ACCOUNT_LAST_ACTIVE_REFRESH_PREFIX}{account_id}"
@staticmethod
@redis_fallback(default_return=True)
def _should_refresh_account_last_active(account_id: str) -> bool:
return bool(
redis_client.set(
AccountService._get_account_last_active_refresh_key(account_id),
1,
ex=int(ACCOUNT_LAST_ACTIVE_REFRESH_INTERVAL.total_seconds()),
nx=True,
)
)
@staticmethod
def _refresh_account_last_active(account: Account) -> None:
now = naive_utc_now()
refresh_before = now - ACCOUNT_LAST_ACTIVE_REFRESH_INTERVAL
if account.last_active_at >= refresh_before:
return
if not AccountService._should_refresh_account_last_active(account.id):
return
db.session.execute(
update(Account)
.where(Account.id == account.id, Account.last_active_at < refresh_before)
.values(last_active_at=now, updated_at=func.current_timestamp())
)
db.session.commit()
@staticmethod
def _store_refresh_token(refresh_token: str, account_id: str):
redis_client.setex(AccountService._get_refresh_token_key(refresh_token), REFRESH_TOKEN_EXPIRY, account_id)
@@ -188,9 +224,7 @@ class AccountService:
available_ta.current = True
db.session.commit()
if naive_utc_now() - account.last_active_at > timedelta(minutes=10):
account.last_active_at = naive_utc_now()
db.session.commit()
AccountService._refresh_account_last_active(account)
# NOTE: make sure account is accessible outside of a db session
# This ensures that it will work correctly after upgrading to Flask version 3.1.2
db.session.refresh(account)
+30
View File
@@ -170,6 +170,16 @@ class _AutomaticProcessRule(BaseModel):
mode: Literal[ProcessRuleMode.AUTOMATIC]
summary_index_setting: _SummaryIndexSetting | None = None
@field_validator("summary_index_setting", mode="before")
@classmethod
def _normalize_summary_index_setting(cls, v: Any) -> Any:
"""Treat dicts with enable=None (or missing enable) as None (#36602)."""
if v is None:
return None
if isinstance(v, dict) and v.get("enable") is None:
return None
return v
class _CustomProcessRule(BaseModel):
model_config = ConfigDict(extra="allow")
@@ -178,6 +188,16 @@ class _CustomProcessRule(BaseModel):
rules: _EstimateRules
summary_index_setting: _SummaryIndexSetting | None = None
@field_validator("summary_index_setting", mode="before")
@classmethod
def _normalize_summary_index_setting(cls, v: Any) -> Any:
"""Treat dicts with enable=None (or missing enable) as None (#36602)."""
if v is None:
return None
if isinstance(v, dict) and v.get("enable") is None:
return None
return v
class _HierarchicalProcessRule(BaseModel):
model_config = ConfigDict(extra="allow")
@@ -186,6 +206,16 @@ class _HierarchicalProcessRule(BaseModel):
rules: _EstimateRules
summary_index_setting: _SummaryIndexSetting | None = None
@field_validator("summary_index_setting", mode="before")
@classmethod
def _normalize_summary_index_setting(cls, v: Any) -> Any:
"""Treat dicts with enable=None (or missing enable) as None (#36602)."""
if v is None:
return None
if isinstance(v, dict) and v.get("enable") is None:
return None
return v
_EstimateProcessRule = Annotated[
_AutomaticProcessRule | _CustomProcessRule | _HierarchicalProcessRule,
@@ -1036,6 +1036,48 @@ class TestSegmentListAdvancedCases:
assert status == 200
assert response["total"] == 1
def test_segment_list_postgres_keyword_filter_handles_scalar_keywords(self, app: Flask):
api = DatasetDocumentSegmentListApi()
method = unwrap(api.get)
dataset = MagicMock()
document = MagicMock()
pagination = MagicMock(items=[], total=0, pages=0)
with (
app.test_request_context("/?keyword=test"),
patch(
"controllers.console.datasets.datasets_segments.current_account_with_tenant",
return_value=(MagicMock(), "11111111-1111-1111-1111-111111111111"),
),
patch(
"controllers.console.datasets.datasets_segments.DatasetService.get_dataset",
return_value=dataset,
),
patch(
"controllers.console.datasets.datasets_segments.DatasetService.check_dataset_permission",
return_value=None,
),
patch(
"controllers.console.datasets.datasets_segments.DocumentService.get_document",
return_value=document,
),
patch(
"controllers.console.datasets.datasets_segments.dify_config",
SimpleNamespace(SQLALCHEMY_DATABASE_URI_SCHEME="postgresql"),
),
patch(
"controllers.console.datasets.datasets_segments.db.paginate",
return_value=pagination,
) as paginate_mock,
):
method(api, "22222222-2222-2222-2222-222222222222", "33333333-3333-3333-3333-333333333333")
query = paginate_mock.call_args.kwargs["select"]
sql = str(query.compile(compile_kwargs={"literal_binds": True}))
assert "jsonb_array_elements_text(CASE" in sql
assert "ELSE CAST('[]' AS JSONB)" in sql
def test_segment_list_permission_denied(self, app: Flask):
"""Test segment list with permission denied"""
api = DatasetDocumentSegmentListApi()
@@ -1,11 +1,12 @@
import json
from urllib.parse import quote
import pytest
from pytest_mock import MockerFixture
from core.plugin.endpoint.exc import EndpointSetupFailedError
from core.plugin.entities.plugin_daemon import PluginDaemonInnerError
from core.plugin.impl.base import BasePluginClient
from core.plugin.impl.base import PLUGIN_DAEMON_MAX_PATH_LENGTH, BasePluginClient
from core.trigger.errors import (
EventIgnoreError,
TriggerInvokeError,
@@ -67,6 +68,36 @@ class TestBasePluginClientImpl:
assert result == ["hello", "world"]
assert stream_mock.call_args.kwargs["data"] == {"k": "v"}
@pytest.mark.parametrize(
"path",
[
"plugin/tenant/%252e%252e%252ftarget",
"plugin/tenant/%2e%2e%252ftarget",
],
)
def test_prepare_request_rejects_encoded_traversal_with_encoded_separator(self, path: str):
client = BasePluginClient()
with pytest.raises(ValueError, match="traversal sequence detected"):
client._prepare_request(path, None, None, None, None)
def test_prepare_request_rejects_path_exceeding_max_length(self):
client = BasePluginClient()
path = "a" * (PLUGIN_DAEMON_MAX_PATH_LENGTH + 1)
with pytest.raises(ValueError, match="path length exceeds"):
client._prepare_request(path, None, None, None, None)
def test_prepare_request_rejects_excessively_encoded_path(self):
client = BasePluginClient()
segment = "..%2Ftarget"
for _ in range(9):
segment = quote(segment, safe="")
path = f"plugin/tenant/{segment}"
with pytest.raises(ValueError, match="too deeply encoded"):
client._prepare_request(path, None, None, None, None)
def test_request_with_plugin_daemon_response_handles_request_exception(self, mocker: MockerFixture):
client = BasePluginClient()
mocker.patch.object(client, "_request", side_effect=RuntimeError("boom"))
@@ -436,7 +436,10 @@ class TestAccountService:
mock_db_dependencies["db"].session.scalar.return_value = mock_tenant_join
# Mock datetime
with patch("services.account_service.datetime") as mock_datetime:
with (
patch("services.account_service.datetime") as mock_datetime,
patch.object(AccountService, "_refresh_account_last_active") as mock_refresh_last_active,
):
mock_now = datetime.now()
mock_datetime.now.return_value = mock_now
mock_datetime.UTC = "UTC"
@@ -447,6 +450,7 @@ class TestAccountService:
# Verify results
assert result == mock_account
assert mock_account.set_tenant_id.called
mock_refresh_last_active.assert_called_once_with(mock_account)
def test_load_user_not_found(self, mock_db_dependencies):
"""Test user loading when user does not exist."""
@@ -483,7 +487,10 @@ class TestAccountService:
mock_db_dependencies["db"].session.scalar.side_effect = [None, mock_available_tenant]
# Mock datetime
with patch("services.account_service.datetime") as mock_datetime:
with (
patch("services.account_service.datetime") as mock_datetime,
patch.object(AccountService, "_refresh_account_last_active") as mock_refresh_last_active,
):
mock_now = datetime.now()
mock_datetime.now.return_value = mock_now
mock_datetime.UTC = "UTC"
@@ -495,6 +502,7 @@ class TestAccountService:
assert result == mock_account
assert mock_available_tenant.current is True
self._assert_database_operations_called(mock_db_dependencies["db"])
mock_refresh_last_active.assert_called_once_with(mock_account)
def test_load_user_no_tenants(self, mock_db_dependencies):
"""Test user loading when user has no tenants at all."""
@@ -517,6 +525,68 @@ class TestAccountService:
# Verify results
assert result is None
def test_refresh_account_last_active_uses_redis_gate_and_conditional_update(self, mock_db_dependencies):
"""Test last-active refresh is gated in Redis and conditionally written to DB."""
mock_account = TestAccountAssociatedDataFactory.create_account_mock()
now = datetime(2026, 6, 2, 2, 45, 49)
mock_account.last_active_at = now - timedelta(minutes=15)
with (
patch("services.account_service.naive_utc_now", return_value=now),
patch("services.account_service.redis_client") as mock_redis_client,
):
mock_redis_client.set.return_value = True
AccountService._refresh_account_last_active(mock_account)
mock_redis_client.set.assert_called_once_with(
"account_last_active_refresh:user-123",
1,
ex=600,
nx=True,
)
mock_db_dependencies["db"].session.execute.assert_called_once()
mock_db_dependencies["db"].session.commit.assert_called_once()
def test_refresh_account_last_active_skips_db_when_redis_gate_exists(self, mock_db_dependencies):
"""Test concurrent refresh attempts do not enqueue duplicate DB updates."""
mock_account = TestAccountAssociatedDataFactory.create_account_mock()
now = datetime(2026, 6, 2, 2, 45, 49)
mock_account.last_active_at = now - timedelta(minutes=15)
with (
patch("services.account_service.naive_utc_now", return_value=now),
patch("services.account_service.redis_client") as mock_redis_client,
):
mock_redis_client.set.return_value = None
AccountService._refresh_account_last_active(mock_account)
mock_redis_client.set.assert_called_once_with(
"account_last_active_refresh:user-123",
1,
ex=600,
nx=True,
)
mock_db_dependencies["db"].session.execute.assert_not_called()
mock_db_dependencies["db"].session.commit.assert_not_called()
def test_refresh_account_last_active_skips_recent_account(self, mock_db_dependencies):
"""Test recent activity does not touch Redis or DB."""
mock_account = TestAccountAssociatedDataFactory.create_account_mock()
now = datetime(2026, 6, 2, 2, 45, 49)
mock_account.last_active_at = now - timedelta(minutes=5)
with (
patch("services.account_service.naive_utc_now", return_value=now),
patch("services.account_service.redis_client") as mock_redis_client,
):
AccountService._refresh_account_last_active(mock_account)
mock_redis_client.set.assert_not_called()
mock_db_dependencies["db"].session.execute.assert_not_called()
mock_db_dependencies["db"].session.commit.assert_not_called()
class TestTenantService:
"""
@@ -1,7 +1,7 @@
'use client'
import type { ButtonProps } from '@langgenius/dify-ui/button'
import type { FormInputItem, UserAction } from '@/app/components/workflow/nodes/human-input/types'
import type { SiteInfo } from '@/models/share'
import type { CustomConfigValueType, SiteInfo } from '@/models/share'
import type { HumanInputFormError } from '@/service/use-share'
import { Button } from '@langgenius/dify-ui/button'
import { cn } from '@langgenius/dify-ui/cn'
@@ -25,7 +25,10 @@ import { useParams } from '@/next/navigation'
import { useGetHumanInputForm, useSubmitHumanInputForm } from '@/service/use-share'
export type FormData = {
site: { site: SiteInfo }
site: {
site: SiteInfo
custom_config?: Record<string, CustomConfigValueType> | null
}
form_content: string
inputs: FormInputItem[]
resolved_default_values: Record<string, string>
@@ -46,6 +49,8 @@ const FormContent = () => {
const { data: formData, isLoading, error } = useGetHumanInputForm(token)
const removeWebappBrand = formData?.site?.custom_config?.remove_webapp_brand === true
const expired = (error as HumanInputFormError | null)?.code === 'human_input_form_expired'
const submitted = (error as HumanInputFormError | null)?.code === 'human_input_form_submitted'
const rateLimitExceeded = (error as HumanInputFormError | null)?.code === 'web_form_rate_limit_exceeded'
@@ -111,15 +116,17 @@ const FormContent = () => {
</div>
<div className="shrink-0 system-2xs-regular-uppercase text-text-tertiary">{t('humanInput.submissionID', { id: token, ns: 'share' })}</div>
</div>
<div className="flex flex-row-reverse px-2 py-3">
<div className={cn(
'flex shrink-0 items-center gap-1.5 px-1',
)}
>
<div className="system-2xs-medium-uppercase text-text-tertiary">{t('chat.poweredBy', { ns: 'share' })}</div>
<DifyLogo size="small" />
{!removeWebappBrand && (
<div className="flex flex-row-reverse px-2 py-3">
<div className={cn(
'flex shrink-0 items-center gap-1.5 px-1',
)}
>
<div className="system-2xs-medium-uppercase text-text-tertiary">{t('chat.poweredBy', { ns: 'share' })}</div>
<DifyLogo size="small" />
</div>
</div>
</div>
)}
</div>
</div>
)
@@ -272,15 +279,17 @@ const FormContent = () => {
</div>
<ExpirationTime expirationTime={formData.expiration_time * 1000} />
</div>
<div className="flex flex-row-reverse px-2 py-3">
<div className={cn(
'flex shrink-0 items-center gap-1.5 px-1',
)}
>
<div className="system-2xs-medium-uppercase text-text-tertiary">{t('chat.poweredBy', { ns: 'share' })}</div>
<DifyLogo size="small" />
{!removeWebappBrand && (
<div className="flex flex-row-reverse px-2 py-3">
<div className={cn(
'flex shrink-0 items-center gap-1.5 px-1',
)}
>
<div className="system-2xs-medium-uppercase text-text-tertiary">{t('chat.poweredBy', { ns: 'share' })}</div>
<DifyLogo size="small" />
</div>
</div>
</div>
)}
</div>
</div>
)