Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
acd6942d21 |
@@ -116,6 +116,14 @@ S3_ACCESS_KEY=your-access-key
|
||||
S3_SECRET_KEY=your-secret-key
|
||||
S3_REGION=your-region
|
||||
S3_ADDRESS_STYLE=auto
|
||||
# Optional public base URL for objects in the bucket. When set, signed file
|
||||
# previews are served by 302-redirecting to "<base>/<object-key>" so that bytes
|
||||
# are delivered directly by the object store / CDN. Examples:
|
||||
# Cloudflare R2 custom domain: https://cdn.example.com
|
||||
# MinIO public endpoint: https://minio.example.com/your-bucket
|
||||
# Aliyun OSS public domain: https://your-bucket.oss-cn-hangzhou.aliyuncs.com
|
||||
# Leave empty to keep the default API-streamed behavior.
|
||||
S3_PUBLIC_BASE_URL=
|
||||
|
||||
# Workflow run and Conversation archive storage (S3-compatible)
|
||||
ARCHIVE_STORAGE_ENABLED=false
|
||||
|
||||
@@ -43,3 +43,16 @@ class S3StorageConfig(BaseSettings):
|
||||
description="Use AWS managed IAM roles for authentication instead of access/secret keys",
|
||||
default=False,
|
||||
)
|
||||
|
||||
S3_PUBLIC_BASE_URL: str | None = Field(
|
||||
description=(
|
||||
"Optional public base URL for objects in the bucket "
|
||||
"(e.g., a Cloudflare R2 custom domain, MinIO public endpoint, or "
|
||||
"OSS public domain). When set, signed file previews are served via "
|
||||
"302 redirect to '<base>/<object-key>' so that bytes are delivered "
|
||||
"directly by the object store / CDN instead of proxied by Dify's API. "
|
||||
"Trailing slashes are ignored. Leave empty to keep the default "
|
||||
"API-streamed behavior."
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from urllib.parse import quote
|
||||
|
||||
from flask import Response, request
|
||||
from flask import Response, redirect, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import NotFound
|
||||
@@ -64,7 +64,7 @@ class ImagePreviewApi(Resource):
|
||||
sign = args.sign
|
||||
|
||||
try:
|
||||
generator, mimetype = FileService(db.engine).get_image_preview(
|
||||
public_url, generator, mimetype = FileService(db.engine).get_image_preview(
|
||||
file_id=file_id,
|
||||
timestamp=timestamp,
|
||||
nonce=nonce,
|
||||
@@ -73,6 +73,9 @@ class ImagePreviewApi(Resource):
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
if public_url:
|
||||
return redirect(public_url, code=302)
|
||||
|
||||
return Response(generator, mimetype=mimetype)
|
||||
|
||||
|
||||
@@ -103,7 +106,7 @@ class FilePreviewApi(Resource):
|
||||
args = FilePreviewQuery.model_validate(request.args.to_dict(flat=True)) # type: ignore
|
||||
|
||||
try:
|
||||
generator, upload_file = FileService(db.engine).get_file_generator_by_file_id(
|
||||
public_url, generator, upload_file = FileService(db.engine).get_file_generator_by_file_id(
|
||||
file_id=file_id,
|
||||
timestamp=args.timestamp,
|
||||
nonce=args.nonce,
|
||||
@@ -112,6 +115,9 @@ class FilePreviewApi(Resource):
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
if public_url:
|
||||
return redirect(public_url, code=302)
|
||||
|
||||
response = Response(
|
||||
generator,
|
||||
mimetype=upload_file.mime_type,
|
||||
@@ -175,10 +181,13 @@ class WorkspaceWebappLogoApi(Resource):
|
||||
raise NotFound("webapp logo is not found")
|
||||
|
||||
try:
|
||||
generator, mimetype = FileService(db.engine).get_public_image_preview(
|
||||
public_url, generator, mimetype = FileService(db.engine).get_public_image_preview(
|
||||
webapp_logo_file_id,
|
||||
)
|
||||
except services.errors.file.UnsupportedFileTypeError:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
if public_url:
|
||||
return redirect(public_url, code=302)
|
||||
|
||||
return Response(generator, mimetype=mimetype)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from urllib.parse import quote
|
||||
|
||||
from flask import Response, request
|
||||
from flask import Response, redirect, request
|
||||
from flask_restx import Resource
|
||||
from pydantic import BaseModel, Field
|
||||
from werkzeug.exceptions import Forbidden, NotFound
|
||||
@@ -57,6 +57,10 @@ class ToolFileApi(Resource):
|
||||
|
||||
try:
|
||||
tool_file_manager = ToolFileManager()
|
||||
public_url, tool_file = tool_file_manager.get_public_url_and_file_by_tool_file_id(file_id)
|
||||
if public_url and tool_file:
|
||||
return redirect(public_url, code=302)
|
||||
|
||||
stream, tool_file = tool_file_manager.get_file_generator_by_tool_file_id(
|
||||
file_id,
|
||||
)
|
||||
|
||||
@@ -225,6 +225,23 @@ class ToolFileManager:
|
||||
|
||||
return stream, self._build_graph_file_reference(tool_file)
|
||||
|
||||
def get_public_url_and_file_by_tool_file_id(self, tool_file_id: str) -> tuple[str | None, File | None]:
|
||||
"""
|
||||
Resolve a tool file to a public URL when the storage backend exposes one.
|
||||
|
||||
Returns (public_url, file_reference). If the backend has no public URL
|
||||
configured, returns (None, file_reference) and callers should fall back
|
||||
to the streaming path.
|
||||
"""
|
||||
with session_factory.create_session() as session:
|
||||
tool_file: ToolFile | None = session.scalar(select(ToolFile).where(ToolFile.id == tool_file_id).limit(1))
|
||||
|
||||
if not tool_file:
|
||||
return None, None
|
||||
|
||||
public_url = storage.get_public_url(tool_file.file_key)
|
||||
return public_url, self._build_graph_file_reference(tool_file)
|
||||
|
||||
|
||||
# init tool_file_parser
|
||||
from graphon.file.tool_file_parser import set_tool_file_manager_factory
|
||||
|
||||
@@ -119,6 +119,9 @@ class Storage:
|
||||
def delete(self, filename: str):
|
||||
return self.storage_runner.delete(filename)
|
||||
|
||||
def get_public_url(self, filename: str) -> str | None:
|
||||
return self.storage_runner.get_public_url(filename)
|
||||
|
||||
def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
|
||||
return self.storage_runner.scan(path, files=files, directories=directories)
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
from collections.abc import Generator
|
||||
from urllib.parse import quote
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
@@ -17,6 +18,8 @@ class AwsS3Storage(BaseStorage):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bucket_name = dify_config.S3_BUCKET_NAME
|
||||
public_base_url = dify_config.S3_PUBLIC_BASE_URL
|
||||
self.public_base_url = public_base_url.rstrip("/") if public_base_url else None
|
||||
if dify_config.S3_USE_AWS_MANAGED_IAM:
|
||||
logger.info("Using AWS managed IAM role for S3")
|
||||
|
||||
@@ -85,3 +88,8 @@ class AwsS3Storage(BaseStorage):
|
||||
|
||||
def delete(self, filename: str):
|
||||
self.client.delete_object(Bucket=self.bucket_name, Key=filename)
|
||||
|
||||
def get_public_url(self, filename: str) -> str | None:
|
||||
if not self.public_base_url:
|
||||
return None
|
||||
return f"{self.public_base_url}/{quote(filename, safe='/')}"
|
||||
|
||||
@@ -31,6 +31,17 @@ class BaseStorage(ABC):
|
||||
def delete(self, filename: str):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_public_url(self, filename: str) -> str | None:
|
||||
"""
|
||||
Return a publicly accessible URL for the given object, or None if the
|
||||
backend is not configured to serve content publicly.
|
||||
|
||||
When set, file controllers will 302-redirect signed preview requests to
|
||||
this URL after verifying the signature, so that the bytes themselves are
|
||||
served by the object store / CDN instead of streamed through Dify's API.
|
||||
"""
|
||||
return None
|
||||
|
||||
def scan(self, path, files=True, directories=False) -> list[str]:
|
||||
"""
|
||||
Scan files and directories in the given path.
|
||||
|
||||
@@ -210,9 +210,12 @@ class FileService:
|
||||
if extension.lower() not in IMAGE_EXTENSIONS:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
generator = storage.load(upload_file.key, stream=True)
|
||||
public_url = storage.get_public_url(upload_file.key)
|
||||
if public_url:
|
||||
return public_url, None, upload_file.mime_type
|
||||
|
||||
return generator, upload_file.mime_type
|
||||
generator = storage.load(upload_file.key, stream=True)
|
||||
return None, generator, upload_file.mime_type
|
||||
|
||||
def get_file_generator_by_file_id(self, file_id: str, timestamp: str, nonce: str, sign: str):
|
||||
result = file_helpers.verify_file_signature(upload_file_id=file_id, timestamp=timestamp, nonce=nonce, sign=sign)
|
||||
@@ -225,9 +228,12 @@ class FileService:
|
||||
if not upload_file:
|
||||
raise NotFound("File not found or signature is invalid")
|
||||
|
||||
generator = storage.load(upload_file.key, stream=True)
|
||||
public_url = storage.get_public_url(upload_file.key)
|
||||
if public_url:
|
||||
return public_url, None, upload_file
|
||||
|
||||
return generator, upload_file
|
||||
generator = storage.load(upload_file.key, stream=True)
|
||||
return None, generator, upload_file
|
||||
|
||||
def get_public_image_preview(self, file_id: str):
|
||||
with self._session_maker(expire_on_commit=False) as session:
|
||||
@@ -241,9 +247,12 @@ class FileService:
|
||||
if extension.lower() not in IMAGE_EXTENSIONS:
|
||||
raise UnsupportedFileTypeError()
|
||||
|
||||
generator = storage.load(upload_file.key)
|
||||
public_url = storage.get_public_url(upload_file.key)
|
||||
if public_url:
|
||||
return public_url, None, upload_file.mime_type
|
||||
|
||||
return generator, upload_file.mime_type
|
||||
generator = storage.load(upload_file.key)
|
||||
return None, generator, upload_file.mime_type
|
||||
|
||||
def get_file_content(self, file_id: str) -> str:
|
||||
with self._session_maker(expire_on_commit=False) as session:
|
||||
|
||||
@@ -49,6 +49,7 @@ class TestImagePreviewApi:
|
||||
|
||||
generator = iter([b"img"])
|
||||
mock_file_service.return_value.get_image_preview.return_value = (
|
||||
None,
|
||||
generator,
|
||||
"image/png",
|
||||
)
|
||||
@@ -60,6 +61,30 @@ class TestImagePreviewApi:
|
||||
|
||||
assert response.mimetype == "image/png"
|
||||
|
||||
@patch.object(module, "FileService")
|
||||
def test_redirects_to_public_url(self, mock_file_service):
|
||||
module.request = fake_request(
|
||||
{
|
||||
"timestamp": "123",
|
||||
"nonce": "abc",
|
||||
"sign": "sig",
|
||||
}
|
||||
)
|
||||
|
||||
mock_file_service.return_value.get_image_preview.return_value = (
|
||||
"https://cdn.example.com/upload_files/tenant/abc.png",
|
||||
None,
|
||||
"image/png",
|
||||
)
|
||||
|
||||
api = module.ImagePreviewApi()
|
||||
get_fn = unwrap(api.get)
|
||||
|
||||
response = get_fn("file-id")
|
||||
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"] == "https://cdn.example.com/upload_files/tenant/abc.png"
|
||||
|
||||
@patch.object(module, "FileService")
|
||||
def test_unsupported_file_type(self, mock_file_service):
|
||||
module.request = fake_request(
|
||||
@@ -98,6 +123,7 @@ class TestFilePreviewApi:
|
||||
upload_file = DummyUploadFile(size=100)
|
||||
|
||||
mock_file_service.return_value.get_file_generator_by_file_id.return_value = (
|
||||
None,
|
||||
generator,
|
||||
upload_file,
|
||||
)
|
||||
@@ -112,6 +138,32 @@ class TestFilePreviewApi:
|
||||
assert "Accept-Ranges" not in response.headers
|
||||
mock_enforce.assert_called_once()
|
||||
|
||||
@patch.object(module, "FileService")
|
||||
def test_redirects_to_public_url(self, mock_file_service):
|
||||
module.request = fake_request(
|
||||
{
|
||||
"timestamp": "123",
|
||||
"nonce": "abc",
|
||||
"sign": "sig",
|
||||
"as_attachment": False,
|
||||
}
|
||||
)
|
||||
|
||||
upload_file = DummyUploadFile(size=100)
|
||||
mock_file_service.return_value.get_file_generator_by_file_id.return_value = (
|
||||
"https://cdn.example.com/upload_files/tenant/abc.bin",
|
||||
None,
|
||||
upload_file,
|
||||
)
|
||||
|
||||
api = module.FilePreviewApi()
|
||||
get_fn = unwrap(api.get)
|
||||
|
||||
response = get_fn("file-id")
|
||||
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"] == "https://cdn.example.com/upload_files/tenant/abc.bin"
|
||||
|
||||
@patch.object(module, "enforce_download_for_html")
|
||||
@patch.object(module, "FileService")
|
||||
def test_as_attachment(self, mock_file_service, mock_enforce):
|
||||
@@ -132,6 +184,7 @@ class TestFilePreviewApi:
|
||||
)
|
||||
|
||||
mock_file_service.return_value.get_file_generator_by_file_id.return_value = (
|
||||
None,
|
||||
generator,
|
||||
upload_file,
|
||||
)
|
||||
@@ -175,6 +228,7 @@ class TestWorkspaceWebappLogoApi:
|
||||
generator = iter([b"logo"])
|
||||
|
||||
mock_file_service.return_value.get_public_image_preview.return_value = (
|
||||
None,
|
||||
generator,
|
||||
"image/png",
|
||||
)
|
||||
@@ -186,6 +240,24 @@ class TestWorkspaceWebappLogoApi:
|
||||
|
||||
assert response.mimetype == "image/png"
|
||||
|
||||
@patch.object(module, "FileService")
|
||||
@patch.object(module.TenantService, "get_custom_config")
|
||||
def test_redirects_to_public_url(self, mock_config, mock_file_service):
|
||||
mock_config.return_value = {"replace_webapp_logo": "logo-id"}
|
||||
mock_file_service.return_value.get_public_image_preview.return_value = (
|
||||
"https://cdn.example.com/upload_files/tenant/logo.png",
|
||||
None,
|
||||
"image/png",
|
||||
)
|
||||
|
||||
api = module.WorkspaceWebappLogoApi()
|
||||
get_fn = unwrap(api.get)
|
||||
|
||||
response = get_fn("workspace-id")
|
||||
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"] == "https://cdn.example.com/upload_files/tenant/logo.png"
|
||||
|
||||
@patch.object(module.TenantService, "get_custom_config")
|
||||
def test_logo_not_configured(self, mock_config):
|
||||
mock_config.return_value = {}
|
||||
|
||||
@@ -50,6 +50,10 @@ class TestToolFileApi:
|
||||
stream = iter([b"data"])
|
||||
tool_file = DummyToolFile(size=100)
|
||||
|
||||
mock_tool_file_manager.return_value.get_public_url_and_file_by_tool_file_id.return_value = (
|
||||
None,
|
||||
tool_file,
|
||||
)
|
||||
mock_tool_file_manager.return_value.get_file_generator_by_tool_file_id.return_value = (
|
||||
stream,
|
||||
tool_file,
|
||||
@@ -69,6 +73,37 @@ class TestToolFileApi:
|
||||
sign="sig",
|
||||
)
|
||||
|
||||
@patch.object(module, "verify_tool_file_signature", return_value=True)
|
||||
@patch.object(module, "ToolFileManager")
|
||||
def test_redirects_to_public_url(
|
||||
self,
|
||||
mock_tool_file_manager,
|
||||
mock_verify,
|
||||
):
|
||||
module.request = fake_request(
|
||||
{
|
||||
"timestamp": "123",
|
||||
"nonce": "abc",
|
||||
"sign": "sig",
|
||||
"as_attachment": False,
|
||||
}
|
||||
)
|
||||
|
||||
tool_file = DummyToolFile(size=100)
|
||||
mock_tool_file_manager.return_value.get_public_url_and_file_by_tool_file_id.return_value = (
|
||||
"https://cdn.example.com/tool_files/abc.txt",
|
||||
tool_file,
|
||||
)
|
||||
|
||||
api = module.ToolFileApi()
|
||||
get_fn = unwrap(api.get)
|
||||
|
||||
response = get_fn("file-id", "txt")
|
||||
|
||||
assert response.status_code == 302
|
||||
assert response.headers["Location"] == "https://cdn.example.com/tool_files/abc.txt"
|
||||
mock_tool_file_manager.return_value.get_file_generator_by_tool_file_id.assert_not_called()
|
||||
|
||||
@patch.object(module, "verify_tool_file_signature", return_value=True)
|
||||
@patch.object(module, "ToolFileManager")
|
||||
def test_as_attachment(
|
||||
@@ -91,6 +126,10 @@ class TestToolFileApi:
|
||||
filename="doc.pdf",
|
||||
)
|
||||
|
||||
mock_tool_file_manager.return_value.get_public_url_and_file_by_tool_file_id.return_value = (
|
||||
None,
|
||||
tool_file,
|
||||
)
|
||||
mock_tool_file_manager.return_value.get_file_generator_by_tool_file_id.return_value = (
|
||||
stream,
|
||||
tool_file,
|
||||
@@ -137,6 +176,10 @@ class TestToolFileApi:
|
||||
}
|
||||
)
|
||||
|
||||
mock_tool_file_manager.return_value.get_public_url_and_file_by_tool_file_id.return_value = (
|
||||
None,
|
||||
None,
|
||||
)
|
||||
mock_tool_file_manager.return_value.get_file_generator_by_tool_file_id.return_value = (
|
||||
None,
|
||||
None,
|
||||
@@ -164,6 +207,10 @@ class TestToolFileApi:
|
||||
}
|
||||
)
|
||||
|
||||
mock_tool_file_manager.return_value.get_public_url_and_file_by_tool_file_id.return_value = (
|
||||
None,
|
||||
DummyToolFile(),
|
||||
)
|
||||
mock_tool_file_manager.return_value.get_file_generator_by_tool_file_id.side_effect = Exception("boom")
|
||||
|
||||
api = module.ToolFileApi()
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
"""Primarily used for testing merged cell scenarios"""
|
||||
|
||||
import gc
|
||||
import io
|
||||
import os
|
||||
import tempfile
|
||||
import warnings
|
||||
from collections import UserDict
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from docx import Document
|
||||
@@ -375,21 +377,23 @@ def test_close_is_idempotent():
|
||||
extractor.temp_file.close.assert_called_once()
|
||||
|
||||
|
||||
async def _async_close() -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_close_closes_awaitable_close_result():
|
||||
def test_close_handles_async_close_mock():
|
||||
extractor = object.__new__(WordExtractor)
|
||||
extractor._closed = False
|
||||
extractor.temp_file = MagicMock()
|
||||
close_result = _async_close()
|
||||
extractor.temp_file.close = MagicMock(return_value=close_result)
|
||||
extractor.temp_file.close = AsyncMock()
|
||||
|
||||
extractor.close()
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
extractor.close()
|
||||
gc.collect()
|
||||
|
||||
assert close_result.cr_frame is None
|
||||
extractor.temp_file.close.assert_called_once()
|
||||
assert not [
|
||||
warning
|
||||
for warning in caught
|
||||
if issubclass(warning.category, RuntimeWarning) and "AsyncMockMixin._execute_mock_call" in str(warning.message)
|
||||
]
|
||||
|
||||
|
||||
def test_extract_images_handles_invalid_external_cases(monkeypatch):
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from extensions.storage.aws_s3_storage import AwsS3Storage
|
||||
|
||||
|
||||
def _build_storage(public_base_url: str | None = None) -> AwsS3Storage:
|
||||
with patch("extensions.storage.aws_s3_storage.dify_config", autospec=True) as mock_config:
|
||||
mock_config.S3_BUCKET_NAME = "test-bucket"
|
||||
mock_config.S3_PUBLIC_BASE_URL = public_base_url
|
||||
mock_config.S3_USE_AWS_MANAGED_IAM = False
|
||||
mock_config.S3_ACCESS_KEY = "ak"
|
||||
mock_config.S3_SECRET_KEY = "sk"
|
||||
mock_config.S3_ENDPOINT = "https://example.com"
|
||||
mock_config.S3_REGION = "auto"
|
||||
mock_config.S3_ADDRESS_STYLE = "auto"
|
||||
|
||||
with patch("extensions.storage.aws_s3_storage.boto3") as mock_boto3:
|
||||
client = Mock()
|
||||
client.head_bucket.return_value = None
|
||||
mock_boto3.client.return_value = client
|
||||
mock_boto3.Session.return_value.client.return_value = client
|
||||
return AwsS3Storage()
|
||||
|
||||
|
||||
class TestAwsS3StoragePublicUrl:
|
||||
def test_returns_none_when_public_base_url_unset(self):
|
||||
storage = _build_storage(public_base_url=None)
|
||||
assert storage.get_public_url("upload_files/tenant/abc.png") is None
|
||||
|
||||
def test_returns_none_when_public_base_url_empty_string(self):
|
||||
storage = _build_storage(public_base_url="")
|
||||
assert storage.get_public_url("upload_files/tenant/abc.png") is None
|
||||
|
||||
def test_composes_url_when_configured(self):
|
||||
storage = _build_storage(public_base_url="https://cdn.example.com")
|
||||
assert (
|
||||
storage.get_public_url("upload_files/tenant/abc.png")
|
||||
== "https://cdn.example.com/upload_files/tenant/abc.png"
|
||||
)
|
||||
|
||||
def test_strips_trailing_slash(self):
|
||||
storage = _build_storage(public_base_url="https://cdn.example.com/")
|
||||
assert (
|
||||
storage.get_public_url("upload_files/tenant/abc.png")
|
||||
== "https://cdn.example.com/upload_files/tenant/abc.png"
|
||||
)
|
||||
|
||||
def test_preserves_path_separators_in_key(self):
|
||||
# Object key path separators must not be percent-encoded.
|
||||
storage = _build_storage(public_base_url="https://cdn.example.com")
|
||||
url = storage.get_public_url("a/b/c.txt")
|
||||
assert url == "https://cdn.example.com/a/b/c.txt"
|
||||
|
||||
def test_quotes_unsafe_characters_in_key(self):
|
||||
storage = _build_storage(public_base_url="https://cdn.example.com")
|
||||
url = storage.get_public_url("upload_files/has space.png")
|
||||
assert url == "https://cdn.example.com/upload_files/has%20space.png"
|
||||
|
||||
|
||||
class TestAwsS3StorageBucketCheck:
|
||||
def test_init_handles_403_on_head_bucket(self):
|
||||
# Regression: R2 / hardened buckets often return 403 on head_bucket; the
|
||||
# constructor must swallow the error instead of crashing.
|
||||
with patch("extensions.storage.aws_s3_storage.dify_config", autospec=True) as mock_config:
|
||||
mock_config.S3_BUCKET_NAME = "test-bucket"
|
||||
mock_config.S3_PUBLIC_BASE_URL = None
|
||||
mock_config.S3_USE_AWS_MANAGED_IAM = False
|
||||
mock_config.S3_ACCESS_KEY = "ak"
|
||||
mock_config.S3_SECRET_KEY = "sk"
|
||||
mock_config.S3_ENDPOINT = "https://example.com"
|
||||
mock_config.S3_REGION = "auto"
|
||||
mock_config.S3_ADDRESS_STYLE = "auto"
|
||||
|
||||
with patch("extensions.storage.aws_s3_storage.boto3") as mock_boto3:
|
||||
client = Mock()
|
||||
client.head_bucket.side_effect = ClientError(
|
||||
{"Error": {"Code": "403", "Message": "Forbidden"}}, "HeadBucket"
|
||||
)
|
||||
mock_boto3.client.return_value = client
|
||||
storage = AwsS3Storage()
|
||||
assert storage.bucket_name == "test-bucket"
|
||||
client.create_bucket.assert_not_called()
|
||||
@@ -253,15 +253,39 @@ class TestFileService:
|
||||
patch("services.file_service.storage") as mock_storage,
|
||||
):
|
||||
mock_verify.return_value = True
|
||||
mock_storage.get_public_url.return_value = None
|
||||
mock_storage.load.return_value = iter([b"chunk1"])
|
||||
|
||||
# Execute
|
||||
gen, mime = file_service.get_image_preview("file_id", "ts", "nonce", "sign")
|
||||
public_url, gen, mime = file_service.get_image_preview("file_id", "ts", "nonce", "sign")
|
||||
|
||||
# Assert
|
||||
assert public_url is None
|
||||
assert list(gen) == [b"chunk1"]
|
||||
assert mime == "image/jpeg"
|
||||
|
||||
def test_get_image_preview_redirects_when_storage_has_public_url(self, file_service, mock_db_session):
|
||||
upload_file = MagicMock(spec=UploadFile)
|
||||
upload_file.id = "file_id"
|
||||
upload_file.extension = "jpg"
|
||||
upload_file.mime_type = "image/jpeg"
|
||||
upload_file.key = "upload_files/tenant/abc.jpg"
|
||||
mock_db_session.scalar.return_value = upload_file
|
||||
|
||||
with (
|
||||
patch("services.file_service.file_helpers.verify_image_signature") as mock_verify,
|
||||
patch("services.file_service.storage") as mock_storage,
|
||||
):
|
||||
mock_verify.return_value = True
|
||||
mock_storage.get_public_url.return_value = "https://cdn.example.com/upload_files/tenant/abc.jpg"
|
||||
|
||||
public_url, gen, mime = file_service.get_image_preview("file_id", "ts", "nonce", "sign")
|
||||
|
||||
assert public_url == "https://cdn.example.com/upload_files/tenant/abc.jpg"
|
||||
assert gen is None
|
||||
assert mime == "image/jpeg"
|
||||
mock_storage.load.assert_not_called()
|
||||
|
||||
def test_get_image_preview_invalid_sig(self, file_service):
|
||||
with patch("services.file_service.file_helpers.verify_image_signature") as mock_verify:
|
||||
mock_verify.return_value = False
|
||||
@@ -296,12 +320,33 @@ class TestFileService:
|
||||
patch("services.file_service.storage") as mock_storage,
|
||||
):
|
||||
mock_verify.return_value = True
|
||||
mock_storage.get_public_url.return_value = None
|
||||
mock_storage.load.return_value = iter([b"chunk"])
|
||||
|
||||
gen, file = file_service.get_file_generator_by_file_id("file_id", "ts", "nonce", "sign")
|
||||
public_url, gen, file = file_service.get_file_generator_by_file_id("file_id", "ts", "nonce", "sign")
|
||||
assert public_url is None
|
||||
assert list(gen) == [b"chunk"]
|
||||
assert file == upload_file
|
||||
|
||||
def test_get_file_generator_by_file_id_redirects_when_storage_has_public_url(self, file_service, mock_db_session):
|
||||
upload_file = MagicMock(spec=UploadFile)
|
||||
upload_file.id = "file_id"
|
||||
upload_file.key = "upload_files/tenant/abc.bin"
|
||||
mock_db_session.scalar.return_value = upload_file
|
||||
|
||||
with (
|
||||
patch("services.file_service.file_helpers.verify_file_signature") as mock_verify,
|
||||
patch("services.file_service.storage") as mock_storage,
|
||||
):
|
||||
mock_verify.return_value = True
|
||||
mock_storage.get_public_url.return_value = "https://cdn.example.com/upload_files/tenant/abc.bin"
|
||||
|
||||
public_url, gen, file = file_service.get_file_generator_by_file_id("file_id", "ts", "nonce", "sign")
|
||||
assert public_url == "https://cdn.example.com/upload_files/tenant/abc.bin"
|
||||
assert gen is None
|
||||
assert file == upload_file
|
||||
mock_storage.load.assert_not_called()
|
||||
|
||||
def test_get_file_generator_by_file_id_invalid_sig(self, file_service):
|
||||
with patch("services.file_service.file_helpers.verify_file_signature") as mock_verify:
|
||||
mock_verify.return_value = False
|
||||
@@ -324,11 +369,29 @@ class TestFileService:
|
||||
mock_db_session.scalar.return_value = upload_file
|
||||
|
||||
with patch("services.file_service.storage") as mock_storage:
|
||||
mock_storage.get_public_url.return_value = None
|
||||
mock_storage.load.return_value = b"image content"
|
||||
gen, mime = file_service.get_public_image_preview("file_id")
|
||||
public_url, gen, mime = file_service.get_public_image_preview("file_id")
|
||||
assert public_url is None
|
||||
assert gen == b"image content"
|
||||
assert mime == "image/png"
|
||||
|
||||
def test_get_public_image_preview_redirects_when_storage_has_public_url(self, file_service, mock_db_session):
|
||||
upload_file = MagicMock(spec=UploadFile)
|
||||
upload_file.id = "file_id"
|
||||
upload_file.extension = "png"
|
||||
upload_file.mime_type = "image/png"
|
||||
upload_file.key = "upload_files/tenant/logo.png"
|
||||
mock_db_session.scalar.return_value = upload_file
|
||||
|
||||
with patch("services.file_service.storage") as mock_storage:
|
||||
mock_storage.get_public_url.return_value = "https://cdn.example.com/upload_files/tenant/logo.png"
|
||||
public_url, gen, mime = file_service.get_public_image_preview("file_id")
|
||||
assert public_url == "https://cdn.example.com/upload_files/tenant/logo.png"
|
||||
assert gen is None
|
||||
assert mime == "image/png"
|
||||
mock_storage.load.assert_not_called()
|
||||
|
||||
def test_get_public_image_preview_not_found(self, file_service, mock_db_session):
|
||||
mock_db_session.scalar.return_value = None
|
||||
with pytest.raises(NotFound, match="File not found or signature is invalid"):
|
||||
|
||||
@@ -483,6 +483,14 @@ S3_ADDRESS_STYLE=auto
|
||||
# Whether to use AWS managed IAM roles for authenticating with the S3 service.
|
||||
# If set to false, the access key and secret key must be provided.
|
||||
S3_USE_AWS_MANAGED_IAM=false
|
||||
# Optional public base URL for objects in the bucket. When set, signed file
|
||||
# previews are served by 302-redirecting to "<base>/<object-key>" so that bytes
|
||||
# are delivered directly by the object store / CDN. Examples:
|
||||
# Cloudflare R2 custom domain: https://cdn.example.com
|
||||
# MinIO public endpoint: https://minio.example.com/your-bucket
|
||||
# Aliyun OSS public domain: https://your-bucket.oss-cn-hangzhou.aliyuncs.com
|
||||
# Leave empty to keep the default API-streamed behavior.
|
||||
S3_PUBLIC_BASE_URL=
|
||||
|
||||
# Workflow run and Conversation archive storage (S3-compatible)
|
||||
ARCHIVE_STORAGE_ENABLED=false
|
||||
|
||||
@@ -136,6 +136,7 @@ x-shared-env: &shared-api-worker-env
|
||||
S3_SECRET_KEY: ${S3_SECRET_KEY:-}
|
||||
S3_ADDRESS_STYLE: ${S3_ADDRESS_STYLE:-auto}
|
||||
S3_USE_AWS_MANAGED_IAM: ${S3_USE_AWS_MANAGED_IAM:-false}
|
||||
S3_PUBLIC_BASE_URL: ${S3_PUBLIC_BASE_URL:-}
|
||||
ARCHIVE_STORAGE_ENABLED: ${ARCHIVE_STORAGE_ENABLED:-false}
|
||||
ARCHIVE_STORAGE_ENDPOINT: ${ARCHIVE_STORAGE_ENDPOINT:-}
|
||||
ARCHIVE_STORAGE_ARCHIVE_BUCKET: ${ARCHIVE_STORAGE_ARCHIVE_BUCKET:-}
|
||||
|
||||
@@ -10,7 +10,7 @@ export default function SignInLayout({ children }: any) {
|
||||
return (
|
||||
<>
|
||||
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
|
||||
<div className={cn('flex w-full min-w-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<div className={cn('flex w-full shrink-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<Header />
|
||||
<div className={
|
||||
cn(
|
||||
@@ -20,7 +20,7 @@ export default function SignInLayout({ children }: any) {
|
||||
)
|
||||
}
|
||||
>
|
||||
<div className="flex w-full flex-col md:w-[400px]">
|
||||
<div className="flex w-[400px] flex-col">
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -14,10 +14,10 @@ export default function SignInLayout({ children }: PropsWithChildren) {
|
||||
return (
|
||||
<>
|
||||
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
|
||||
<div className={cn('flex w-full min-w-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<div className={cn('flex w-full shrink-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
{/* <Header /> */}
|
||||
<div className={cn('flex w-full grow flex-col items-center justify-center px-6 md:px-[108px]')}>
|
||||
<div className="flex w-full justify-center md:w-[440px] lg:w-[600px]">
|
||||
<div className="flex justify-center md:w-[440px] lg:w-[600px]">
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -31,10 +31,10 @@ export default function SignInLayout({ children }: any) {
|
||||
return (
|
||||
<>
|
||||
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
|
||||
<div className={cn('flex w-full min-w-0 flex-col items-center rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<div className={cn('flex w-full shrink-0 flex-col items-center rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<Header />
|
||||
<div className={cn('flex w-full grow flex-col items-center justify-center px-6 md:px-[108px]')}>
|
||||
<div className="flex w-full flex-col md:w-[400px]">
|
||||
<div className="flex flex-col md:w-[400px]">
|
||||
{isLoggedIn
|
||||
? (
|
||||
<AppContextProvider>
|
||||
|
||||
@@ -10,7 +10,7 @@ export default function SignInLayout({ children }: any) {
|
||||
return (
|
||||
<>
|
||||
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
|
||||
<div className={cn('flex w-full min-w-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<div className={cn('flex w-full shrink-0 flex-col rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<Header />
|
||||
<div className={
|
||||
cn(
|
||||
@@ -20,7 +20,7 @@ export default function SignInLayout({ children }: any) {
|
||||
)
|
||||
}
|
||||
>
|
||||
<div className="flex w-full flex-col md:w-[400px]">
|
||||
<div className="flex flex-col md:w-[400px]">
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -12,10 +12,10 @@ export default function SignInLayout({ children }: any) {
|
||||
return (
|
||||
<>
|
||||
<div className={cn('flex min-h-screen w-full justify-center bg-background-default-burn p-6')}>
|
||||
<div className={cn('flex w-full min-w-0 flex-col items-center rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<div className={cn('flex w-full shrink-0 flex-col items-center rounded-2xl border border-effects-highlight bg-background-default-subtle')}>
|
||||
<Header />
|
||||
<div className={cn('flex w-full grow flex-col items-center justify-center px-6 md:px-[108px]')}>
|
||||
<div className="flex w-full flex-col md:w-[400px]">
|
||||
<div className="flex flex-col md:w-[400px]">
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"analysis.ms": "мс",
|
||||
"analysis.title": "Анализ",
|
||||
"analysis.tokenPS": "Токен/с",
|
||||
"analysis.tokenUsage.consumed": "Потреблено",
|
||||
"analysis.tokenUsage.consumed": "Потрачено",
|
||||
"analysis.tokenUsage.explanation": "Отражает ежедневное использование токенов языковой модели для приложения, полезно для целей контроля затрат.",
|
||||
"analysis.tokenUsage.title": "Использование токенов",
|
||||
"analysis.totalConversations.explanation": "Ежедневное количество чатов с LLM; проектирование/отладка не учитываются.",
|
||||
@@ -62,7 +62,7 @@
|
||||
"overview.appInfo.enableTooltip.description": "Чтобы включить эту функцию, добавьте на холст узел ввода пользователя. (Может уже существовать в черновике, вступает в силу после публикации)",
|
||||
"overview.appInfo.enableTooltip.learnMore": "Узнать больше",
|
||||
"overview.appInfo.explanation": "Готовое к использованию веб-приложение ИИ",
|
||||
"overview.appInfo.launch": "Запустить",
|
||||
"overview.appInfo.launch": "Баркас",
|
||||
"overview.appInfo.preUseReminder": "Пожалуйста, включите веб-приложение перед продолжением.",
|
||||
"overview.appInfo.preview": "Предварительный просмотр",
|
||||
"overview.appInfo.qrcode.download": "Скачать QR-код",
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"embedding.automatic": "Автоматически",
|
||||
"embedding.childMaxTokens": "Наследник",
|
||||
"embedding.childMaxTokens": "Ребёнок",
|
||||
"embedding.completed": "Встраивание завершено",
|
||||
"embedding.custom": "Пользовательский",
|
||||
"embedding.docName": "Имя документа",
|
||||
"embedding.docName": "Предварительная обработка документа",
|
||||
"embedding.economy": "Экономичный режим",
|
||||
"embedding.error": "Ошибка расчета эмбеддингов",
|
||||
"embedding.estimate": "Оценка",
|
||||
"embedding.hierarchical": "Иерархический",
|
||||
"embedding.estimate": "Оценочное потребление",
|
||||
"embedding.hierarchical": "Родитель-дочерний",
|
||||
"embedding.highQuality": "Режим высокого качества",
|
||||
"embedding.mode": "Правило сегментации",
|
||||
"embedding.parentMaxTokens": "Родитель",
|
||||
@@ -16,7 +16,7 @@
|
||||
"embedding.previewTip": "Предварительный просмотр абзацев будет доступен после завершения расчета эмбеддингов",
|
||||
"embedding.processing": "Расчет эмбеддингов...",
|
||||
"embedding.resume": "Возобновить обработку",
|
||||
"embedding.segmentLength": "Длина сегментов",
|
||||
"embedding.segmentLength": "Длина фрагментов",
|
||||
"embedding.segments": "Абзацы",
|
||||
"embedding.stop": "Остановить обработку",
|
||||
"embedding.textCleaning": "Предварительная очистка текста",
|
||||
@@ -279,25 +279,25 @@
|
||||
"metadata.type.webPage": "Веб-страница",
|
||||
"metadata.type.wikipediaEntry": "Статья в Википедии",
|
||||
"segment.addAnother": "Добавить еще один",
|
||||
"segment.addChildChunk": "Добавить дочерний фрагмент",
|
||||
"segment.addChunk": "Добавить фрагмент",
|
||||
"segment.addChildChunk": "Добавить дочерний чанк",
|
||||
"segment.addChunk": "Добавить чанк",
|
||||
"segment.addKeyWord": "Добавить ключевое слово",
|
||||
"segment.allFilesUploaded": "Все файлы должны быть загружены перед сохранением",
|
||||
"segment.answerEmpty": "Ответ не может быть пустым",
|
||||
"segment.answerPlaceholder": "добавьте ответ здесь",
|
||||
"segment.characters_one": "символ",
|
||||
"segment.characters_other": "символы",
|
||||
"segment.childChunk": "Дочерний фрагмент",
|
||||
"segment.childChunkAdded": "Добавлен 1 дочерний фрагмент",
|
||||
"segment.childChunks_one": "ДОЧЕРНИЙ ФРАГМЕНТ",
|
||||
"segment.childChunks_other": "ДОЧЕРНИЕ ФРАГМЕНТЫ",
|
||||
"segment.chunk": "Фрагмент",
|
||||
"segment.chunkAdded": "Добавлен 1 фрагмент",
|
||||
"segment.chunkDetail": "Детали фрагмента",
|
||||
"segment.chunks_one": "ФРАГМЕНТ",
|
||||
"segment.chunks_other": "ФРАГМЕНТЫ",
|
||||
"segment.characters_one": "характер",
|
||||
"segment.characters_other": "письмена",
|
||||
"segment.childChunk": "Чайлд-Чанк",
|
||||
"segment.childChunkAdded": "Добавлен 1 дочерний чанк",
|
||||
"segment.childChunks_one": "ДОЧЕРНИЙ ЧАНК",
|
||||
"segment.childChunks_other": "ДЕТСКИЕ КУСОЧКИ",
|
||||
"segment.chunk": "Ломоть",
|
||||
"segment.chunkAdded": "Добавлен 1 блок",
|
||||
"segment.chunkDetail": "Деталь Чанка",
|
||||
"segment.chunks_one": "ЛОМОТЬ",
|
||||
"segment.chunks_other": "КУСКИ",
|
||||
"segment.clearFilter": "Очистить фильтр",
|
||||
"segment.collapseChunks": "Свернуть фрагменты",
|
||||
"segment.collapseChunks": "Сворачивание кусков",
|
||||
"segment.contentEmpty": "Содержимое не может быть пустым",
|
||||
"segment.contentPlaceholder": "добавьте содержимое здесь",
|
||||
"segment.dateTimeFormat": "MM/DD/YYYY HH:mm",
|
||||
@@ -307,15 +307,15 @@
|
||||
"segment.editParentChunk": "Редактирование родительского блока",
|
||||
"segment.edited": "ОТРЕДАКТИРОВАНЫ",
|
||||
"segment.editedAt": "Отредактировано в",
|
||||
"segment.empty": "Фрагмент не найден",
|
||||
"segment.expandChunks": "Развернуть фрагменты",
|
||||
"segment.empty": "Чанк не найден",
|
||||
"segment.expandChunks": "Развернуть чанки",
|
||||
"segment.hitCount": "Количество обращений",
|
||||
"segment.keywordDuplicate": "Ключевое слово уже существует",
|
||||
"segment.keywordEmpty": "Ключевое слово не может быть пустым",
|
||||
"segment.keywordError": "Максимальная длина ключевого слова - 20",
|
||||
"segment.keywords": "Ключевые слова",
|
||||
"segment.newChildChunk": "Новый дочерний фрагмент",
|
||||
"segment.newChunk": "Новый фрагмент",
|
||||
"segment.newChildChunk": "Новый дочерний чанк",
|
||||
"segment.newChunk": "Новый чанк",
|
||||
"segment.newQaSegment": "Новый сегмент вопрос-ответ",
|
||||
"segment.newTextSegment": "Новый текстовый сегмент",
|
||||
"segment.paragraphs": "Абзацы",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"blocks.agent": "Агент",
|
||||
"blocks.answer": "Ответ",
|
||||
"blocks.assigner": "Назначение переменных",
|
||||
"blocks.assigner": "Назначение переменной",
|
||||
"blocks.code": "Код",
|
||||
"blocks.datasource": "Источник данных",
|
||||
"blocks.datasource-empty": "Пустой источник данных",
|
||||
@@ -17,10 +17,10 @@
|
||||
"blocks.list-operator": "Оператор списка",
|
||||
"blocks.llm": "LLM",
|
||||
"blocks.loop": "Цикл",
|
||||
"blocks.loop-end": "Конец цикла",
|
||||
"blocks.loop-end": "Выйти из цикла",
|
||||
"blocks.loop-start": "Начало цикла",
|
||||
"blocks.originalStartNode": "исходный начальный узел",
|
||||
"blocks.parameter-extractor": "Экстрактор параметров",
|
||||
"blocks.parameter-extractor": "Извлечение параметров",
|
||||
"blocks.question-classifier": "Классификатор вопросов",
|
||||
"blocks.start": "Начало",
|
||||
"blocks.template-transform": "Шаблон",
|
||||
@@ -29,7 +29,7 @@
|
||||
"blocks.trigger-schedule": "Триггер расписания",
|
||||
"blocks.trigger-webhook": "Вебхук-триггер",
|
||||
"blocks.variable-aggregator": "Агрегатор переменных",
|
||||
"blocks.variable-assigner": "Назначение переменных",
|
||||
"blocks.variable-assigner": "Агрегатор переменных",
|
||||
"blocksAbout.agent": "Вызов больших языковых моделей для ответа на вопросы или обработки естественного языка",
|
||||
"blocksAbout.answer": "Определите содержимое ответа в чате",
|
||||
"blocksAbout.assigner": "Узел назначения переменной используется для назначения значений записываемым переменным (например, переменным разговора).",
|
||||
@@ -485,7 +485,7 @@
|
||||
"nodes.common.pluginNotInstalled": "Плагин не установлен",
|
||||
"nodes.common.pluginsNotInstalled": "{{count}} плагинов не установлено",
|
||||
"nodes.common.retry.maxRetries": "максимальное количество повторных попыток",
|
||||
"nodes.common.retry.ms": "мс",
|
||||
"nodes.common.retry.ms": "госпожа",
|
||||
"nodes.common.retry.retries": "{{num}} Повторных попыток",
|
||||
"nodes.common.retry.retry": "Снова пробовать",
|
||||
"nodes.common.retry.retryFailed": "Повторная попытка не удалась",
|
||||
|
||||
Reference in New Issue
Block a user