if session unauthorized, rejoin

- Introduced AmplitudeProvider component for initializing Amplitude analytics.
- Integrated user tracking in AppContextProvider to report user and workspace information.
- Added event tracking for workflow log filter selections.
- Updated package.json and pnpm-lock.yaml to include @amplitude/unified dependency.

api/.env.example

@@ -30,6 +30,9 @@ INTERNAL_FILES_URL=http://127.0.0.1:5001
 # The time in seconds after the signature is rejected
 FILES_ACCESS_TIMEOUT=300
 
+# Collaboration mode toggle
+ENABLE_COLLABORATION_MODE=false
+
 # Access token expiration time in minutes
 ACCESS_TOKEN_EXPIRE_MINUTES=60
 

api/app.py

@@ -1,3 +1,4 @@
+import os
 import sys
 
 
@@ -8,10 +9,16 @@ def is_db_command():
 
 
 # create app
+celery = None
+flask_app = None
+socketio_app = None
+
 if is_db_command():
     from app_factory import create_migrations_app
 
     app = create_migrations_app()
+    socketio_app = app
+    flask_app = app
 else:
     # Gunicorn and Celery handle monkey patching automatically in production by
     # specifying the `gevent` worker class. Manual monkey patching is not required here.
@@ -22,8 +29,15 @@ else:
 
     from app_factory import create_app
 
-    app = create_app()
-    celery = app.extensions["celery"]
+    socketio_app, flask_app = create_app()
+    app = flask_app
+    celery = flask_app.extensions["celery"]
 
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=5001)
+    from gevent import pywsgi
+    from geventwebsocket.handler import WebSocketHandler
+
+    host = os.environ.get("HOST", "0.0.0.0")
+    port = int(os.environ.get("PORT", 5001))
+    server = pywsgi.WSGIServer((host, port), socketio_app, handler_class=WebSocketHandler)
+    server.serve_forever()

api/app_factory.py

@@ -31,14 +31,22 @@ def create_flask_app_with_configs() -> DifyApp:
     return dify_app
 
 
-def create_app() -> DifyApp:
+def create_app() -> tuple[any, DifyApp]:
     start_time = time.perf_counter()
     app = create_flask_app_with_configs()
     initialize_extensions(app)
+
+    import socketio
+
+    from extensions.ext_socketio import sio
+
+    sio.app = app
+    socketio_app = socketio.WSGIApp(sio, app)
+
     end_time = time.perf_counter()
     if dify_config.DEBUG:
         logger.info("Finished create_app (%s ms)", round((end_time - start_time) * 1000, 2))
-    return app
+    return socketio_app, app
 
 
 def initialize_extensions(app: DifyApp):

api/configs/feature/__init__.py

@@ -871,6 +871,16 @@ class MailConfig(BaseSettings):
         default=None,
     )
 
+    ENABLE_TRIAL_APP: bool = Field(
+        description="Enable trial app",
+        default=False,
+    )
+
+    ENABLE_EXPLORE_BANNER: bool = Field(
+        description="Enable explore banner",
+        default=False,
+    )
+
 
 class RagEtlConfig(BaseSettings):
     """
@@ -1083,6 +1093,13 @@ class PositionConfig(BaseSettings):
         return {item.strip() for item in self.POSITION_TOOL_EXCLUDES.split(",") if item.strip() != ""}
 
 
+class CollaborationConfig(BaseSettings):
+    ENABLE_COLLABORATION_MODE: bool = Field(
+        description="Whether to enable collaboration mode features across the workspace",
+        default=False,
+    )
+
+
 class LoginConfig(BaseSettings):
     ENABLE_EMAIL_CODE_LOGIN: bool = Field(
         description="whether to enable email code login",
@@ -1142,6 +1159,13 @@ class SwaggerUIConfig(BaseSettings):
     )
 
 
+class TenantSelfTaskQueueConfig(BaseSettings):
+    TENANT_SELF_TASK_QUEUE_PULL_SIZE: int = Field(
+        description="Default batch size for tenant self task queue pull operations",
+        default=1,
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -1166,11 +1190,13 @@ class FeatureConfig(
     RagEtlConfig,
     RepositoryConfig,
     SecurityConfig,
+    TenantSelfTaskQueueConfig,
     ToolConfig,
     UpdateConfig,
     WorkflowConfig,
     WorkflowNodeExecutionConfig,
     WorkspaceConfig,
+    CollaborationConfig,
     LoginConfig,
     AccountConfig,
     SwaggerUIConfig,

api/configs/feature/hosted_service/__init__.py

@@ -8,6 +8,11 @@ class HostedCreditConfig(BaseSettings):
         default="",
     )
 
+    HOSTED_POOL_CREDITS: int = Field(
+        description="Pool credits for hosted service",
+        default=200,
+    )
+
     def get_model_credits(self, model_name: str) -> int:
         """
         Get credit value for a specific model name.
@@ -60,19 +65,46 @@ class HostedOpenAiConfig(BaseSettings):
 
     HOSTED_OPENAI_TRIAL_MODELS: str = Field(
         description="Comma-separated list of available models for trial access",
-        default="gpt-3.5-turbo,"
-        "gpt-3.5-turbo-1106,"
-        "gpt-3.5-turbo-instruct,"
+        default="gpt-4,"
+        "gpt-4-turbo-preview,"
+        "gpt-4-turbo-2024-04-09,"
+        "gpt-4-1106-preview,"
+        "gpt-4-0125-preview,"
+        "gpt-4-turbo,"
+        "gpt-4.1,"
+        "gpt-4.1-2025-04-14,"
+        "gpt-4.1-mini,"
+        "gpt-4.1-mini-2025-04-14,"
+        "gpt-4.1-nano,"
+        "gpt-4.1-nano-2025-04-14,"
+        "gpt-3.5-turbo,"
         "gpt-3.5-turbo-16k,"
         "gpt-3.5-turbo-16k-0613,"
+        "gpt-3.5-turbo-1106,"
         "gpt-3.5-turbo-0613,"
         "gpt-3.5-turbo-0125,"
-        "text-davinci-003",
-    )
-
-    HOSTED_OPENAI_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="Quota limit for hosted OpenAI service usage",
-        default=200,
+        "gpt-3.5-turbo-instruct,"
+        "text-davinci-003,"
+        "chatgpt-4o-latest,"
+        "gpt-4o,"
+        "gpt-4o-2024-05-13,"
+        "gpt-4o-2024-08-06,"
+        "gpt-4o-2024-11-20,"
+        "gpt-4o-audio-preview,"
+        "gpt-4o-audio-preview-2025-06-03,"
+        "gpt-4o-mini,"
+        "gpt-4o-mini-2024-07-18,"
+        "o3-mini,"
+        "o3-mini-2025-01-31,"
+        "gpt-5-mini-2025-08-07,"
+        "gpt-5-mini,"
+        "o4-mini,"
+        "o4-mini-2025-04-16,"
+        "gpt-5-chat-latest,"
+        "gpt-5,"
+        "gpt-5-2025-08-07,"
+        "gpt-5-nano,"
+        "gpt-5-nano-2025-08-07",
     )
 
     HOSTED_OPENAI_PAID_ENABLED: bool = Field(
@@ -87,6 +119,13 @@ class HostedOpenAiConfig(BaseSettings):
         "gpt-4-turbo-2024-04-09,"
         "gpt-4-1106-preview,"
         "gpt-4-0125-preview,"
+        "gpt-4-turbo,"
+        "gpt-4.1,"
+        "gpt-4.1-2025-04-14,"
+        "gpt-4.1-mini,"
+        "gpt-4.1-mini-2025-04-14,"
+        "gpt-4.1-nano,"
+        "gpt-4.1-nano-2025-04-14,"
         "gpt-3.5-turbo,"
         "gpt-3.5-turbo-16k,"
         "gpt-3.5-turbo-16k-0613,"
@@ -94,7 +133,150 @@ class HostedOpenAiConfig(BaseSettings):
         "gpt-3.5-turbo-0613,"
         "gpt-3.5-turbo-0125,"
         "gpt-3.5-turbo-instruct,"
-        "text-davinci-003",
+        "text-davinci-003,"
+        "chatgpt-4o-latest,"
+        "gpt-4o,"
+        "gpt-4o-2024-05-13,"
+        "gpt-4o-2024-08-06,"
+        "gpt-4o-2024-11-20,"
+        "gpt-4o-audio-preview,"
+        "gpt-4o-audio-preview-2025-06-03,"
+        "gpt-4o-mini,"
+        "gpt-4o-mini-2024-07-18,"
+        "o3-mini,"
+        "o3-mini-2025-01-31,"
+        "gpt-5-mini-2025-08-07,"
+        "gpt-5-mini,"
+        "o4-mini,"
+        "o4-mini-2025-04-16,"
+        "gpt-5-chat-latest,"
+        "gpt-5,"
+        "gpt-5-2025-08-07,"
+        "gpt-5-nano,"
+        "gpt-5-nano-2025-08-07",
+    )
+
+
+class HostedGeminiConfig(BaseSettings):
+    """
+    Configuration for fetching Gemini service
+    """
+
+    HOSTED_GEMINI_API_KEY: str | None = Field(
+        description="API key for hosted Gemini service",
+        default=None,
+    )
+
+    HOSTED_GEMINI_API_BASE: str | None = Field(
+        description="Base URL for hosted Gemini API",
+        default=None,
+    )
+
+    HOSTED_GEMINI_API_ORGANIZATION: str | None = Field(
+        description="Organization ID for hosted Gemini service",
+        default=None,
+    )
+
+    HOSTED_GEMINI_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted Gemini service",
+        default=False,
+    )
+
+    HOSTED_GEMINI_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for trial access",
+        default="gemini-2.5-flash,gemini-2.0-flash,gemini-2.0-flash-lite,",
+    )
+
+    HOSTED_GEMINI_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted gemini service",
+        default=False,
+    )
+
+    HOSTED_GEMINI_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="gemini-2.5-flash,gemini-2.0-flash,gemini-2.0-flash-lite,",
+    )
+
+
+class HostedXAIConfig(BaseSettings):
+    """
+    Configuration for fetching XAI service
+    """
+
+    HOSTED_XAI_API_KEY: str | None = Field(
+        description="API key for hosted XAI service",
+        default=None,
+    )
+
+    HOSTED_XAI_API_BASE: str | None = Field(
+        description="Base URL for hosted XAI API",
+        default=None,
+    )
+
+    HOSTED_XAI_API_ORGANIZATION: str | None = Field(
+        description="Organization ID for hosted XAI service",
+        default=None,
+    )
+
+    HOSTED_XAI_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted XAI service",
+        default=False,
+    )
+
+    HOSTED_XAI_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for trial access",
+        default="grok-3,grok-3-mini,grok-3-mini-fast",
+    )
+
+    HOSTED_XAI_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted XAI service",
+        default=False,
+    )
+
+    HOSTED_XAI_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="grok-3,grok-3-mini,grok-3-mini-fast",
+    )
+
+
+class HostedDeepseekConfig(BaseSettings):
+    """
+    Configuration for fetching Deepseek service
+    """
+
+    HOSTED_DEEPSEEK_API_KEY: str | None = Field(
+        description="API key for hosted Deepseek service",
+        default=None,
+    )
+
+    HOSTED_DEEPSEEK_API_BASE: str | None = Field(
+        description="Base URL for hosted Deepseek API",
+        default=None,
+    )
+
+    HOSTED_DEEPSEEK_API_ORGANIZATION: str | None = Field(
+        description="Organization ID for hosted Deepseek service",
+        default=None,
+    )
+
+    HOSTED_DEEPSEEK_TRIAL_ENABLED: bool = Field(
+        description="Enable trial access to hosted Deepseek service",
+        default=False,
+    )
+
+    HOSTED_DEEPSEEK_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for trial access",
+        default="deepseek-chat,deepseek-reasoner",
+    )
+
+    HOSTED_DEEPSEEK_PAID_ENABLED: bool = Field(
+        description="Enable paid access to hosted XAI service",
+        default=False,
+    )
+
+    HOSTED_DEEPSEEK_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="deepseek-chat,deepseek-reasoner",
     )
 
 
@@ -144,16 +326,30 @@ class HostedAnthropicConfig(BaseSettings):
         default=False,
     )
 
-    HOSTED_ANTHROPIC_QUOTA_LIMIT: NonNegativeInt = Field(
-        description="Quota limit for hosted Anthropic service usage",
-        default=600000,
-    )
-
     HOSTED_ANTHROPIC_PAID_ENABLED: bool = Field(
         description="Enable paid access to hosted Anthropic service",
         default=False,
     )
 
+    HOSTED_ANTHROPIC_TRIAL_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="claude-opus-4-20250514,"
+        "claude-sonnet-4-20250514,"
+        "claude-3-5-haiku-20241022,"
+        "claude-3-opus-20240229,"
+        "claude-3-7-sonnet-20250219,"
+        "claude-3-haiku-20240307",
+    )
+    HOSTED_ANTHROPIC_PAID_MODELS: str = Field(
+        description="Comma-separated list of available models for paid access",
+        default="claude-opus-4-20250514,"
+        "claude-sonnet-4-20250514,"
+        "claude-3-5-haiku-20241022,"
+        "claude-3-opus-20240229,"
+        "claude-3-7-sonnet-20250219,"
+        "claude-3-haiku-20240307",
+    )
+
 
 class HostedMinmaxConfig(BaseSettings):
     """
@@ -250,5 +446,8 @@ class HostedServiceConfig(
     HostedModerationConfig,
     # credit config
     HostedCreditConfig,
+    HostedGeminiConfig,
+    HostedXAIConfig,
+    HostedDeepseekConfig,
 ):
     pass

api/controllers/console/__init__.py

@@ -58,11 +58,13 @@ from .app import (
     mcp_server,
     message,
     model_config,
+    online_user,
     ops_trace,
     site,
     statistic,
     workflow,
     workflow_app_log,
+    workflow_comment,
     workflow_draft_variable,
     workflow_run,
     workflow_statistic,
@@ -106,10 +108,12 @@ from .datasets.rag_pipeline import (
 
 # Import explore controllers
 from .explore import (
+    banner,
     installed_app,
     parameter,
     recommended_app,
     saved_message,
+    trial,
 )
 
 # Import tag controllers
@@ -143,6 +147,7 @@ __all__ = [
     "apikey",
     "app",
     "audio",
+    "banner",
     "billing",
     "bp",
     "completion",
@@ -196,6 +201,7 @@ __all__ = [
     "statistic",
     "tags",
     "tool_providers",
+    "trial",
     "version",
     "website",
     "workflow",

api/controllers/console/admin.py

@@ -16,7 +16,7 @@ from controllers.console import api, console_ns
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
 from libs.token import extract_access_token
-from models.model import App, InstalledApp, RecommendedApp
+from models.model import App, ExporleBanner, InstalledApp, RecommendedApp, TrialApp
 
 
 def admin_required(view: Callable[P, R]):
@@ -52,6 +52,8 @@ class InsertExploreAppListApi(Resource):
                 "language": fields.String(required=True, description="Language code"),
                 "category": fields.String(required=True, description="App category"),
                 "position": fields.Integer(required=True, description="Display position"),
+                "can_trial": fields.Boolean(required=True, description="Can trial"),
+                "trial_limit": fields.Integer(required=True, description="Trial limit"),
             },
         )
     )
@@ -71,6 +73,8 @@ class InsertExploreAppListApi(Resource):
             .add_argument("language", type=supported_language, required=True, nullable=False, location="json")
             .add_argument("category", type=str, required=True, nullable=False, location="json")
             .add_argument("position", type=int, required=True, nullable=False, location="json")
+            .add_argument("can_trial", type=bool, required=True, nullable=False, location="json")
+            .add_argument("trial_limit", type=int, required=True, nullable=False, location="json")
         )
         args = parser.parse_args()
 
@@ -108,6 +112,20 @@ class InsertExploreAppListApi(Resource):
                 )
 
                 db.session.add(recommended_app)
+                if args["can_trial"]:
+                    trial_app = db.session.execute(
+                        select(TrialApp).where(TrialApp.app_id == args["app_id"])
+                    ).scalar_one_or_none()
+                    if not trial_app:
+                        db.session.add(
+                            TrialApp(
+                                app_id=args["app_id"],
+                                tenant_id=app.tenant_id,
+                                trial_limit=args["trial_limit"],
+                            )
+                        )
+                    else:
+                        trial_app.trial_limit = args["trial_limit"]
 
                 app.is_public = True
                 db.session.commit()
@@ -122,6 +140,20 @@ class InsertExploreAppListApi(Resource):
                 recommended_app.category = args["category"]
                 recommended_app.position = args["position"]
 
+                if args["can_trial"]:
+                    trial_app = db.session.execute(
+                        select(TrialApp).where(TrialApp.app_id == args["app_id"])
+                    ).scalar_one_or_none()
+                    if not trial_app:
+                        db.session.add(
+                            TrialApp(
+                                app_id=args["app_id"],
+                                tenant_id=app.tenant_id,
+                                trial_limit=args["trial_limit"],
+                            )
+                        )
+                    else:
+                        trial_app.trial_limit = args["trial_limit"]
                 app.is_public = True
 
                 db.session.commit()
@@ -167,7 +199,83 @@ class InsertExploreAppApi(Resource):
             for installed_app in installed_apps:
                 session.delete(installed_app)
 
+            trial_app = session.execute(
+                select(TrialApp).where(TrialApp.app_id == recommended_app.app_id)
+            ).scalar_one_or_none()
+            if trial_app:
+                session.delete(trial_app)
+
         db.session.delete(recommended_app)
         db.session.commit()
 
         return {"result": "success"}, 204
+
+
+@console_ns.route("/admin/insert-explore-banner")
+class InsertExploreBanner(Resource):
+    @api.doc("insert_explore_banner")
+    @api.doc(description="Insert an explore banner")
+    @api.expect(
+        api.model(
+            "InsertExploreBannerRequest",
+            {
+                "content": fields.String(required=True, description="Banner content"),
+                "link": fields.String(required=True, description="Banner link"),
+                "sort": fields.Integer(required=True, description="Banner sort"),
+            },
+        )
+    )
+    @api.response(200, "Banner inserted successfully")
+    @admin_required
+    @only_edition_cloud
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("category", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("title", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("description", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("img-src", type=str, required=True, nullable=False, location="json")
+
+        parser.add_argument("language", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("link", type=str, required=True, nullable=False, location="json")
+        parser.add_argument("sort", type=int, required=True, nullable=False, location="json")
+
+        args = parser.parse_args()
+
+        content = {
+            "category": args["category"],
+            "title": args["title"],
+            "description": args["description"],
+            "img-src": args["img-src"],
+        }
+
+        if not args["language"]:
+            args["language"] = "en-US"
+
+        banner = ExporleBanner(
+            content=content,
+            link=args["link"],
+            sort=args["sort"],
+            language=args["language"],
+        )
+        db.session.add(banner)
+        db.session.commit()
+
+        return {"result": "success"}, 200
+
+
+@console_ns.route("/admin/delete-explore-banner/<uuid:banner_id>")
+class DeleteExploreBanner(Resource):
+    @api.doc("delete_explore_banner")
+    @api.doc(description="Delete an explore banner")
+    @api.response(204, "Banner deleted successfully")
+    @admin_required
+    @only_edition_cloud
+    def delete(self, banner_id):
+        banner = db.session.execute(select(ExporleBanner).where(ExporleBanner.id == banner_id)).scalar_one_or_none()
+        if not banner:
+            raise NotFound(f"Banner '{banner_id}' is not found")
+
+        db.session.delete(banner)
+        db.session.commit()
+
+        return {"result": "success"}, 204

api/controllers/console/app/error.py

@@ -115,3 +115,9 @@ class InvokeRateLimitError(BaseHTTPException):
     error_code = "rate_limit_error"
     description = "Rate Limit Error"
     code = 429
+
+
+class NeedAddIdsError(BaseHTTPException):
+    error_code = "need_add_ids"
+    description = "Need to add ids."
+    code = 400

api/controllers/console/app/online_user.py

@@ -0,0 +1,339 @@
+import json
+import time
+
+from werkzeug.wrappers import Request as WerkzeugRequest
+
+from extensions.ext_redis import redis_client
+from extensions.ext_socketio import sio
+from libs.passport import PassportService
+from libs.token import extract_access_token
+from services.account_service import AccountService
+
+SESSION_STATE_TTL_SECONDS = 3600
+WORKFLOW_ONLINE_USERS_PREFIX = "workflow_online_users:"
+WORKFLOW_LEADER_PREFIX = "workflow_leader:"
+WS_SID_MAP_PREFIX = "ws_sid_map:"
+
+
+def _workflow_key(workflow_id: str) -> str:
+    return f"{WORKFLOW_ONLINE_USERS_PREFIX}{workflow_id}"
+
+
+def _leader_key(workflow_id: str) -> str:
+    return f"{WORKFLOW_LEADER_PREFIX}{workflow_id}"
+
+
+def _sid_key(sid: str) -> str:
+    return f"{WS_SID_MAP_PREFIX}{sid}"
+
+
+def _refresh_session_state(workflow_id: str, sid: str) -> None:
+    """
+    Refresh TTLs for workflow + session keys so healthy sessions do not linger forever after crashes.
+    """
+    workflow_key = _workflow_key(workflow_id)
+    sid_key = _sid_key(sid)
+    if redis_client.exists(workflow_key):
+        redis_client.expire(workflow_key, SESSION_STATE_TTL_SECONDS)
+    if redis_client.exists(sid_key):
+        redis_client.expire(sid_key, SESSION_STATE_TTL_SECONDS)
+
+
+@sio.on("connect")
+def socket_connect(sid, environ, auth):
+    """
+    WebSocket connect event, do authentication here.
+    """
+    token = None
+    if auth and isinstance(auth, dict):
+        token = auth.get("token")
+
+    if not token:
+        try:
+            request_environ = WerkzeugRequest(environ)
+            token = extract_access_token(request_environ)
+        except Exception:
+            token = None
+
+    if not token:
+        return False
+
+    try:
+        decoded = PassportService().verify(token)
+        user_id = decoded.get("user_id")
+        if not user_id:
+            return False
+
+        with sio.app.app_context():
+            user = AccountService.load_logged_in_account(account_id=user_id)
+            if not user:
+                return False
+
+            sio.save_session(sid, {"user_id": user.id, "username": user.name, "avatar": user.avatar})
+
+            return True
+
+    except Exception:
+        return False
+
+
+@sio.on("user_connect")
+def handle_user_connect(sid, data):
+    """
+    Handle user connect event. Each session (tab) is treated as an independent collaborator.
+    """
+
+    workflow_id = data.get("workflow_id")
+    if not workflow_id:
+        return {"msg": "workflow_id is required"}, 400
+
+    session = sio.get_session(sid)
+    user_id = session.get("user_id")
+
+    if not user_id:
+        return {"msg": "unauthorized"}, 401
+
+    # Each session is stored independently with sid as key
+    session_info = {
+        "user_id": user_id,
+        "username": session.get("username", "Unknown"),
+        "avatar": session.get("avatar", None),
+        "sid": sid,
+        "connected_at": int(time.time()),  # Add timestamp to differentiate tabs
+    }
+
+    workflow_key = _workflow_key(workflow_id)
+    # Store session info with sid as key
+    redis_client.hset(workflow_key, sid, json.dumps(session_info))
+    redis_client.set(
+        _sid_key(sid),
+        json.dumps({"workflow_id": workflow_id, "user_id": user_id}),
+        ex=SESSION_STATE_TTL_SECONDS,
+    )
+    _refresh_session_state(workflow_id, sid)
+
+    # Leader election: first session becomes the leader
+    leader_sid = get_or_set_leader(workflow_id, sid)
+    is_leader = leader_sid == sid
+
+    sio.enter_room(sid, workflow_id)
+    broadcast_online_users(workflow_id)
+
+    # Notify this session of their leader status
+    sio.emit("status", {"isLeader": is_leader}, room=sid)
+
+    return {"msg": "connected", "user_id": user_id, "sid": sid, "isLeader": is_leader}
+
+
+@sio.on("disconnect")
+def handle_disconnect(sid):
+    """
+    Handle session disconnect event. Remove the specific session from online users.
+    """
+    mapping = redis_client.get(_sid_key(sid))
+    if mapping:
+        data = json.loads(mapping)
+        workflow_id = data["workflow_id"]
+
+        # Remove this specific session
+        redis_client.hdel(_workflow_key(workflow_id), sid)
+        redis_client.delete(_sid_key(sid))
+
+        # Handle leader re-election if the leader session disconnected
+        handle_leader_disconnect(workflow_id, sid)
+
+        broadcast_online_users(workflow_id)
+
+
+def _clear_session_state(workflow_id: str, sid: str) -> None:
+    redis_client.hdel(_workflow_key(workflow_id), sid)
+    redis_client.delete(_sid_key(sid))
+
+
+def _is_session_active(workflow_id: str, sid: str) -> bool:
+    if not sid:
+        return False
+
+    try:
+        if not sio.manager.is_connected(sid, "/"):
+            return False
+    except AttributeError:
+        return False
+
+    if not redis_client.hexists(_workflow_key(workflow_id), sid):
+        return False
+
+    if not redis_client.exists(_sid_key(sid)):
+        return False
+
+    return True
+
+
+def get_or_set_leader(workflow_id: str, sid: str) -> str:
+    """
+    Get current leader session or set this session as leader if no valid leader exists.
+    Returns the leader session id (sid).
+    """
+    raw_leader = redis_client.get(_leader_key(workflow_id))
+    current_leader = raw_leader.decode("utf-8") if isinstance(raw_leader, bytes) else raw_leader
+    leader_replaced = False
+
+    if current_leader and not _is_session_active(workflow_id, current_leader):
+        _clear_session_state(workflow_id, current_leader)
+        redis_client.delete(_leader_key(workflow_id))
+        current_leader = None
+        leader_replaced = True
+
+    if not current_leader:
+        redis_client.set(_leader_key(workflow_id), sid, ex=SESSION_STATE_TTL_SECONDS)  # Expire in 1 hour
+        if leader_replaced:
+            broadcast_leader_change(workflow_id, sid)
+        return sid
+
+    return current_leader
+
+
+def handle_leader_disconnect(workflow_id, disconnected_sid):
+    """
+    Handle leader re-election when a session disconnects.
+    If the disconnected session was the leader, elect a new leader from remaining sessions.
+    """
+    current_leader = redis_client.get(_leader_key(workflow_id))
+
+    if current_leader:
+        current_leader = current_leader.decode("utf-8") if isinstance(current_leader, bytes) else current_leader
+
+        if current_leader == disconnected_sid:
+            # Leader session disconnected, elect a new leader
+            sessions_json = redis_client.hgetall(_workflow_key(workflow_id))
+
+            if sessions_json:
+                # Get the first remaining session as new leader
+                new_leader_sid = list(sessions_json.keys())[0]
+                if isinstance(new_leader_sid, bytes):
+                    new_leader_sid = new_leader_sid.decode("utf-8")
+
+                redis_client.set(_leader_key(workflow_id), new_leader_sid, ex=SESSION_STATE_TTL_SECONDS)
+
+                # Notify all sessions about the new leader
+                broadcast_leader_change(workflow_id, new_leader_sid)
+            else:
+                # No sessions left, remove leader
+                redis_client.delete(_leader_key(workflow_id))
+
+
+def broadcast_leader_change(workflow_id, new_leader_sid):
+    """
+    Broadcast leader change to all sessions in the workflow.
+    """
+    sessions_json = redis_client.hgetall(_workflow_key(workflow_id))
+
+    for sid, session_info_json in sessions_json.items():
+        try:
+            sid_str = sid.decode("utf-8") if isinstance(sid, bytes) else sid
+            is_leader = sid_str == new_leader_sid
+            # Emit to each session whether they are the new leader
+            sio.emit("status", {"isLeader": is_leader}, room=sid_str)
+        except Exception:
+            continue
+
+
+def get_current_leader(workflow_id):
+    """
+    Get the current leader for a workflow.
+    """
+    leader = redis_client.get(_leader_key(workflow_id))
+    return leader.decode("utf-8") if leader and isinstance(leader, bytes) else leader
+
+
+def broadcast_online_users(workflow_id):
+    """
+    Broadcast online users to the workflow room.
+    Each session is shown as a separate user (even if same person has multiple tabs).
+    """
+    sessions_json = redis_client.hgetall(_workflow_key(workflow_id))
+    users = []
+
+    for sid, session_info_json in sessions_json.items():
+        try:
+            session_info = json.loads(session_info_json)
+            # Each session appears as a separate "user" in the UI
+            users.append(
+                {
+                    "user_id": session_info["user_id"],
+                    "username": session_info["username"],
+                    "avatar": session_info.get("avatar"),
+                    "sid": session_info["sid"],
+                    "connected_at": session_info.get("connected_at"),
+                }
+            )
+        except Exception:
+            continue
+
+    # Sort by connection time to maintain consistent order
+    users.sort(key=lambda x: x.get("connected_at") or 0)
+
+    # Get current leader session
+    leader_sid = get_current_leader(workflow_id)
+
+    sio.emit("online_users", {"workflow_id": workflow_id, "users": users, "leader": leader_sid}, room=workflow_id)
+
+
+@sio.on("collaboration_event")
+def handle_collaboration_event(sid, data):
+    """
+    Handle general collaboration events, include:
+    1. mouse_move
+    2. vars_and_features_update
+    3. sync_request (ask leader to update graph)
+    4. app_state_update
+    5. mcp_server_update
+    6. workflow_update
+    7. comments_update
+    8. node_panel_presence
+
+    """
+    mapping = redis_client.get(_sid_key(sid))
+
+    if not mapping:
+        return {"msg": "unauthorized"}, 401
+
+    mapping_data = json.loads(mapping)
+    workflow_id = mapping_data["workflow_id"]
+    user_id = mapping_data["user_id"]
+    _refresh_session_state(workflow_id, sid)
+
+    event_type = data.get("type")
+    event_data = data.get("data")
+    timestamp = data.get("timestamp", int(time.time()))
+
+    if not event_type:
+        return {"msg": "invalid event type"}, 400
+
+    sio.emit(
+        "collaboration_update",
+        {"type": event_type, "userId": user_id, "data": event_data, "timestamp": timestamp},
+        room=workflow_id,
+        skip_sid=sid,
+    )
+
+    return {"msg": "event_broadcasted"}
+
+
+@sio.on("graph_event")
+def handle_graph_event(sid, data):
+    """
+    Handle graph events - simple broadcast relay.
+    """
+    mapping = redis_client.get(_sid_key(sid))
+
+    if not mapping:
+        return {"msg": "unauthorized"}, 401
+
+    mapping_data = json.loads(mapping)
+    workflow_id = mapping_data["workflow_id"]
+    _refresh_session_state(workflow_id, sid)
+
+    sio.emit("graph_update", data, room=workflow_id, skip_sid=sid)
+
+    return {"msg": "graph_update_broadcasted"}

api/controllers/console/app/workflow.py

@@ -5,10 +5,12 @@ from typing import cast
 
 from flask import abort, request
 from flask_restx import Resource, fields, inputs, marshal_with, reqparse
+from pydantic_core import ValidationError
 from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
+from configs import dify_config
 from controllers.console import api, console_ns
 from controllers.console.app.error import ConversationCompletedError, DraftWorkflowNotExist, DraftWorkflowNotSync
 from controllers.console.app.wraps import get_app_model
@@ -21,7 +23,9 @@ from core.file.models import File
 from core.helper.trace_id_helper import get_external_trace_id
 from core.workflow.graph_engine.manager import GraphEngineManager
 from extensions.ext_database import db
+from extensions.ext_redis import redis_client
 from factories import file_factory, variable_factory
+from fields.online_user_fields import online_user_list_fields
 from fields.workflow_fields import workflow_fields, workflow_pagination_fields
 from fields.workflow_run_fields import workflow_run_node_execution_fields
 from libs import helper
@@ -99,6 +103,7 @@ class DraftWorkflowApi(Resource):
                 "hash": fields.String(description="Workflow hash for validation"),
                 "environment_variables": fields.List(fields.Raw, required=True, description="Environment variables"),
                 "conversation_variables": fields.List(fields.Raw, description="Conversation variables"),
+                "memory_blocks": fields.List(fields.Raw, description="Memory blocks"),
             },
         )
     )
@@ -133,6 +138,8 @@ class DraftWorkflowApi(Resource):
                 .add_argument("hash", type=str, required=False, location="json")
                 .add_argument("environment_variables", type=list, required=True, location="json")
                 .add_argument("conversation_variables", type=list, required=False, location="json")
+                .add_argument("force_upload", type=bool, required=False, default=False, location="json")
+                .add_argument("memory_blocks", type=list, required=False, location="json")
             )
             args = parser.parse_args()
         elif "text/plain" in content_type:
@@ -150,6 +157,8 @@ class DraftWorkflowApi(Resource):
                     "hash": data.get("hash"),
                     "environment_variables": data.get("environment_variables"),
                     "conversation_variables": data.get("conversation_variables"),
+                    "memory_blocks": data.get("memory_blocks"),
+                    "force_upload": data.get("force_upload", False),
                 }
             except json.JSONDecodeError:
                 return {"message": "Invalid JSON data"}, 400
@@ -166,6 +175,10 @@ class DraftWorkflowApi(Resource):
             conversation_variables = [
                 variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
             ]
+            memory_blocks_list = args.get("memory_blocks") or []
+            from core.memory.entities import MemoryBlockSpec
+
+            memory_blocks = [MemoryBlockSpec.model_validate(obj) for obj in memory_blocks_list]
             workflow = workflow_service.sync_draft_workflow(
                 app_model=app_model,
                 graph=args["graph"],
@@ -174,9 +187,13 @@ class DraftWorkflowApi(Resource):
                 account=current_user,
                 environment_variables=environment_variables,
                 conversation_variables=conversation_variables,
+                force_upload=args.get("force_upload", False),
+                memory_blocks=memory_blocks,
             )
         except WorkflowHashNotEqualError:
             raise DraftWorkflowNotSync()
+        except ValidationError as e:
+            return {"message": str(e)}, 400
 
         return {
             "result": "success",
@@ -745,6 +762,45 @@ class ConvertToWorkflowApi(Resource):
         }
 
 
+@console_ns.route("/apps/<uuid:app_id>/workflows/draft/config")
+class WorkflowConfigApi(Resource):
+    """Resource for workflow configuration."""
+
+    @api.doc("get_workflow_config")
+    @api.doc(description="Get workflow configuration")
+    @api.doc(params={"app_id": "Application ID"})
+    @api.response(200, "Workflow configuration retrieved successfully")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def get(self, app_model: App):
+        return {
+            "parallel_depth_limit": dify_config.WORKFLOW_PARALLEL_DEPTH_LIMIT,
+        }
+
+
+class WorkflowFeaturesApi(Resource):
+    """Update draft workflow features."""
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def post(self, app_model: App):
+        parser = reqparse.RequestParser()
+        parser.add_argument("features", type=dict, required=True, location="json")
+        args = parser.parse_args()
+
+        features = args.get("features")
+
+        # Update draft workflow features
+        workflow_service = WorkflowService()
+        workflow_service.update_draft_workflow_features(app_model=app_model, features=features, account=current_user)
+
+        return {"result": "success"}
+
+
 @console_ns.route("/apps/<uuid:app_id>/workflows")
 class PublishedAllWorkflowApi(Resource):
     @api.doc("get_all_published_workflows")
@@ -926,3 +982,105 @@ class DraftWorkflowNodeLastRunApi(Resource):
         if node_exec is None:
             raise NotFound("last run not found")
         return node_exec
+
+
+class WorkflowOnlineUsersApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(online_user_list_fields)
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("workflow_ids", type=str, required=True, location="args")
+        args = parser.parse_args()
+
+        workflow_ids = [id.strip() for id in args["workflow_ids"].split(",")]
+
+        results = []
+        for workflow_id in workflow_ids:
+            users_json = redis_client.hgetall(f"workflow_online_users:{workflow_id}")
+
+            users = []
+            for _, user_info_json in users_json.items():
+                try:
+                    users.append(json.loads(user_info_json))
+                except Exception:
+                    continue
+            results.append({"workflow_id": workflow_id, "users": users})
+
+        return {"data": results}
+
+
+api.add_resource(
+    DraftWorkflowApi,
+    "/apps/<uuid:app_id>/workflows/draft",
+)
+api.add_resource(
+    WorkflowConfigApi,
+    "/apps/<uuid:app_id>/workflows/draft/config",
+)
+api.add_resource(
+    WorkflowFeaturesApi,
+    "/apps/<uuid:app_id>/workflows/draft/features",
+)
+api.add_resource(
+    AdvancedChatDraftWorkflowRunApi,
+    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/run",
+)
+api.add_resource(
+    DraftWorkflowRunApi,
+    "/apps/<uuid:app_id>/workflows/draft/run",
+)
+api.add_resource(
+    WorkflowTaskStopApi,
+    "/apps/<uuid:app_id>/workflow-runs/tasks/<string:task_id>/stop",
+)
+api.add_resource(
+    DraftWorkflowNodeRunApi,
+    "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/run",
+)
+api.add_resource(
+    AdvancedChatDraftRunIterationNodeApi,
+    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/iteration/nodes/<string:node_id>/run",
+)
+api.add_resource(
+    WorkflowDraftRunIterationNodeApi,
+    "/apps/<uuid:app_id>/workflows/draft/iteration/nodes/<string:node_id>/run",
+)
+api.add_resource(
+    AdvancedChatDraftRunLoopNodeApi,
+    "/apps/<uuid:app_id>/advanced-chat/workflows/draft/loop/nodes/<string:node_id>/run",
+)
+api.add_resource(
+    WorkflowDraftRunLoopNodeApi,
+    "/apps/<uuid:app_id>/workflows/draft/loop/nodes/<string:node_id>/run",
+)
+api.add_resource(
+    PublishedWorkflowApi,
+    "/apps/<uuid:app_id>/workflows/publish",
+)
+api.add_resource(
+    PublishedAllWorkflowApi,
+    "/apps/<uuid:app_id>/workflows",
+)
+api.add_resource(
+    DefaultBlockConfigsApi,
+    "/apps/<uuid:app_id>/workflows/default-workflow-block-configs",
+)
+api.add_resource(
+    DefaultBlockConfigApi,
+    "/apps/<uuid:app_id>/workflows/default-workflow-block-configs/<string:block_type>",
+)
+api.add_resource(
+    ConvertToWorkflowApi,
+    "/apps/<uuid:app_id>/convert-to-workflow",
+)
+api.add_resource(
+    WorkflowByIdApi,
+    "/apps/<uuid:app_id>/workflows/<string:workflow_id>",
+)
+api.add_resource(
+    DraftWorkflowNodeLastRunApi,
+    "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/last-run",
+)
+api.add_resource(WorkflowOnlineUsersApi, "/apps/workflows/online-users")

api/controllers/console/app/workflow_comment.py

@@ -0,0 +1,240 @@
+import logging
+
+from flask_restx import Resource, fields, marshal_with, reqparse
+
+from controllers.console import api
+from controllers.console.app.wraps import get_app_model
+from controllers.console.wraps import account_initialization_required, setup_required
+from fields.member_fields import account_with_role_fields
+from fields.workflow_comment_fields import (
+    workflow_comment_basic_fields,
+    workflow_comment_create_fields,
+    workflow_comment_detail_fields,
+    workflow_comment_reply_create_fields,
+    workflow_comment_reply_update_fields,
+    workflow_comment_resolve_fields,
+    workflow_comment_update_fields,
+)
+from libs.login import current_user, login_required
+from models import App
+from services.account_service import TenantService
+from services.workflow_comment_service import WorkflowCommentService
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowCommentListApi(Resource):
+    """API for listing and creating workflow comments."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_basic_fields, envelope="data")
+    def get(self, app_model: App):
+        """Get all comments for a workflow."""
+        comments = WorkflowCommentService.get_comments(tenant_id=current_user.current_tenant_id, app_id=app_model.id)
+
+        return comments
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_create_fields)
+    def post(self, app_model: App):
+        """Create a new workflow comment."""
+        parser = reqparse.RequestParser()
+        parser.add_argument("position_x", type=float, required=True, location="json")
+        parser.add_argument("position_y", type=float, required=True, location="json")
+        parser.add_argument("content", type=str, required=True, location="json")
+        parser.add_argument("mentioned_user_ids", type=list, location="json", default=[])
+        args = parser.parse_args()
+
+        result = WorkflowCommentService.create_comment(
+            tenant_id=current_user.current_tenant_id,
+            app_id=app_model.id,
+            created_by=current_user.id,
+            content=args.content,
+            position_x=args.position_x,
+            position_y=args.position_y,
+            mentioned_user_ids=args.mentioned_user_ids,
+        )
+
+        return result, 201
+
+
+class WorkflowCommentDetailApi(Resource):
+    """API for managing individual workflow comments."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_detail_fields)
+    def get(self, app_model: App, comment_id: str):
+        """Get a specific workflow comment."""
+        comment = WorkflowCommentService.get_comment(
+            tenant_id=current_user.current_tenant_id, app_id=app_model.id, comment_id=comment_id
+        )
+
+        return comment
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_update_fields)
+    def put(self, app_model: App, comment_id: str):
+        """Update a workflow comment."""
+        parser = reqparse.RequestParser()
+        parser.add_argument("content", type=str, required=True, location="json")
+        parser.add_argument("position_x", type=float, required=False, location="json")
+        parser.add_argument("position_y", type=float, required=False, location="json")
+        parser.add_argument("mentioned_user_ids", type=list, location="json", default=[])
+        args = parser.parse_args()
+
+        result = WorkflowCommentService.update_comment(
+            tenant_id=current_user.current_tenant_id,
+            app_id=app_model.id,
+            comment_id=comment_id,
+            user_id=current_user.id,
+            content=args.content,
+            position_x=args.position_x,
+            position_y=args.position_y,
+            mentioned_user_ids=args.mentioned_user_ids,
+        )
+
+        return result
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    def delete(self, app_model: App, comment_id: str):
+        """Delete a workflow comment."""
+        WorkflowCommentService.delete_comment(
+            tenant_id=current_user.current_tenant_id,
+            app_id=app_model.id,
+            comment_id=comment_id,
+            user_id=current_user.id,
+        )
+
+        return {"result": "success"}, 204
+
+
+class WorkflowCommentResolveApi(Resource):
+    """API for resolving and reopening workflow comments."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_resolve_fields)
+    def post(self, app_model: App, comment_id: str):
+        """Resolve a workflow comment."""
+        comment = WorkflowCommentService.resolve_comment(
+            tenant_id=current_user.current_tenant_id,
+            app_id=app_model.id,
+            comment_id=comment_id,
+            user_id=current_user.id,
+        )
+
+        return comment
+
+
+class WorkflowCommentReplyApi(Resource):
+    """API for managing comment replies."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_reply_create_fields)
+    def post(self, app_model: App, comment_id: str):
+        """Add a reply to a workflow comment."""
+        # Validate comment access first
+        WorkflowCommentService.validate_comment_access(
+            comment_id=comment_id, tenant_id=current_user.current_tenant_id, app_id=app_model.id
+        )
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("content", type=str, required=True, location="json")
+        parser.add_argument("mentioned_user_ids", type=list, location="json", default=[])
+        args = parser.parse_args()
+
+        result = WorkflowCommentService.create_reply(
+            comment_id=comment_id,
+            content=args.content,
+            created_by=current_user.id,
+            mentioned_user_ids=args.mentioned_user_ids,
+        )
+
+        return result, 201
+
+
+class WorkflowCommentReplyDetailApi(Resource):
+    """API for managing individual comment replies."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with(workflow_comment_reply_update_fields)
+    def put(self, app_model: App, comment_id: str, reply_id: str):
+        """Update a comment reply."""
+        # Validate comment access first
+        WorkflowCommentService.validate_comment_access(
+            comment_id=comment_id, tenant_id=current_user.current_tenant_id, app_id=app_model.id
+        )
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("content", type=str, required=True, location="json")
+        parser.add_argument("mentioned_user_ids", type=list, location="json", default=[])
+        args = parser.parse_args()
+
+        reply = WorkflowCommentService.update_reply(
+            reply_id=reply_id, user_id=current_user.id, content=args.content, mentioned_user_ids=args.mentioned_user_ids
+        )
+
+        return reply
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    def delete(self, app_model: App, comment_id: str, reply_id: str):
+        """Delete a comment reply."""
+        # Validate comment access first
+        WorkflowCommentService.validate_comment_access(
+            comment_id=comment_id, tenant_id=current_user.current_tenant_id, app_id=app_model.id
+        )
+
+        WorkflowCommentService.delete_reply(reply_id=reply_id, user_id=current_user.id)
+
+        return {"result": "success"}, 204
+
+
+class WorkflowCommentMentionUsersApi(Resource):
+    """API for getting mentionable users for workflow comments."""
+
+    @login_required
+    @setup_required
+    @account_initialization_required
+    @get_app_model
+    @marshal_with({"users": fields.List(fields.Nested(account_with_role_fields))})
+    def get(self, app_model: App):
+        """Get all users in current tenant for mentions."""
+        members = TenantService.get_tenant_members(current_user.current_tenant)
+        return {"users": members}
+
+
+# Register API routes
+api.add_resource(WorkflowCommentListApi, "/apps/<uuid:app_id>/workflow/comments")
+api.add_resource(WorkflowCommentDetailApi, "/apps/<uuid:app_id>/workflow/comments/<string:comment_id>")
+api.add_resource(WorkflowCommentResolveApi, "/apps/<uuid:app_id>/workflow/comments/<string:comment_id>/resolve")
+api.add_resource(WorkflowCommentReplyApi, "/apps/<uuid:app_id>/workflow/comments/<string:comment_id>/replies")
+api.add_resource(
+    WorkflowCommentReplyDetailApi, "/apps/<uuid:app_id>/workflow/comments/<string:comment_id>/replies/<string:reply_id>"
+)
+api.add_resource(WorkflowCommentMentionUsersApi, "/apps/<uuid:app_id>/workflow/comments/mention-users")

api/controllers/console/app/workflow_draft_variable.py

@@ -19,8 +19,8 @@ from core.variables.segments import ArrayFileSegment, FileSegment, Segment
 from core.variables.types import SegmentType
 from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
 from extensions.ext_database import db
+from factories import variable_factory
 from factories.file_factory import build_from_mapping, build_from_mappings
-from factories.variable_factory import build_segment_with_type
 from libs.login import current_user, login_required
 from models import Account, App, AppMode
 from models.workflow import WorkflowDraftVariable
@@ -355,7 +355,7 @@ class VariableApi(Resource):
                 if len(raw_value) > 0 and not isinstance(raw_value[0], dict):
                     raise InvalidArgumentError(description=f"expected dict for files[0], got {type(raw_value)}")
                 raw_value = build_from_mappings(mappings=raw_value, tenant_id=app_model.tenant_id)
-            new_value = build_segment_with_type(variable.value_type, raw_value)
+            new_value = variable_factory.build_segment_with_type(variable.value_type, raw_value)
         draft_var_srv.update_variable(variable, name=new_name, value=new_value)
         db.session.commit()
         return variable
@@ -448,8 +448,35 @@ class ConversationVariableCollectionApi(Resource):
         db.session.commit()
         return _get_variable_list(app_model, CONVERSATION_VARIABLE_NODE_ID)
 
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=AppMode.ADVANCED_CHAT)
+    def post(self, app_model: App):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("conversation_variables", type=list, required=True, location="json")
+        args = parser.parse_args()
+
+        workflow_service = WorkflowService()
+
+        conversation_variables_list = args.get("conversation_variables") or []
+        conversation_variables = [
+            variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
+        ]
+
+        workflow_service.update_draft_workflow_conversation_variables(
+            app_model=app_model,
+            account=current_user,
+            conversation_variables=conversation_variables,
+        )
+
+        return {"result": "success"}
+
 
-@console_ns.route("/apps/<uuid:app_id>/workflows/draft/system-variables")
 class SystemVariableCollectionApi(Resource):
     @api.doc("get_system_variables")
     @api.doc(description="Get system variables for workflow")
@@ -499,3 +526,44 @@ class EnvironmentVariableCollectionApi(Resource):
             )
 
         return {"items": env_vars_list}
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @get_app_model(mode=[AppMode.ADVANCED_CHAT, AppMode.WORKFLOW])
+    def post(self, app_model: App):
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("environment_variables", type=list, required=True, location="json")
+        args = parser.parse_args()
+
+        workflow_service = WorkflowService()
+
+        environment_variables_list = args.get("environment_variables") or []
+        environment_variables = [
+            variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
+        ]
+
+        workflow_service.update_draft_workflow_environment_variables(
+            app_model=app_model,
+            account=current_user,
+            environment_variables=environment_variables,
+        )
+
+        return {"result": "success"}
+
+
+api.add_resource(
+    WorkflowVariableCollectionApi,
+    "/apps/<uuid:app_id>/workflows/draft/variables",
+)
+api.add_resource(NodeVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/variables")
+api.add_resource(VariableApi, "/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>")
+api.add_resource(VariableResetApi, "/apps/<uuid:app_id>/workflows/draft/variables/<uuid:variable_id>/reset")
+
+api.add_resource(ConversationVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/conversation-variables")
+api.add_resource(SystemVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/system-variables")
+api.add_resource(EnvironmentVariableCollectionApi, "/apps/<uuid:app_id>/workflows/draft/environment-variables")

api/controllers/console/app/wraps.py

@@ -23,6 +23,15 @@ def _load_app_model(app_id: str) -> App | None:
     return app_model
 
 
+def _load_app_model_with_trial(app_id: str) -> App | None:
+    app_model = (
+        db.session.query(App)
+        .where(App.id == app_id, App.status == "normal")
+        .first()
+    )
+    return app_model
+
+
 def get_app_model(view: Callable[P, R] | None = None, *, mode: Union[AppMode, list[AppMode], None] = None):
     def decorator(view_func: Callable[P1, R1]):
         @wraps(view_func)
@@ -62,3 +71,44 @@ def get_app_model(view: Callable[P, R] | None = None, *, mode: Union[AppMode, li
         return decorator
     else:
         return decorator(view)
+
+
+def get_app_model_with_trial(view: Callable[P, R] | None = None, *, mode: Union[AppMode, list[AppMode], None] = None):
+    def decorator(view_func: Callable[P, R]):
+        @wraps(view_func)
+        def decorated_view(*args: P.args, **kwargs: P.kwargs):
+            if not kwargs.get("app_id"):
+                raise ValueError("missing app_id in path parameters")
+
+            app_id = kwargs.get("app_id")
+            app_id = str(app_id)
+
+            del kwargs["app_id"]
+
+            app_model = _load_app_model_with_trial(app_id)
+
+            if not app_model:
+                raise AppNotFoundError()
+
+            app_mode = AppMode.value_of(app_model.mode)
+
+            if mode is not None:
+                if isinstance(mode, list):
+                    modes = mode
+                else:
+                    modes = [mode]
+
+                if app_mode not in modes:
+                    mode_values = {m.value for m in modes}
+                    raise AppNotFoundError(f"App mode is not in the supported list: {mode_values}")
+
+            kwargs["app_model"] = app_model
+
+            return view_func(*args, **kwargs)
+
+        return decorated_view
+
+    if view is None:
+        return decorator
+    else:
+        return decorator(view)

api/controllers/console/auth/login.py

@@ -25,10 +25,13 @@ from controllers.console.wraps import email_password_login_enabled, setup_requir
 from events.tenant_event import tenant_was_created
 from libs.helper import email, extract_remote_ip
 from libs.login import current_account_with_tenant
+from libs.passport import PassportService
 from libs.token import (
+    check_csrf_token,
     clear_access_token_from_cookie,
     clear_csrf_token_from_cookie,
     clear_refresh_token_from_cookie,
+    extract_access_token,
     extract_refresh_token,
     set_access_token_to_cookie,
     set_csrf_token_to_cookie,
@@ -289,3 +292,18 @@ class RefreshTokenApi(Resource):
             return response
         except Exception as e:
             return {"result": "fail", "message": str(e)}, 401
+
+
+# this api helps frontend to check whether user is authenticated
+# TODO: remove in the future. frontend should redirect to login page by catching 401 status
+@console_ns.route("/login/status")
+class LoginStatus(Resource):
+    def get(self):
+        token = extract_access_token(request)
+        res = True
+        try:
+            validated = PassportService().verify(token=token)
+            check_csrf_token(request=request, user_id=validated.get("user_id"))
+        except:
+            res = False
+        return {"logged_in": res}

api/controllers/console/explore/banner.py

@@ -0,0 +1,43 @@
+from flask import request
+from flask_restx import Resource
+
+from controllers.console import api
+from controllers.console.explore.wraps import explore_banner_enabled
+from extensions.ext_database import db
+from models.model import ExporleBanner
+
+
+class BannerApi(Resource):
+    """Resource for banner list."""
+
+    @explore_banner_enabled
+    def get(self):
+        """Get banner list."""
+        language = request.args.get("language", "en-US")
+
+        # Build base query for enabled banners
+        base_query = db.session.query(ExporleBanner).where(ExporleBanner.status == "enabled")
+        
+        # Try to get banners in the requested language
+        banners = base_query.where(ExporleBanner.language == language).order_by(ExporleBanner.sort).all()
+        
+        # Fallback to en-US if no banners found and language is not en-US
+        if not banners and language != "en-US":
+            banners = base_query.where(ExporleBanner.language == "en-US").order_by(ExporleBanner.sort).all()
+        # Convert banners to serializable format
+        result = []
+        for banner in banners:
+            banner_data = {
+                "id": banner.id,
+                "content": banner.content,  # Already parsed as JSON by SQLAlchemy
+                "link": banner.link,
+                "sort": banner.sort,
+                "status": banner.status,
+                "created_at": banner.created_at.isoformat() if banner.created_at else None,
+            }
+            result.append(banner_data)
+
+        return result
+
+
+api.add_resource(BannerApi, "/explore/banners")

api/controllers/console/explore/error.py

@@ -29,3 +29,25 @@ class AppAccessDeniedError(BaseHTTPException):
     error_code = "access_denied"
     description = "App access denied."
     code = 403
+
+
+class TrialAppNotAllowed(BaseHTTPException):
+    """*403* `Trial App Not Allowed`
+
+    Raise if the user has reached the trial app limit.
+    """
+
+    error_code = "trial_app_not_allowed"
+    code = 403
+    description = "the app is not allowed to be trial."
+
+
+class TrialAppLimitExceeded(BaseHTTPException):
+    """*403* `Trial App Limit Exceeded`
+
+    Raise if the user has exceeded the trial app limit.
+    """
+
+    error_code = "trial_app_limit_exceeded"
+    code = 403
+    description = "The user has exceeded the trial app limit."

api/controllers/console/explore/recommended_app.py

@@ -27,6 +27,7 @@ recommended_app_fields = {
     "category": fields.String,
     "position": fields.Integer,
     "is_listed": fields.Boolean,
+    "can_trial": fields.Boolean,
 }
 
 recommended_app_list_fields = {

api/controllers/console/explore/trial.py

@@ -0,0 +1,514 @@
+import logging
+from typing import Any, cast
+
+from flask import request
+from flask_restx import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
+import services
+from controllers.common import fields
+from controllers.common.fields import build_site_model
+from controllers.console import api
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    NeedAddIdsError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
+from controllers.console.app.wraps import get_app_model_with_trial
+from controllers.console.explore.error import (
+    AppSuggestedQuestionsAfterAnswerDisabledError,
+    NotChatAppError,
+    NotCompletionAppError,
+    NotWorkflowAppError,
+)
+from controllers.console.explore.wraps import TrialAppResource, trial_feature_enable
+from controllers.service_api import service_api_ns
+from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
+from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
+from core.app.apps.base_app_queue_manager import AppQueueManager
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.errors.error import (
+    ModelCurrentlyNotSupportError,
+    ProviderTokenNotInitError,
+    QuotaExceededError,
+)
+from core.model_runtime.errors.invoke import InvokeError
+from core.workflow.graph_engine.manager import GraphEngineManager
+from extensions.ext_database import db
+from fields.app_fields import app_detail_fields_with_site
+from fields.dataset_fields import dataset_fields
+from fields.workflow_fields import workflow_fields
+from libs import helper
+from libs.helper import uuid_value
+from libs.login import current_user
+from models import Account
+from models.account import TenantStatus
+from models.model import AppMode, Site
+from models.workflow import Workflow
+from services.app_generate_service import AppGenerateService
+from services.app_service import AppService
+from services.audio_service import AudioService
+from services.dataset_service import DatasetService
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
+from services.errors.conversation import ConversationNotExistsError
+from services.errors.llm import InvokeRateLimitError
+from services.errors.message import (
+    MessageNotExistsError,
+    SuggestedQuestionsAfterAnswerDisabledError,
+)
+from services.message_service import MessageService
+from services.recommended_app_service import RecommendedAppService
+
+logger = logging.getLogger(__name__)
+
+
+class TrialAppWorkflowRunApi(TrialAppResource):
+    def post(self, trial_app):
+        """
+        Run workflow
+        """
+        app_model = trial_app
+        if not app_model:
+            raise NotWorkflowAppError()
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode != AppMode.WORKFLOW:
+            raise NotWorkflowAppError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
+        parser.add_argument("files", type=list, required=False, location="json")
+        args = parser.parse_args()
+        assert current_user is not None
+        try:
+            app_id = app_model.id
+            user_id = current_user.id
+            response = AppGenerateService.generate(
+                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.EXPLORE, streaming=True
+            )
+            RecommendedAppService.add_trial_app_record(app_id, user_id)
+            return helper.compact_generate_response(response)
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except InvokeRateLimitError as ex:
+            raise InvokeRateLimitHttpError(ex.description)
+        except ValueError as e:
+            raise e
+        except Exception:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+
+class TrialAppWorkflowTaskStopApi(TrialAppResource):
+    def post(self, trial_app, task_id: str):
+        """
+        Stop workflow task
+        """
+        app_model = trial_app
+        if not app_model:
+            raise NotWorkflowAppError()
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode != AppMode.WORKFLOW:
+            raise NotWorkflowAppError()
+        assert current_user is not None
+
+        # Stop using both mechanisms for backward compatibility
+        # Legacy stop flag mechanism (without user check)
+        AppQueueManager.set_stop_flag_no_user_check(task_id)
+
+        # New graph engine command channel mechanism
+        GraphEngineManager.send_stop_command(task_id)
+
+        return {"result": "success"}
+
+
+class TrialChatApi(TrialAppResource):
+    @trial_feature_enable
+    def post(self, trial_app):
+        app_model = trial_app
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
+            raise NotChatAppError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("inputs", type=dict, required=True, location="json")
+        parser.add_argument("query", type=str, required=True, location="json")
+        parser.add_argument("files", type=list, required=False, location="json")
+        parser.add_argument("conversation_id", type=uuid_value, location="json")
+        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
+        parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
+        args = parser.parse_args()
+
+        args["auto_generate_name"] = False
+
+        try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
+            
+            # Get IDs before they might be detached from session
+            app_id = app_model.id
+            user_id = current_user.id
+            
+            response = AppGenerateService.generate(
+                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.EXPLORE, streaming=True
+            )
+            RecommendedAppService.add_trial_app_record(app_id, user_id)
+            return helper.compact_generate_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logger.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except InvokeRateLimitError as ex:
+            raise InvokeRateLimitHttpError(ex.description)
+        except ValueError as e:
+            raise e
+        except Exception:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+
+class TrialMessageSuggestedQuestionApi(TrialAppResource):
+    @trial_feature_enable
+    def get(self, trial_app, message_id):
+        app_model = trial_app
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
+            raise NotChatAppError()
+
+        message_id = str(message_id)
+
+        try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
+            questions = MessageService.get_suggested_questions_after_answer(
+                app_model=app_model, user=current_user, message_id=message_id, invoke_from=InvokeFrom.EXPLORE
+            )
+        except MessageNotExistsError:
+            raise NotFound("Message not found")
+        except ConversationNotExistsError:
+            raise NotFound("Conversation not found")
+        except SuggestedQuestionsAfterAnswerDisabledError:
+            raise AppSuggestedQuestionsAfterAnswerDisabledError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except Exception:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+        return {"data": questions}
+
+
+class TrialChatAudioApi(TrialAppResource):
+    @trial_feature_enable
+    def post(self, trial_app):
+        app_model = trial_app
+
+        file = request.files["file"]
+
+        try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
+            
+            # Get IDs before they might be detached from session
+            app_id = app_model.id
+            user_id = current_user.id
+            
+            response = AudioService.transcript_asr(app_model=app_model, file=file, end_user=None)
+            RecommendedAppService.add_trial_app_record(app_id, user_id)
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logger.exception("App model config broken.")
+            raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+
+class TrialChatTextApi(TrialAppResource):
+    @trial_feature_enable
+    def post(self, trial_app):
+        app_model = trial_app
+        try:
+            parser = reqparse.RequestParser()
+            parser.add_argument("message_id", type=str, required=False, location="json")
+            parser.add_argument("voice", type=str, location="json")
+            parser.add_argument("text", type=str, location="json")
+            parser.add_argument("streaming", type=bool, location="json")
+            args = parser.parse_args()
+
+            message_id = args.get("message_id", None)
+            text = args.get("text", None)
+            voice = args.get("voice", None)
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
+            
+            # Get IDs before they might be detached from session
+            app_id = app_model.id
+            user_id = current_user.id
+            
+            response = AudioService.transcript_tts(app_model=app_model, text=text, voice=voice, message_id=message_id)
+            RecommendedAppService.add_trial_app_record(app_id, user_id)
+            return response
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logger.exception("App model config broken.")
+            raise AppUnavailableError()
+        except NoAudioUploadedServiceError:
+            raise NoAudioUploadedError()
+        except AudioTooLargeServiceError as e:
+            raise AudioTooLargeError(str(e))
+        except UnsupportedAudioTypeServiceError:
+            raise UnsupportedAudioTypeError()
+        except ProviderNotSupportSpeechToTextServiceError:
+            raise ProviderNotSupportSpeechToTextError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception as e:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+
+class TrialCompletionApi(TrialAppResource):
+    @trial_feature_enable
+    def post(self, trial_app):
+        app_model = trial_app
+        if app_model.mode != "completion":
+            raise NotCompletionAppError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("inputs", type=dict, required=True, location="json")
+        parser.add_argument("query", type=str, location="json", default="")
+        parser.add_argument("files", type=list, required=False, location="json")
+        parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
+        parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
+        args = parser.parse_args()
+
+        streaming = args["response_mode"] == "streaming"
+        args["auto_generate_name"] = False
+
+        try:
+            if not isinstance(current_user, Account):
+                raise ValueError("current_user must be an Account instance")
+            
+            # Get IDs before they might be detached from session
+            app_id = app_model.id
+            user_id = current_user.id
+            
+            response = AppGenerateService.generate(
+                app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.EXPLORE, streaming=streaming
+            )
+
+            RecommendedAppService.add_trial_app_record(app_id, user_id)
+            return helper.compact_generate_response(response)
+        except services.errors.conversation.ConversationNotExistsError:
+            raise NotFound("Conversation Not Exists.")
+        except services.errors.conversation.ConversationCompletedError:
+            raise ConversationCompletedError()
+        except services.errors.app_model_config.AppModelConfigBrokenError:
+            logger.exception("App model config broken.")
+            raise AppUnavailableError()
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception:
+            logger.exception("internal server error.")
+            raise InternalServerError()
+
+
+class TrialSitApi(Resource):
+    """Resource for trial app sites."""
+
+    @trial_feature_enable
+    @get_app_model_with_trial
+    @service_api_ns.marshal_with(build_site_model(service_api_ns))
+    def get(self, app_model):
+        """Retrieve app site info.
+
+        Returns the site configuration for the application including theme, icons, and text.
+        """
+        site = db.session.query(Site).where(Site.app_id == app_model.id).first()
+
+        if not site:
+            raise Forbidden()
+
+        assert app_model.tenant
+        if app_model.tenant.status == TenantStatus.ARCHIVE:
+            raise Forbidden()
+
+        return site
+
+
+class TrialAppParameterApi(Resource):
+    """Resource for app variables."""
+
+    @trial_feature_enable
+    @get_app_model_with_trial
+    @marshal_with(fields.parameters_fields)
+    def get(self, app_model):
+        """Retrieve app parameters."""
+
+        if app_model is None:
+            raise AppUnavailableError()
+
+        if app_model.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
+            workflow = app_model.workflow
+            if workflow is None:
+                raise AppUnavailableError()
+
+            features_dict = workflow.features_dict
+            user_input_form = workflow.user_input_form(to_old_structure=True)
+        else:
+            app_model_config = app_model.app_model_config
+            if app_model_config is None:
+                raise AppUnavailableError()
+
+            features_dict = app_model_config.to_dict()
+
+            user_input_form = features_dict.get("user_input_form", [])
+
+        return get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
+
+
+class AppApi(Resource):
+    @trial_feature_enable
+    @get_app_model_with_trial
+    @marshal_with(app_detail_fields_with_site)
+    def get(self, app_model):
+        """Get app detail"""
+
+        app_service = AppService()
+        app_model = app_service.get_app(app_model)
+
+        return app_model
+
+
+class AppWorkflowApi(Resource):
+    @trial_feature_enable
+    @get_app_model_with_trial
+    @marshal_with(workflow_fields)
+    def get(self, app_model):
+        """Get workflow detail"""
+        if not app_model.workflow_id:
+            raise AppUnavailableError()
+
+        workflow = (
+            db.session.query(Workflow)
+            .where(
+                Workflow.id == app_model.workflow_id,
+            )
+            .first()
+        )
+        return workflow
+
+
+class DatasetListApi(Resource):
+    @trial_feature_enable
+    @get_app_model_with_trial
+    def get(self, app_model):
+        page = request.args.get("page", default=1, type=int)
+        limit = request.args.get("limit", default=20, type=int)
+        ids = request.args.getlist("ids")
+      
+        tenant_id = app_model.tenant_id
+        if ids:
+            datasets, total = DatasetService.get_datasets_by_ids(ids, tenant_id)
+        else:
+            raise NeedAddIdsError()
+
+        data = cast(list[dict[str, Any]], marshal(datasets, dataset_fields))
+
+        response = {"data": data, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page}
+        return response
+
+
+api.add_resource(TrialChatApi, "/trial-apps/<uuid:app_id>/chat-messages", endpoint="trial_app_chat_completion")
+
+api.add_resource(
+    TrialMessageSuggestedQuestionApi,
+    "/trial-apps/<uuid:app_id>/messages/<uuid:message_id>/suggested-questions",
+    endpoint="trial_app_suggested_question",
+)
+
+api.add_resource(TrialChatAudioApi, "/trial-apps/<uuid:app_id>/audio-to-text", endpoint="trial_app_audio")
+api.add_resource(TrialChatTextApi, "/trial-apps/<uuid:app_id>/text-to-audio", endpoint="trial_app_text")
+
+api.add_resource(TrialCompletionApi, "/trial-apps/<uuid:app_id>/completion-messages", endpoint="trial_app_completion")
+
+api.add_resource(TrialSitApi, "/trial-apps/<uuid:app_id>/site")
+
+api.add_resource(TrialAppParameterApi, "/trial-apps/<uuid:app_id>/parameters", endpoint="trial_app_parameters")
+
+api.add_resource(AppApi, "/trial-apps/<uuid:app_id>", endpoint="trial_app")
+
+api.add_resource(TrialAppWorkflowRunApi, "/trial-apps/<uuid:app_id>/workflows/run", endpoint="trial_app_workflow_run")
+api.add_resource(TrialAppWorkflowTaskStopApi, "/trial-apps/<uuid:app_id>/workflows/tasks/<string:task_id>/stop")
+
+api.add_resource(AppWorkflowApi, "/trial-apps/<uuid:app_id>/workflows", endpoint="trial_app_workflow")
+api.add_resource(DatasetListApi, "/trial-apps/<uuid:app_id>/datasets", endpoint="trial_app_datasets")

api/controllers/console/explore/wraps.py

@@ -2,14 +2,16 @@ from collections.abc import Callable
 from functools import wraps
 from typing import Concatenate, ParamSpec, TypeVar
 
+from flask import abort
+from flask_login import current_user
 from flask_restx import Resource
 from werkzeug.exceptions import NotFound
 
-from controllers.console.explore.error import AppAccessDeniedError
+from controllers.console.explore.error import AppAccessDeniedError, TrialAppLimitExceeded, TrialAppNotAllowed
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from libs.login import current_account_with_tenant, login_required
-from models import InstalledApp
+from models import AccountTrialAppRecord, App, InstalledApp, TrialApp
 from services.enterprise.enterprise_service import EnterpriseService
 from services.feature_service import FeatureService
 
@@ -71,6 +73,59 @@ def user_allowed_to_access_app(view: Callable[Concatenate[InstalledApp, P], R] |
     return decorator
 
 
+def trial_app_required(view: Callable[Concatenate[App, P], R] | None = None):
+    def decorator(view: Callable[Concatenate[App, P], R]):
+        @wraps(view)
+        def decorated(app_id: str, *args: P.args, **kwargs: P.kwargs):
+            trial_app = db.session.query(TrialApp).where(TrialApp.app_id == str(app_id)).first()
+
+            if trial_app is None:
+                raise TrialAppNotAllowed()
+            app = trial_app.app
+
+            if app is None:
+                raise TrialAppNotAllowed()
+
+            account_trial_app_record = (
+                db.session.query(AccountTrialAppRecord)
+                .where(AccountTrialAppRecord.account_id == current_user.id, AccountTrialAppRecord.app_id == app_id)
+                .first()
+            )
+            if account_trial_app_record:
+                if account_trial_app_record.count >= trial_app.trial_limit:
+                    raise TrialAppLimitExceeded()
+
+            return view(app, *args, **kwargs)
+
+        return decorated
+
+    if view:
+        return decorator(view)
+    return decorator
+
+
+def trial_feature_enable(view: Callable[..., R]) -> Callable[..., R]:
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        features = FeatureService.get_system_features()
+        if not features.enable_trial_app:
+            abort(403, "Trial app feature is not enabled.")
+        return view(*args, **kwargs)
+
+    return decorated
+
+
+def explore_banner_enabled(view: Callable[..., R]) -> Callable[..., R]:
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        features = FeatureService.get_system_features()
+        if not features.enable_explore_banner:
+            abort(403, "Explore banner feature is not enabled.")
+        return view(*args, **kwargs)
+
+    return decorated
+
+
 class InstalledAppResource(Resource):
     # must be reversed if there are multiple decorators
 
@@ -80,3 +135,13 @@ class InstalledAppResource(Resource):
         account_initialization_required,
         login_required,
     ]
+
+
+class TrialAppResource(Resource):
+    # must be reversed if there are multiple decorators
+
+    method_decorators = [
+        trial_app_required,
+        account_initialization_required,
+        login_required,
+    ]

api/controllers/console/workspace/account.py

@@ -32,6 +32,7 @@ from controllers.console.wraps import (
     only_edition_cloud,
     setup_required,
 )
+from core.file import helpers as file_helpers
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.datetime_utils import naive_utc_now
@@ -128,6 +129,17 @@ class AccountNameApi(Resource):
 
 @console_ns.route("/account/avatar")
 class AccountAvatarApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("avatar", type=str, required=True, location="args")
+        args = parser.parse_args()
+
+        avatar_url = file_helpers.get_signed_file_url(args["avatar"])
+        return {"avatar_url": avatar_url}
+
     @setup_required
     @login_required
     @account_initialization_required

api/controllers/console/workspace/workspace.py

@@ -51,6 +51,9 @@ tenant_fields = {
     "in_trial": fields.Boolean,
     "trial_end_reason": fields.String,
     "custom_config": fields.Raw(attribute="custom_config"),
+    "trial_credits": fields.Integer,
+    "trial_credits_used": fields.Integer,
+    "next_credit_reset_date": fields.Integer,
 }
 
 tenants_fields = {

api/controllers/service_api/__init__.py

@@ -19,6 +19,7 @@ from .app import (
     annotation,
     app,
     audio,
+    chatflow_memory,
     completion,
     conversation,
     file,
@@ -40,6 +41,7 @@ __all__ = [
     "annotation",
     "app",
     "audio",
+    "chatflow_memory",
     "completion",
     "conversation",
     "dataset",

api/controllers/service_api/app/chatflow_memory.py

@@ -0,0 +1,109 @@
+from flask_restx import Resource, reqparse
+
+from controllers.service_api import api
+from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
+from core.memory.entities import MemoryBlock, MemoryCreatedBy
+from core.workflow.runtime.variable_pool import VariablePool
+from models import App, EndUser
+from services.chatflow_memory_service import ChatflowMemoryService
+from services.workflow_service import WorkflowService
+
+
+class MemoryListApi(Resource):
+    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
+    def get(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("conversation_id", required=False, type=str | None, default=None)
+        parser.add_argument("memory_id", required=False, type=str | None, default=None)
+        parser.add_argument("version", required=False, type=int | None, default=None)
+        args = parser.parse_args()
+        conversation_id: str | None = args.get("conversation_id")
+        memory_id = args.get("memory_id")
+        version = args.get("version")
+
+        if conversation_id:
+            result = ChatflowMemoryService.get_persistent_memories_with_conversation(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), conversation_id, version
+            )
+            session_memories = ChatflowMemoryService.get_session_memories_with_conversation(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), conversation_id, version
+            )
+            result = [*result, *session_memories]
+        else:
+            result = ChatflowMemoryService.get_persistent_memories(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), version
+            )
+
+        if memory_id:
+            result = [it for it in result if it.spec.id == memory_id]
+        return [it for it in result if it.spec.end_user_visible]
+
+
+class MemoryEditApi(Resource):
+    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
+    def put(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("id", type=str, required=True)
+        parser.add_argument("conversation_id", type=str | None, required=False, default=None)
+        parser.add_argument("node_id", type=str | None, required=False, default=None)
+        parser.add_argument("update", type=str, required=True)
+        args = parser.parse_args()
+        workflow = WorkflowService().get_published_workflow(app_model)
+        update = args.get("update")
+        conversation_id = args.get("conversation_id")
+        node_id = args.get("node_id")
+        if not isinstance(update, str):
+            return {"error": "Invalid update"}, 400
+        if not workflow:
+            return {"error": "Workflow not found"}, 404
+        memory_spec = next((it for it in workflow.memory_blocks if it.id == args["id"]), None)
+        if not memory_spec:
+            return {"error": "Memory not found"}, 404
+
+        # First get existing memory
+        existing_memory = ChatflowMemoryService.get_memory_by_spec(
+            spec=memory_spec,
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            created_by=MemoryCreatedBy(end_user_id=end_user.id),
+            conversation_id=conversation_id,
+            node_id=node_id,
+            is_draft=False,
+        )
+
+        # Create updated memory instance with incremented version
+        updated_memory = MemoryBlock(
+            spec=existing_memory.spec,
+            tenant_id=existing_memory.tenant_id,
+            app_id=existing_memory.app_id,
+            conversation_id=existing_memory.conversation_id,
+            node_id=existing_memory.node_id,
+            value=update,  # New value
+            version=existing_memory.version + 1,  # Increment version for update
+            edited_by_user=True,
+            created_by=existing_memory.created_by,
+        )
+
+        ChatflowMemoryService.save_memory(updated_memory, VariablePool(), False)
+        return "", 204
+
+
+class MemoryDeleteApi(Resource):
+    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
+    def delete(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("id", type=str, required=False, default=None)
+        args = parser.parse_args()
+        memory_id = args.get("id")
+
+        if memory_id:
+            ChatflowMemoryService.delete_memory(app_model, memory_id, MemoryCreatedBy(end_user_id=end_user.id))
+            return "", 204
+        else:
+            ChatflowMemoryService.delete_all_user_memories(app_model, MemoryCreatedBy(end_user_id=end_user.id))
+            return "", 200
+
+
+api.add_resource(MemoryListApi, "/memories")
+api.add_resource(MemoryEditApi, "/memory-edit")
+api.add_resource(MemoryDeleteApi, "/memories")

api/controllers/web/__init__.py

@@ -18,6 +18,7 @@ web_ns = Namespace("web", description="Web application API operations", path="/"
 from . import (
     app,
     audio,
+    chatflow_memory,
     completion,
     conversation,
     feature,
@@ -39,6 +40,7 @@ __all__ = [
     "app",
     "audio",
     "bp",
+    "chatflow_memory",
     "completion",
     "conversation",
     "feature",

api/controllers/web/chatflow_memory.py

@@ -0,0 +1,108 @@
+from flask_restx import reqparse
+
+from controllers.web import api
+from controllers.web.wraps import WebApiResource
+from core.memory.entities import MemoryBlock, MemoryCreatedBy
+from core.workflow.runtime.variable_pool import VariablePool
+from models import App, EndUser
+from services.chatflow_memory_service import ChatflowMemoryService
+from services.workflow_service import WorkflowService
+
+
+class MemoryListApi(WebApiResource):
+    def get(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("conversation_id", required=False, type=str | None, default=None)
+        parser.add_argument("memory_id", required=False, type=str | None, default=None)
+        parser.add_argument("version", required=False, type=int | None, default=None)
+        args = parser.parse_args()
+        conversation_id: str | None = args.get("conversation_id")
+        memory_id = args.get("memory_id")
+        version = args.get("version")
+
+        if conversation_id:
+            result = ChatflowMemoryService.get_persistent_memories_with_conversation(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), conversation_id, version
+            )
+            session_memories = ChatflowMemoryService.get_session_memories_with_conversation(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), conversation_id, version
+            )
+            result = [*result, *session_memories]
+        else:
+            result = ChatflowMemoryService.get_persistent_memories(
+                app_model, MemoryCreatedBy(end_user_id=end_user.id), version
+            )
+
+        if memory_id:
+            result = [it for it in result if it.spec.id == memory_id]
+        return [it for it in result if it.spec.end_user_visible]
+
+
+class MemoryEditApi(WebApiResource):
+    def put(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("id", type=str, required=True)
+        parser.add_argument("conversation_id", type=str | None, required=False, default=None)
+        parser.add_argument("node_id", type=str | None, required=False, default=None)
+        parser.add_argument("update", type=str, required=True)
+        args = parser.parse_args()
+        workflow = WorkflowService().get_published_workflow(app_model)
+        update = args.get("update")
+        conversation_id = args.get("conversation_id")
+        node_id = args.get("node_id")
+        if not isinstance(update, str):
+            return {"error": "Update must be a string"}, 400
+        if not workflow:
+            return {"error": "Workflow not found"}, 404
+        memory_spec = next((it for it in workflow.memory_blocks if it.id == args["id"]), None)
+        if not memory_spec:
+            return {"error": "Memory not found"}, 404
+        if not memory_spec.end_user_editable:
+            return {"error": "Memory not editable"}, 403
+
+        # First get existing memory
+        existing_memory = ChatflowMemoryService.get_memory_by_spec(
+            spec=memory_spec,
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            created_by=MemoryCreatedBy(end_user_id=end_user.id),
+            conversation_id=conversation_id,
+            node_id=node_id,
+            is_draft=False,
+        )
+
+        # Create updated memory instance with incremented version
+        updated_memory = MemoryBlock(
+            spec=existing_memory.spec,
+            tenant_id=existing_memory.tenant_id,
+            app_id=existing_memory.app_id,
+            conversation_id=existing_memory.conversation_id,
+            node_id=existing_memory.node_id,
+            value=update,  # New value
+            version=existing_memory.version + 1,  # Increment version for update
+            edited_by_user=True,
+            created_by=existing_memory.created_by,
+        )
+
+        ChatflowMemoryService.save_memory(updated_memory, VariablePool(), False)
+        return "", 204
+
+
+class MemoryDeleteApi(WebApiResource):
+    def delete(self, app_model: App, end_user: EndUser):
+        parser = reqparse.RequestParser()
+        parser.add_argument("id", type=str, required=False, default=None)
+        args = parser.parse_args()
+        memory_id = args.get("id")
+
+        if memory_id:
+            ChatflowMemoryService.delete_memory(app_model, memory_id, MemoryCreatedBy(end_user_id=end_user.id))
+            return "", 204
+        else:
+            ChatflowMemoryService.delete_all_user_memories(app_model, MemoryCreatedBy(end_user_id=end_user.id))
+            return "", 200
+
+
+api.add_resource(MemoryListApi, "/memories")
+api.add_resource(MemoryEditApi, "/memory-edit")
+api.add_resource(MemoryDeleteApi, "/memories")

api/core/app/apps/advanced_chat/app_runner.py

@@ -1,10 +1,11 @@
 import logging
 import time
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping, MutableMapping, Sequence
 from typing import Any, cast
 
 from sqlalchemy import select
 from sqlalchemy.orm import Session
+from typing_extensions import override
 
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfig
 from core.app.apps.base_app_queue_manager import AppQueueManager
@@ -20,6 +21,8 @@ from core.app.entities.queue_entities import (
     QueueTextChunkEvent,
 )
 from core.app.features.annotation_reply.annotation_reply import AnnotationReplyFeature
+from core.memory.entities import MemoryCreatedBy, MemoryScope
+from core.model_runtime.entities import AssistantPromptMessage, UserPromptMessage
 from core.moderation.base import ModerationError
 from core.moderation.input_moderation import InputModeration
 from core.variables.variables import VariableUnion
@@ -27,6 +30,7 @@ from core.workflow.enums import WorkflowType
 from core.workflow.graph_engine.command_channels.redis_channel import RedisChannel
 from core.workflow.graph_engine.layers.base import GraphEngineLayer
 from core.workflow.graph_engine.layers.persistence import PersistenceWorkflowInfo, WorkflowPersistenceLayer
+from core.workflow.graph_events import GraphRunSucceededEvent
 from core.workflow.repositories.workflow_execution_repository import WorkflowExecutionRepository
 from core.workflow.repositories.workflow_node_execution_repository import WorkflowNodeExecutionRepository
 from core.workflow.runtime import GraphRuntimeState, VariablePool
@@ -39,6 +43,8 @@ from models import Workflow
 from models.enums import UserFrom
 from models.model import App, Conversation, Message, MessageAnnotation
 from models.workflow import ConversationVariable
+from services.chatflow_history_service import ChatflowHistoryService
+from services.chatflow_memory_service import ChatflowMemoryService
 
 logger = logging.getLogger(__name__)
 
@@ -81,6 +87,10 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
         self._workflow_node_execution_repository = workflow_node_execution_repository
 
     def run(self):
+        ChatflowMemoryService.wait_for_sync_memory_completion(
+            workflow=self._workflow, conversation_id=self.conversation.id
+        )
+
         app_config = self.application_generate_entity.app_config
         app_config = cast(AdvancedChatAppConfig, app_config)
 
@@ -143,6 +153,7 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
                 # Based on the definition of `VariableUnion`,
                 # `list[Variable]` can be safely used as `list[VariableUnion]` since they are compatible.
                 conversation_variables=conversation_variables,
+                memory_blocks=self._fetch_memory_blocks(),
             )
 
             # init graph
@@ -206,6 +217,31 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
         for event in generator:
             self._handle_event(workflow_entry, event)
 
+        try:
+            self._check_app_memory_updates(variable_pool)
+        except Exception as e:
+            logger.exception("Failed to check app memory updates", exc_info=e)
+
+    @override
+    def _handle_event(self, workflow_entry: WorkflowEntry, event: Any) -> None:
+        super()._handle_event(workflow_entry, event)
+        if isinstance(event, GraphRunSucceededEvent):
+            workflow_outputs = event.outputs
+            if not workflow_outputs:
+                logger.warning("Chatflow output is empty.")
+                return
+            assistant_message = workflow_outputs.get("answer")
+            if not assistant_message:
+                logger.warning("Chatflow output does not contain 'answer'.")
+                return
+            if not isinstance(assistant_message, str):
+                logger.warning("Chatflow output 'answer' is not a string.")
+                return
+            try:
+                self._sync_conversation_to_chatflow_tables(assistant_message)
+            except Exception as e:
+                logger.exception("Failed to sync conversation to memory tables", exc_info=e)
+
     def handle_input_moderation(
         self,
         app_record: App,
@@ -403,3 +439,67 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
 
         # Return combined list
         return existing_variables + new_variables
+
+    def _fetch_memory_blocks(self) -> Mapping[str, str]:
+        """fetch all memory blocks for current app"""
+
+        memory_blocks_dict: MutableMapping[str, str] = {}
+        is_draft = self.application_generate_entity.invoke_from == InvokeFrom.DEBUGGER
+        conversation_id = self.conversation.id
+        memory_block_specs = self._workflow.memory_blocks
+        # Get runtime memory values
+        memories = ChatflowMemoryService.get_memories_by_specs(
+            memory_block_specs=memory_block_specs,
+            tenant_id=self._workflow.tenant_id,
+            app_id=self._workflow.app_id,
+            node_id=None,
+            conversation_id=conversation_id,
+            is_draft=is_draft,
+            created_by=self._get_created_by(),
+        )
+
+        # Build memory_id -> value mapping
+        for memory in memories:
+            if memory.spec.scope == MemoryScope.APP:
+                # App level: use memory_id directly
+                memory_blocks_dict[memory.spec.id] = memory.value
+            else:  # NODE scope
+                node_id = memory.node_id
+                if not node_id:
+                    logger.warning("Memory block %s has no node_id, skip.", memory.spec.id)
+                    continue
+                key = f"{node_id}.{memory.spec.id}"
+                memory_blocks_dict[key] = memory.value
+
+        return memory_blocks_dict
+
+    def _sync_conversation_to_chatflow_tables(self, assistant_message: str):
+        ChatflowHistoryService.save_app_message(
+            prompt_message=UserPromptMessage(content=(self.application_generate_entity.query)),
+            conversation_id=self.conversation.id,
+            app_id=self._workflow.app_id,
+            tenant_id=self._workflow.tenant_id,
+        )
+        ChatflowHistoryService.save_app_message(
+            prompt_message=AssistantPromptMessage(content=assistant_message),
+            conversation_id=self.conversation.id,
+            app_id=self._workflow.app_id,
+            tenant_id=self._workflow.tenant_id,
+        )
+
+    def _check_app_memory_updates(self, variable_pool: VariablePool):
+        is_draft = self.application_generate_entity.invoke_from == InvokeFrom.DEBUGGER
+
+        ChatflowMemoryService.update_app_memory_if_needed(
+            workflow=self._workflow,
+            conversation_id=self.conversation.id,
+            variable_pool=variable_pool,
+            is_draft=is_draft,
+            created_by=self._get_created_by(),
+        )
+
+    def _get_created_by(self) -> MemoryCreatedBy:
+        if self.application_generate_entity.invoke_from in {InvokeFrom.DEBUGGER, InvokeFrom.EXPLORE}:
+            return MemoryCreatedBy(account_id=self.application_generate_entity.user_id)
+        else:
+            return MemoryCreatedBy(end_user_id=self.application_generate_entity.user_id)

api/core/app/apps/pipeline/pipeline_generator.py

@@ -42,18 +42,13 @@ from core.workflow.repositories.workflow_node_execution_repository import Workfl
 from core.workflow.variable_loader import DUMMY_VARIABLE_LOADER, VariableLoader
 from enums.cloud_plan import CloudPlan
 from extensions.ext_database import db
-from extensions.ext_redis import redis_client
 from libs.flask_utils import preserve_flask_contexts
 from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom
 from models.dataset import Document, DocumentPipelineExecutionLog, Pipeline
 from models.enums import WorkflowRunTriggeredFrom
 from models.model import AppMode
 from services.datasource_provider_service import DatasourceProviderService
-from services.feature_service import FeatureService
-from services.file_service import FileService
 from services.workflow_draft_variable_service import DraftVarLoader, WorkflowDraftVariableService
-from tasks.rag_pipeline.priority_rag_pipeline_run_task import priority_rag_pipeline_run_task
-from tasks.rag_pipeline.rag_pipeline_run_task import rag_pipeline_run_task
 
 logger = logging.getLogger(__name__)
 

api/core/entities/document_task.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class DocumentTask:
+    """Document task entity for document indexing operations.
+    
+    This class represents a document indexing task that can be queued
+    and processed by the document indexing system.
+    """
+    tenant_id: str
+    dataset_id: str
+    document_ids: list[str]

api/core/entities/mcp_provider.py

@@ -14,6 +14,7 @@ from core.helper.provider_cache import NoOpProviderCredentialCache
 from core.mcp.types import OAuthClientInformation, OAuthClientMetadata, OAuthTokens
 from core.tools.entities.common_entities import I18nObject
 from core.tools.entities.tool_entities import ToolProviderType
+from core.tools.utils.encryption import create_provider_encrypter
 
 if TYPE_CHECKING:
     from models.tools import MCPToolProvider
@@ -271,7 +272,6 @@ class MCPProviderEntity(BaseModel):
 
     def _decrypt_dict(self, data: dict[str, Any]) -> dict[str, Any]:
         """Generic method to decrypt dictionary fields"""
-        from core.tools.utils.encryption import create_provider_encrypter
 
         if not data:
             return {}

api/core/hosting_configuration.py

@@ -56,6 +56,9 @@ class HostingConfiguration:
         self.provider_map[f"{DEFAULT_PLUGIN_ID}/minimax/minimax"] = self.init_minimax()
         self.provider_map[f"{DEFAULT_PLUGIN_ID}/spark/spark"] = self.init_spark()
         self.provider_map[f"{DEFAULT_PLUGIN_ID}/zhipuai/zhipuai"] = self.init_zhipuai()
+        self.provider_map[f"{DEFAULT_PLUGIN_ID}/gemini/google"] = self.init_gemini()
+        self.provider_map[f"{DEFAULT_PLUGIN_ID}/x/x"] = self.init_xai()
+        self.provider_map[f"{DEFAULT_PLUGIN_ID}/deepseek/deepseek"] = self.init_deepseek()
 
         self.moderation_config = self.init_moderation_config()
 
@@ -128,7 +131,7 @@ class HostingConfiguration:
         quotas: list[HostingQuota] = []
 
         if dify_config.HOSTED_OPENAI_TRIAL_ENABLED:
-            hosted_quota_limit = dify_config.HOSTED_OPENAI_QUOTA_LIMIT
+            hosted_quota_limit = 0
             trial_models = self.parse_restrict_models_from_env("HOSTED_OPENAI_TRIAL_MODELS")
             trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trial_models)
             quotas.append(trial_quota)
@@ -156,18 +159,49 @@ class HostingConfiguration:
             quota_unit=quota_unit,
         )
 
-    @staticmethod
-    def init_anthropic() -> HostingProvider:
-        quota_unit = QuotaUnit.TOKENS
+    def init_gemini(self) -> HostingProvider:
+        quota_unit = QuotaUnit.CREDITS
+        quotas: list[HostingQuota] = []
+
+        if dify_config.HOSTED_GEMINI_TRIAL_ENABLED:
+            hosted_quota_limit = 0
+            trial_models = self.parse_restrict_models_from_env("HOSTED_GEMINI_TRIAL_MODELS")
+            trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trial_models)
+            quotas.append(trial_quota)
+
+        if dify_config.HOSTED_GEMINI_PAID_ENABLED:
+            paid_models = self.parse_restrict_models_from_env("HOSTED_GEMINI_PAID_MODELS")
+            paid_quota = PaidHostingQuota(restrict_models=paid_models)
+            quotas.append(paid_quota)
+
+        if len(quotas) > 0:
+            credentials = {
+                "google_api_key": dify_config.HOSTED_GEMINI_API_KEY,
+            }
+
+            if dify_config.HOSTED_GEMINI_API_BASE:
+                credentials["google_base_url"] = dify_config.HOSTED_GEMINI_API_BASE
+
+            return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas)
+
+        return HostingProvider(
+            enabled=False,
+            quota_unit=quota_unit,
+        )
+
+    def init_anthropic(self) -> HostingProvider:
+        quota_unit = QuotaUnit.CREDITS
         quotas: list[HostingQuota] = []
 
         if dify_config.HOSTED_ANTHROPIC_TRIAL_ENABLED:
-            hosted_quota_limit = dify_config.HOSTED_ANTHROPIC_QUOTA_LIMIT
-            trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit)
+            hosted_quota_limit = 0
+            trail_models = self.parse_restrict_models_from_env("HOSTED_ANTHROPIC_TRIAL_MODELS")
+            trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models)
             quotas.append(trial_quota)
 
         if dify_config.HOSTED_ANTHROPIC_PAID_ENABLED:
-            paid_quota = PaidHostingQuota()
+            paid_models = self.parse_restrict_models_from_env("HOSTED_ANTHROPIC_PAID_MODELS")
+            paid_quota = PaidHostingQuota(restrict_models=paid_models)
             quotas.append(paid_quota)
 
         if len(quotas) > 0:
@@ -185,6 +219,66 @@ class HostingConfiguration:
             quota_unit=quota_unit,
         )
 
+    def init_xai(self) -> HostingProvider:
+        quota_unit = QuotaUnit.CREDITS
+        quotas: list[HostingQuota] = []
+
+        if dify_config.HOSTED_XAI_TRIAL_ENABLED:
+            hosted_quota_limit = 0
+            trail_models = self.parse_restrict_models_from_env("HOSTED_XAI_TRIAL_MODELS")
+            trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models)
+            quotas.append(trial_quota)
+
+        if dify_config.HOSTED_XAI_PAID_ENABLED:
+            paid_models = self.parse_restrict_models_from_env("HOSTED_XAI_PAID_MODELS")
+            paid_quota = PaidHostingQuota(restrict_models=paid_models)
+            quotas.append(paid_quota)
+
+        if len(quotas) > 0:
+            credentials = {
+                "api_key": dify_config.HOSTED_XAI_API_KEY,
+            }
+
+            if dify_config.HOSTED_XAI_API_BASE:
+                credentials["endpoint_url"] = dify_config.HOSTED_XAI_API_BASE
+
+            return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas)
+
+        return HostingProvider(
+            enabled=False,
+            quota_unit=quota_unit,
+        )
+
+    def init_deepseek(self) -> HostingProvider:
+        quota_unit = QuotaUnit.CREDITS
+        quotas: list[HostingQuota] = []
+
+        if dify_config.HOSTED_DEEPSEEK_TRIAL_ENABLED:
+            hosted_quota_limit = 0
+            trail_models = self.parse_restrict_models_from_env("HOSTED_DEEPSEEK_TRIAL_MODELS")
+            trial_quota = TrialHostingQuota(quota_limit=hosted_quota_limit, restrict_models=trail_models)
+            quotas.append(trial_quota)
+
+        if dify_config.HOSTED_DEEPSEEK_PAID_ENABLED:
+            paid_models = self.parse_restrict_models_from_env("HOSTED_DEEPSEEK_PAID_MODELS")
+            paid_quota = PaidHostingQuota(restrict_models=paid_models)
+            quotas.append(paid_quota)
+
+        if len(quotas) > 0:
+            credentials = {
+                "api_key": dify_config.HOSTED_DEEPSEEK_API_KEY,
+            }
+
+            if dify_config.HOSTED_DEEPSEEK_API_BASE:
+                credentials["endpoint_url"] = dify_config.HOSTED_DEEPSEEK_API_BASE
+
+            return HostingProvider(enabled=True, credentials=credentials, quota_unit=quota_unit, quotas=quotas)
+
+        return HostingProvider(
+            enabled=False,
+            quota_unit=quota_unit,
+        )
+
     @staticmethod
     def init_minimax() -> HostingProvider:
         quota_unit = QuotaUnit.TOKENS

api/core/llm_generator/llm_generator.py

@@ -14,10 +14,12 @@ from core.llm_generator.prompts import (
     JAVASCRIPT_CODE_GENERATOR_PROMPT_TEMPLATE,
     LLM_MODIFY_CODE_SYSTEM,
     LLM_MODIFY_PROMPT_SYSTEM,
+    MEMORY_UPDATE_PROMPT,
     PYTHON_CODE_GENERATOR_PROMPT_TEMPLATE,
     SYSTEM_STRUCTURED_OUTPUT_GENERATE,
     WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE,
 )
+from core.memory.entities import MemoryBlock, MemoryBlockSpec
 from core.model_manager import ModelManager
 from core.model_runtime.entities.llm_entities import LLMResult
 from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
@@ -28,6 +30,7 @@ from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.ops.utils import measure_time
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
 from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey
+from core.workflow.runtime.variable_pool import VariablePool
 from extensions.ext_database import db
 from extensions.ext_storage import storage
 from models import App, Message, WorkflowNodeExecutionModel
@@ -562,3 +565,35 @@ class LLMGenerator:
                 "Failed to invoke LLM model, model: %s", json.dumps(model_config.get("name")), exc_info=True
             )
             return {"error": f"An unexpected error occurred: {str(e)}"}
+
+    @staticmethod
+    def update_memory_block(
+        tenant_id: str,
+        visible_history: Sequence[tuple[str, str]],
+        variable_pool: VariablePool,
+        memory_block: MemoryBlock,
+        memory_spec: MemoryBlockSpec,
+    ) -> str:
+        model_instance = ModelManager().get_model_instance(
+            tenant_id=tenant_id,
+            provider=memory_spec.model.provider,
+            model=memory_spec.model.name,
+            model_type=ModelType.LLM,
+        )
+        formatted_history = ""
+        for sender, message in visible_history:
+            formatted_history += f"{sender}: {message}\n"
+        filled_instruction = variable_pool.convert_template(memory_spec.instruction).text
+        formatted_prompt = PromptTemplateParser(MEMORY_UPDATE_PROMPT).format(
+            inputs={
+                "formatted_history": formatted_history,
+                "current_value": memory_block.value,
+                "instruction": filled_instruction,
+            }
+        )
+        llm_result = model_instance.invoke_llm(
+            prompt_messages=[UserPromptMessage(content=formatted_prompt)],
+            model_parameters=memory_spec.model.completion_params,
+            stream=False,
+        )
+        return llm_result.message.get_text_content()

api/core/llm_generator/prompts.py

@@ -422,3 +422,18 @@ INSTRUCTION_GENERATE_TEMPLATE_PROMPT = """The output of this prompt is not as ex
 You should edit the prompt according to the IDEAL OUTPUT."""
 
 INSTRUCTION_GENERATE_TEMPLATE_CODE = """Please fix the errors in the {{#error_message#}}."""
+
+MEMORY_UPDATE_PROMPT = """
+Based on the following conversation history, update the memory content:
+
+Conversation history:
+{{formatted_history}}
+
+Current memory:
+{{current_value}}
+
+Update instruction:
+{{instruction}}
+
+Please output only the updated memory content, no other text like greeting:
+"""

api/core/memory/entities.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import TYPE_CHECKING, Optional
+from uuid import uuid4
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from core.app.app_config.entities import ModelConfig
+
+
+class MemoryScope(StrEnum):
+    """Memory scope determined by node_id field"""
+    APP = "app"  # node_id is None
+    NODE = "node"  # node_id is not None
+
+
+class MemoryTerm(StrEnum):
+    """Memory term determined by conversation_id field"""
+    SESSION = "session"  # conversation_id is not None
+    PERSISTENT = "persistent"  # conversation_id is None
+
+
+class MemoryStrategy(StrEnum):
+    ON_TURNS = "on_turns"
+
+
+class MemoryScheduleMode(StrEnum):
+    SYNC = "sync"
+    ASYNC = "async"
+
+
+class MemoryBlockSpec(BaseModel):
+    """Memory block specification for workflow configuration"""
+    id: str = Field(
+        default_factory=lambda: str(uuid4()),
+        description="Unique identifier for the memory block",
+    )
+    name: str = Field(description="Display name of the memory block")
+    description: str = Field(default="", description="Description of the memory block")
+    template: str = Field(description="Initial template content for the memory")
+    instruction: str = Field(description="Instructions for updating the memory")
+    scope: MemoryScope = Field(description="Scope of the memory (app or node level)")
+    term: MemoryTerm = Field(description="Term of the memory (session or persistent)")
+    strategy: MemoryStrategy = Field(description="Update strategy for the memory")
+    update_turns: int = Field(gt=0, description="Number of turns between updates")
+    preserved_turns: int = Field(gt=0, description="Number of conversation turns to preserve")
+    schedule_mode: MemoryScheduleMode = Field(description="Synchronous or asynchronous update mode")
+    model: ModelConfig = Field(description="Model configuration for memory updates")
+    end_user_visible: bool = Field(default=False, description="Whether memory is visible to end users")
+    end_user_editable: bool = Field(default=False, description="Whether memory is editable by end users")
+
+
+class MemoryCreatedBy(BaseModel):
+    end_user_id: str | None = None
+    account_id: str | None = None
+
+
+class MemoryBlock(BaseModel):
+    """Runtime memory block instance
+
+    Design Rules:
+    - app_id = None: Global memory (future feature, not implemented yet)
+    - app_id = str: App-specific memory
+    - conversation_id = None: Persistent memory (cross-conversation)
+    - conversation_id = str: Session memory (conversation-specific)
+    - node_id = None: App-level scope
+    - node_id = str: Node-level scope
+
+    These rules implicitly determine scope and term without redundant storage.
+    """
+    spec: MemoryBlockSpec
+    tenant_id: str
+    value: str
+    app_id: str
+    conversation_id: Optional[str] = None
+    node_id: Optional[str] = None
+    edited_by_user: bool = False
+    created_by: MemoryCreatedBy
+    version: int = Field(description="Memory block version number")
+
+
+class MemoryValueData(BaseModel):
+    value: str
+    edited_by_user: bool = False
+
+
+class ChatflowConversationMetadata(BaseModel):
+    """Metadata for chatflow conversation with visible message count"""
+    type: str = "mutable_visible_window"
+    visible_count: int = Field(gt=0, description="Number of visible messages to keep")
+
+
+class MemoryBlockWithConversation(MemoryBlock):
+    """MemoryBlock with optional conversation metadata for session memories"""
+    conversation_metadata: ChatflowConversationMetadata = Field(
+        description="Conversation metadata, only present for session memories"
+    )
+
+    @classmethod
+    def from_memory_block(
+        cls,
+        memory_block: MemoryBlock,
+        conversation_metadata: ChatflowConversationMetadata
+    ) -> MemoryBlockWithConversation:
+        """Create MemoryBlockWithConversation from MemoryBlock"""
+        return cls(
+            spec=memory_block.spec,
+            tenant_id=memory_block.tenant_id,
+            value=memory_block.value,
+            app_id=memory_block.app_id,
+            conversation_id=memory_block.conversation_id,
+            node_id=memory_block.node_id,
+            edited_by_user=memory_block.edited_by_user,
+            created_by=memory_block.created_by,
+            version=memory_block.version,
+            conversation_metadata=conversation_metadata
+        )

api/core/memory/errors.py

@@ -0,0 +1,6 @@
+class MemorySyncTimeoutError(Exception):
+    def __init__(self, app_id: str, conversation_id: str):
+        self.app_id = app_id
+        self.conversation_id = conversation_id
+        self.message = "Memory synchronization timeout after 50 seconds"
+        super().__init__(self.message)

api/core/prompt/entities/advanced_prompt_entities.py

@@ -45,6 +45,12 @@ class MemoryConfig(BaseModel):
         enabled: bool
         size: int | None = None
 
+    mode: Literal["linear", "block"] | None = "linear"
+    block_id: list[str] | None = None
     role_prefix: RolePrefix | None = None
     window: WindowConfig
     query_prompt_template: str | None = None
+
+    @property
+    def is_block_mode(self) -> bool:
+        return self.mode == "block" and bool(self.block_id)

api/core/provider_manager.py

@@ -618,18 +618,18 @@ class ProviderManager:
                 )
 
             for quota in configuration.quotas:
-                if quota.quota_type == ProviderQuotaType.TRIAL:
+                if quota.quota_type in (ProviderQuotaType.TRIAL, ProviderQuotaType.PAID):
                     # Init trial provider records if not exists
-                    if ProviderQuotaType.TRIAL not in provider_quota_to_provider_record_dict:
+                    if quota.quota_type not in provider_quota_to_provider_record_dict:
                         try:
                             # FIXME ignore the type error, only TrialHostingQuota has limit need to change the logic
                             new_provider_record = Provider(
                                 tenant_id=tenant_id,
                                 # TODO: Use provider name with prefix after the data migration.
                                 provider_name=ModelProviderID(provider_name).provider_name,
-                                provider_type=ProviderType.SYSTEM,
-                                quota_type=ProviderQuotaType.TRIAL,
-                                quota_limit=quota.quota_limit,  # type: ignore
+                                provider_type=ProviderType.SYSTEM.value,
+                                quota_type=quota.quota_type,
+                                quota_limit=0,  # type: ignore
                                 quota_used=0,
                                 is_valid=True,
                             )
@@ -641,8 +641,8 @@ class ProviderManager:
                             stmt = select(Provider).where(
                                 Provider.tenant_id == tenant_id,
                                 Provider.provider_name == ModelProviderID(provider_name).provider_name,
-                                Provider.provider_type == ProviderType.SYSTEM,
-                                Provider.quota_type == ProviderQuotaType.TRIAL,
+                                Provider.provider_type == ProviderType.SYSTEM.value,
+                                Provider.quota_type == quota.quota_type,
                             )
                             existed_provider_record = db.session.scalar(stmt)
                             if not existed_provider_record:
@@ -652,7 +652,7 @@ class ProviderManager:
                                 existed_provider_record.is_valid = True
                                 db.session.commit()
 
-                            provider_name_to_provider_records_dict[provider_name].append(existed_provider_record)
+                            provider_name_to_provider_records_dict[provider_name].append(existed_provider_record)                        
 
         return provider_name_to_provider_records_dict
 
@@ -912,6 +912,22 @@ class ProviderManager:
                 provider_record
             )
         quota_configurations = []
+
+        if dify_config.EDITION == "CLOUD":
+            from services.credit_pool_service import CreditPoolService
+
+            trail_pool = CreditPoolService.get_pool(
+                tenant_id=tenant_id,
+                pool_type=ProviderQuotaType.TRIAL.value,
+            )
+            paid_pool = CreditPoolService.get_pool(
+                tenant_id=tenant_id,
+                pool_type=ProviderQuotaType.PAID.value,
+            )
+        else:
+            trail_pool = None
+            paid_pool = None
+
         for provider_quota in provider_hosting_configuration.quotas:
             if provider_quota.quota_type not in quota_type_to_provider_records_dict:
                 if provider_quota.quota_type == ProviderQuotaType.FREE:
@@ -932,16 +948,36 @@ class ProviderManager:
                     raise ValueError("quota_used is None")
                 if provider_record.quota_limit is None:
                     raise ValueError("quota_limit is None")
+                if provider_quota.quota_type == ProviderQuotaType.TRIAL and trail_pool is not None:
+                    quota_configuration = QuotaConfiguration(
+                        quota_type=provider_quota.quota_type,
+                        quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS,
+                        quota_used=trail_pool.quota_used,
+                        quota_limit=trail_pool.quota_limit,
+                        is_valid=trail_pool.quota_limit > trail_pool.quota_used or trail_pool.quota_limit == -1,
+                        restrict_models=provider_quota.restrict_models,
+                    )
 
-                quota_configuration = QuotaConfiguration(
-                    quota_type=provider_quota.quota_type,
-                    quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS,
-                    quota_used=provider_record.quota_used,
-                    quota_limit=provider_record.quota_limit,
-                    is_valid=provider_record.quota_limit > provider_record.quota_used
-                    or provider_record.quota_limit == -1,
-                    restrict_models=provider_quota.restrict_models,
-                )
+                elif provider_quota.quota_type == ProviderQuotaType.PAID and paid_pool is not None:
+                    quota_configuration = QuotaConfiguration(
+                        quota_type=provider_quota.quota_type,
+                        quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS,
+                        quota_used=paid_pool.quota_used,
+                        quota_limit=paid_pool.quota_limit,
+                        is_valid=paid_pool.quota_limit > paid_pool.quota_used or paid_pool.quota_limit == -1,
+                        restrict_models=provider_quota.restrict_models,
+                    )
+
+                else:
+                    quota_configuration = QuotaConfiguration(
+                        quota_type=provider_quota.quota_type,
+                        quota_unit=provider_hosting_configuration.quota_unit or QuotaUnit.TOKENS,
+                        quota_used=provider_record.quota_used,
+                        quota_limit=provider_record.quota_limit,
+                        is_valid=provider_record.quota_limit > provider_record.quota_used
+                        or provider_record.quota_limit == -1,
+                        restrict_models=provider_quota.restrict_models,
+                    )
 
             quota_configurations.append(quota_configuration)
 

api/core/rag/datasource/vdb/oracle/oraclevector.py

@@ -289,7 +289,8 @@ class OracleVector(BaseVector):
                 words = pseg.cut(query)
                 current_entity = ""
                 for word, pos in words:
-                    if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}:  # nr: 人名，ns: 地名，nt: 机构名
+                    # nr: person name, ns: place name, nt: organization name
+                    if pos in {"nr", "Ng", "eng", "nz", "n", "ORG", "v"}:
                         current_entity += word
                     else:
                         if current_entity:

api/core/rag/datasource/vdb/pyvastbase/vastbase_vector.py

@@ -213,7 +213,7 @@ class VastbaseVector(BaseVector):
 
             with self._get_cursor() as cur:
                 cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension))
-                # Vastbase 支持的向量维度取值范围为 [1,16000]
+                # Vastbase supports vector dimensions in range [1, 16000]
                 if dimension <= 16000:
                     cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name))
             redis_client.set(collection_exist_cache_key, 1, ex=3600)

api/core/rag/pipeline/queue.py

@@ -0,0 +1,89 @@
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from extensions.ext_redis import redis_client
+
+TASK_WRAPPER_PREFIX = "__WRAPPER__:"
+
+
+@dataclass
+class TaskWrapper:
+    data: Any
+    
+    def serialize(self) -> str:
+        return json.dumps(self.data, ensure_ascii=False)
+    
+    @classmethod
+    def deserialize(cls, serialized_data: str) -> 'TaskWrapper':
+        data = json.loads(serialized_data)
+        return cls(data)
+
+
+class TenantSelfTaskQueue:
+    """
+    Simple queue for tenant self tasks, used for tenant self task isolation.
+    It uses Redis list to store tasks, and Redis key to store task waiting flag.
+    Support tasks that can be serialized by json.
+    """
+    DEFAULT_TASK_TTL = 60 * 60
+
+    def __init__(self, tenant_id: str, unique_key: str):
+        self.tenant_id = tenant_id
+        self.unique_key = unique_key
+        self.queue = f"tenant_self_{unique_key}_task_queue:{tenant_id}"
+        self.task_key = f"tenant_{unique_key}_task:{tenant_id}"
+
+    def get_task_key(self):
+        return redis_client.get(self.task_key)
+
+    def set_task_waiting_time(self, ttl: int | None = None):
+        ttl = ttl or self.DEFAULT_TASK_TTL
+        redis_client.setex(self.task_key, ttl, 1)
+
+    def delete_task_key(self):
+        redis_client.delete(self.task_key)
+
+    def push_tasks(self, tasks: list):
+        serialized_tasks = []
+        for task in tasks:
+            # Store str list directly, maintaining full compatibility for pipeline scenarios
+            if isinstance(task, str):
+                serialized_tasks.append(task)
+            else:
+                # Use TaskWrapper to do JSON serialization, add prefix for identification
+                wrapper = TaskWrapper(task)
+                serialized_data = wrapper.serialize()
+                serialized_tasks.append(f"{TASK_WRAPPER_PREFIX}{serialized_data}")
+        
+        redis_client.lpush(self.queue, *serialized_tasks)
+    
+    def pull_tasks(self, count: int = 1) -> list:
+        if count <= 0:
+            return []
+        
+        tasks = []
+        for _ in range(count):
+            serialized_task = redis_client.rpop(self.queue)
+            if not serialized_task:
+                break
+            
+            if isinstance(serialized_task, bytes):
+                serialized_task = serialized_task.decode('utf-8')
+            
+            # Check if use TaskWrapper or not
+            if serialized_task.startswith(TASK_WRAPPER_PREFIX):
+                try:
+                    wrapper_data = serialized_task[len(TASK_WRAPPER_PREFIX):]
+                    wrapper = TaskWrapper.deserialize(wrapper_data)
+                    tasks.append(wrapper.data)
+                except (json.JSONDecodeError, TypeError, ValueError):
+                    tasks.append(serialized_task)
+            else:
+                tasks.append(serialized_task)
+        
+        return tasks
+
+    def get_next_task(self):
+        tasks = self.pull_tasks(1)
+        return tasks[0] if tasks else None

api/core/variables/segments.py

@@ -203,6 +203,48 @@ class ArrayFileSegment(ArraySegment):
         return ""
 
 
+class VersionedMemoryValue(BaseModel):
+    current_value: str = None  # type: ignore
+    versions: Mapping[str, str] = {}
+
+    model_config = ConfigDict(frozen=True)
+
+    def add_version(
+        self,
+        new_value: str,
+        version_name: str | None = None
+    ) -> "VersionedMemoryValue":
+        if version_name is None:
+            version_name = str(len(self.versions) + 1)
+        if version_name in self.versions:
+            raise ValueError(f"Version '{version_name}' already exists.")
+        self.current_value = new_value
+        return VersionedMemoryValue(
+            current_value=new_value,
+            versions={
+                version_name: new_value,
+                **self.versions,
+            }
+        )
+
+
+class VersionedMemorySegment(Segment):
+    value_type: SegmentType = SegmentType.VERSIONED_MEMORY
+    value: VersionedMemoryValue = None  # type: ignore
+
+    @property
+    def text(self) -> str:
+        return self.value.current_value
+
+    @property
+    def log(self) -> str:
+        return self.value.current_value
+
+    @property
+    def markdown(self) -> str:
+        return self.value.current_value
+
+
 class ArrayBooleanSegment(ArraySegment):
     value_type: SegmentType = SegmentType.ARRAY_BOOLEAN
     value: Sequence[bool]
@@ -248,6 +290,7 @@ SegmentUnion: TypeAlias = Annotated[
         | Annotated[ArrayObjectSegment, Tag(SegmentType.ARRAY_OBJECT)]
         | Annotated[ArrayFileSegment, Tag(SegmentType.ARRAY_FILE)]
         | Annotated[ArrayBooleanSegment, Tag(SegmentType.ARRAY_BOOLEAN)]
+        | Annotated[VersionedMemorySegment, Tag(SegmentType.VERSIONED_MEMORY)]
     ),
     Discriminator(get_segment_discriminator),
 ]

api/core/variables/types.py

@@ -41,6 +41,8 @@ class SegmentType(StrEnum):
     ARRAY_FILE = "array[file]"
     ARRAY_BOOLEAN = "array[boolean]"
 
+    VERSIONED_MEMORY = "versioned_memory"
+
     NONE = "none"
 
     GROUP = "group"

api/core/variables/variables.py

@@ -22,6 +22,7 @@ from .segments import (
     ObjectSegment,
     Segment,
     StringSegment,
+    VersionedMemorySegment,
     get_segment_discriminator,
 )
 from .types import SegmentType
@@ -106,6 +107,10 @@ class ArrayFileVariable(ArrayFileSegment, ArrayVariable):
     pass
 
 
+class VersionedMemoryVariable(VersionedMemorySegment, Variable):
+    pass
+
+
 class ArrayBooleanVariable(ArrayBooleanSegment, ArrayVariable):
     pass
 
@@ -161,6 +166,7 @@ VariableUnion: TypeAlias = Annotated[
         | Annotated[ArrayFileVariable, Tag(SegmentType.ARRAY_FILE)]
         | Annotated[ArrayBooleanVariable, Tag(SegmentType.ARRAY_BOOLEAN)]
         | Annotated[SecretVariable, Tag(SegmentType.SECRET)]
+        | Annotated[VersionedMemoryVariable, Tag(SegmentType.VERSIONED_MEMORY)]
     ),
     Discriminator(get_segment_discriminator),
 ]

api/core/workflow/constants.py

@@ -1,4 +1,5 @@
 SYSTEM_VARIABLE_NODE_ID = "sys"
 ENVIRONMENT_VARIABLE_NODE_ID = "env"
 CONVERSATION_VARIABLE_NODE_ID = "conversation"
+MEMORY_BLOCK_VARIABLE_NODE_ID = "memory_block"
 RAG_PIPELINE_VARIABLE_NODE_ID = "rag"

api/core/workflow/nodes/llm/llm_utils.py

@@ -6,7 +6,7 @@ from sqlalchemy.orm import Session
 
 from configs import dify_config
 from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
-from core.entities.provider_entities import QuotaUnit
+from core.entities.provider_entities import ProviderQuotaType, QuotaUnit
 from core.file.models import File
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance, ModelManager
@@ -136,21 +136,36 @@ def deduct_llm_quota(tenant_id: str, model_instance: ModelInstance, usage: LLMUs
             used_quota = 1
 
     if used_quota is not None and system_configuration.current_quota_type is not None:
-        with Session(db.engine) as session:
-            stmt = (
-                update(Provider)
-                .where(
-                    Provider.tenant_id == tenant_id,
-                    # TODO: Use provider name with prefix after the data migration.
-                    Provider.provider_name == ModelProviderID(model_instance.provider).provider_name,
-                    Provider.provider_type == ProviderType.SYSTEM,
-                    Provider.quota_type == system_configuration.current_quota_type.value,
-                    Provider.quota_limit > Provider.quota_used,
-                )
-                .values(
-                    quota_used=Provider.quota_used + used_quota,
-                    last_used=naive_utc_now(),
-                )
+        if system_configuration.current_quota_type == ProviderQuotaType.TRIAL:
+            from services.credit_pool_service import CreditPoolService
+
+            CreditPoolService.check_and_deduct_credits(
+                tenant_id=tenant_id,
+                credits_required=used_quota,
             )
-            session.execute(stmt)
-            session.commit()
+        elif system_configuration.current_quota_type == ProviderQuotaType.PAID:
+            from services.credit_pool_service import CreditPoolService
+            CreditPoolService.check_and_deduct_credits(
+                tenant_id=tenant_id,
+                credits_required=used_quota,
+                pool_type="paid",
+            )
+        else:
+            with Session(db.engine) as session:
+                stmt = (
+                    update(Provider)
+                    .where(
+                        Provider.tenant_id == tenant_id,
+                        # TODO: Use provider name with prefix after the data migration.
+                        Provider.provider_name == ModelProviderID(model_instance.provider).provider_name,
+                        Provider.provider_type == ProviderType.SYSTEM.value,
+                        Provider.quota_type == system_configuration.current_quota_type.value,
+                        Provider.quota_limit > Provider.quota_used,
+                    )
+                    .values(
+                        quota_used=Provider.quota_used + used_quota,
+                        last_used=naive_utc_now(),
+                    )
+                )
+                session.execute(stmt)
+                session.commit()

api/core/workflow/nodes/llm/node.py

@@ -7,11 +7,15 @@ import time
 from collections.abc import Generator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Literal
 
-from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from core.app.entities.app_invoke_entities import InvokeFrom, ModelConfigWithCredentialsEntity
 from core.file import FileType, file_manager
 from core.helper.code_executor import CodeExecutor, CodeLanguage
 from core.llm_generator.output_parser.errors import OutputParserError
 from core.llm_generator.output_parser.structured_output import invoke_llm_with_structured_output
+from core.memory.entities import MemoryCreatedBy, MemoryScope
 from core.memory.token_buffer_memory import TokenBufferMemory
 from core.model_manager import ModelInstance, ModelManager
 from core.model_runtime.entities import (
@@ -73,6 +77,8 @@ from core.workflow.nodes.base.entities import BaseNodeData, RetryConfig, Variabl
 from core.workflow.nodes.base.node import Node
 from core.workflow.nodes.base.variable_template_parser import VariableTemplateParser
 from core.workflow.runtime import VariablePool
+from models import UserFrom, Workflow
+from models.engine import db
 
 from . import llm_utils
 from .entities import (
@@ -317,6 +323,11 @@ class LLMNode(Node):
             if self._file_outputs:
                 outputs["files"] = ArrayFileSegment(value=self._file_outputs)
 
+            try:
+                self._handle_chatflow_memory(result_text, variable_pool)
+            except Exception as e:
+                logger.warning("Memory orchestration failed for node %s: %s", self.node_id, str(e))
+
             # Send final chunk event to indicate streaming is complete
             yield StreamChunkEvent(
                 selector=[self._node_id, "text"],
@@ -1228,6 +1239,77 @@ class LLMNode(Node):
     def retry(self) -> bool:
         return self._node_data.retry_config.retry_enabled
 
+    def _handle_chatflow_memory(self, llm_output: str, variable_pool: VariablePool):
+        if not self._node_data.memory or self._node_data.memory.mode != "block":
+            return
+
+        conversation_id_segment = variable_pool.get((SYSTEM_VARIABLE_NODE_ID, SystemVariableKey.CONVERSATION_ID))
+        if not conversation_id_segment:
+            raise ValueError("Conversation ID not found in variable pool.")
+        conversation_id = conversation_id_segment.text
+
+        user_query_segment = variable_pool.get((SYSTEM_VARIABLE_NODE_ID, SystemVariableKey.QUERY))
+        if not user_query_segment:
+            raise ValueError("User query not found in variable pool.")
+        user_query = user_query_segment.text
+
+        from core.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage
+        from services.chatflow_history_service import ChatflowHistoryService
+
+        ChatflowHistoryService.save_node_message(
+            prompt_message=(UserPromptMessage(content=user_query)),
+            node_id=self.node_id,
+            conversation_id=conversation_id,
+            app_id=self.app_id,
+            tenant_id=self.tenant_id,
+        )
+        ChatflowHistoryService.save_node_message(
+            prompt_message=(AssistantPromptMessage(content=llm_output)),
+            node_id=self.node_id,
+            conversation_id=conversation_id,
+            app_id=self.app_id,
+            tenant_id=self.tenant_id,
+        )
+
+        memory_config = self._node_data.memory
+        if not memory_config:
+            return
+        block_ids = memory_config.block_id
+        if not block_ids:
+            return
+
+        # FIXME: This is dirty workaround and may cause incorrect resolution for workflow version
+        with Session(db.engine) as session:
+            stmt = select(Workflow).where(Workflow.tenant_id == self.tenant_id, Workflow.app_id == self.app_id)
+            workflow = session.scalars(stmt).first()
+            if not workflow:
+                raise ValueError("Workflow not found.")
+        memory_blocks = workflow.memory_blocks
+
+        for block_id in block_ids:
+            memory_block_spec = next((block for block in memory_blocks if block.id == block_id), None)
+
+            if memory_block_spec and memory_block_spec.scope == MemoryScope.NODE:
+                is_draft = self.invoke_from == InvokeFrom.DEBUGGER
+                from services.chatflow_memory_service import ChatflowMemoryService
+
+                ChatflowMemoryService.update_node_memory_if_needed(
+                    tenant_id=self.tenant_id,
+                    app_id=self.app_id,
+                    node_id=self.id,
+                    conversation_id=conversation_id,
+                    memory_block_spec=memory_block_spec,
+                    variable_pool=variable_pool,
+                    is_draft=is_draft,
+                    created_by=self._get_user_from_context(),
+                )
+
+    def _get_user_from_context(self) -> MemoryCreatedBy:
+        if self.user_from == UserFrom.ACCOUNT:
+            return MemoryCreatedBy(account_id=self.user_id)
+        else:
+            return MemoryCreatedBy(end_user_id=self.user_id)
+
 
 def _combine_message_content_with_role(
     *, contents: str | list[PromptMessageContentUnionTypes] | None = None, role: PromptMessageRole

api/core/workflow/runtime/variable_pool.py

@@ -9,11 +9,12 @@ from pydantic import BaseModel, Field
 from core.file import File, FileAttribute, file_manager
 from core.variables import Segment, SegmentGroup, Variable
 from core.variables.consts import SELECTORS_LENGTH
-from core.variables.segments import FileSegment, ObjectSegment
-from core.variables.variables import RAGPipelineVariableInput, VariableUnion
+from core.variables.segments import FileSegment, ObjectSegment, VersionedMemoryValue
+from core.variables.variables import RAGPipelineVariableInput, VariableUnion, VersionedMemoryVariable
 from core.workflow.constants import (
     CONVERSATION_VARIABLE_NODE_ID,
     ENVIRONMENT_VARIABLE_NODE_ID,
+    MEMORY_BLOCK_VARIABLE_NODE_ID,
     RAG_PIPELINE_VARIABLE_NODE_ID,
     SYSTEM_VARIABLE_NODE_ID,
 )
@@ -56,6 +57,10 @@ class VariablePool(BaseModel):
         description="RAG pipeline variables.",
         default_factory=list,
     )
+    memory_blocks: Mapping[str, str] = Field(
+        description="Memory blocks.",
+        default_factory=dict,
+    )
 
     def model_post_init(self, context: Any, /):
         # Create a mapping from field names to SystemVariableKey enum values
@@ -76,6 +81,18 @@ class VariablePool(BaseModel):
                 rag_pipeline_variables_map[node_id][key] = value
             for key, value in rag_pipeline_variables_map.items():
                 self.add((RAG_PIPELINE_VARIABLE_NODE_ID, key), value)
+        # Add memory blocks to the variable pool
+        for memory_id, memory_value in self.memory_blocks.items():
+            self.add(
+                [MEMORY_BLOCK_VARIABLE_NODE_ID, memory_id],
+                VersionedMemoryVariable(
+                    value=VersionedMemoryValue(
+                        current_value=memory_value,
+                        versions={"1": memory_value},
+                    ),
+                    name=memory_id,
+                )
+            )
 
     def add(self, selector: Sequence[str], value: Any, /):
         """

api/docker/entrypoint.sh

@@ -39,14 +39,16 @@ elif [[ "${MODE}" == "beat" ]]; then
   exec celery -A app.celery beat --loglevel ${LOG_LEVEL:-INFO}
 else
   if [[ "${DEBUG}" == "true" ]]; then
-    exec flask run --host=${DIFY_BIND_ADDRESS:-0.0.0.0} --port=${DIFY_PORT:-5001} --debug
+    export HOST=${DIFY_BIND_ADDRESS:-0.0.0.0}
+    export PORT=${DIFY_PORT:-5001}
+    exec python -m app
   else
     exec gunicorn \
       --bind "${DIFY_BIND_ADDRESS:-0.0.0.0}:${DIFY_PORT:-5001}" \
       --workers ${SERVER_WORKER_AMOUNT:-1} \
-      --worker-class ${SERVER_WORKER_CLASS:-gevent} \
+      --worker-class ${SERVER_WORKER_CLASS:-geventwebsocket.gunicorn.workers.GeventWebSocketWorker} \
       --worker-connections ${SERVER_WORKER_CONNECTIONS:-10} \
       --timeout ${GUNICORN_TIMEOUT:-200} \
-      app:app
+      app:socketio_app
   fi
 fi

api/events/event_handlers/update_provider_when_message_created.py

@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
 
 from configs import dify_config
 from core.app.entities.app_invoke_entities import AgentChatAppGenerateEntity, ChatAppGenerateEntity
-from core.entities.provider_entities import QuotaUnit, SystemConfiguration
+from core.entities.provider_entities import ProviderQuotaType, QuotaUnit, SystemConfiguration
 from events.message_event import message_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client, redis_fallback
@@ -134,22 +134,38 @@ def handle(sender: Message, **kwargs):
             system_configuration=system_configuration,
             model_name=model_config.model,
         )
-
         if used_quota is not None:
-            quota_update = _ProviderUpdateOperation(
-                filters=_ProviderUpdateFilters(
+            if provider_configuration.system_configuration.current_quota_type == ProviderQuotaType.TRIAL:
+                from services.credit_pool_service import CreditPoolService
+
+                CreditPoolService.check_and_deduct_credits(
                     tenant_id=tenant_id,
-                    provider_name=ModelProviderID(model_config.provider).provider_name,
-                    provider_type=ProviderType.SYSTEM,
-                    quota_type=provider_configuration.system_configuration.current_quota_type.value,
-                ),
-                values=_ProviderUpdateValues(quota_used=Provider.quota_used + used_quota, last_used=current_time),
-                additional_filters=_ProviderUpdateAdditionalFilters(
-                    quota_limit_check=True  # Provider.quota_limit > Provider.quota_used
-                ),
-                description="quota_deduction_update",
-            )
-            updates_to_perform.append(quota_update)
+                    credits_required=used_quota,
+                    pool_type="trial",
+                )
+            elif provider_configuration.system_configuration.current_quota_type == ProviderQuotaType.PAID:
+                from services.credit_pool_service import CreditPoolService
+
+                CreditPoolService.check_and_deduct_credits(
+                    tenant_id=tenant_id,
+                    credits_required=used_quota,
+                    pool_type="paid",
+                )
+            else:
+                quota_update = _ProviderUpdateOperation(
+                    filters=_ProviderUpdateFilters(
+                        tenant_id=tenant_id,
+                        provider_name=ModelProviderID(model_config.provider).provider_name,
+                        provider_type=ProviderType.SYSTEM.value,
+                        quota_type=provider_configuration.system_configuration.current_quota_type.value,
+                    ),
+                    values=_ProviderUpdateValues(quota_used=Provider.quota_used + used_quota, last_used=current_time),
+                    additional_filters=_ProviderUpdateAdditionalFilters(
+                        quota_limit_check=True  # Provider.quota_limit > Provider.quota_used
+                    ),
+                    description="quota_deduction_update",
+                )
+                updates_to_perform.append(quota_update)
 
     # Execute all updates
     start_time = time_module.perf_counter()

api/extensions/ext_blueprints.py

@@ -31,7 +31,13 @@ def init_app(app: DifyApp):
         web_bp,
         resources={r"/*": {"origins": dify_config.WEB_API_CORS_ALLOW_ORIGINS}},
         supports_credentials=True,
-        allow_headers=list(AUTHENTICATED_HEADERS),
+        allow_headers=[
+            "Content-Type",
+            "Authorization",
+            HEADER_NAME_APP_CODE,
+            HEADER_NAME_CSRF_TOKEN,
+            HEADER_NAME_PASSPORT,
+        ],
         methods=["GET", "PUT", "POST", "DELETE", "OPTIONS", "PATCH"],
         expose_headers=["X-Version", "X-Env"],
     )

api/extensions/ext_socketio.py

@@ -0,0 +1,3 @@
+import socketio
+
+sio = socketio.Server(async_mode="gevent", cors_allowed_origins="*")

api/factories/variable_factory.py

@@ -21,6 +21,8 @@ from core.variables.segments import (
     ObjectSegment,
     Segment,
     StringSegment,
+    VersionedMemorySegment,
+    VersionedMemoryValue,
 )
 from core.variables.types import SegmentType
 from core.variables.variables import (
@@ -39,6 +41,7 @@ from core.variables.variables import (
     SecretVariable,
     StringVariable,
     Variable,
+    VersionedMemoryVariable,
 )
 from core.workflow.constants import (
     CONVERSATION_VARIABLE_NODE_ID,
@@ -69,6 +72,7 @@ SEGMENT_TO_VARIABLE_MAP = {
     NoneSegment: NoneVariable,
     ObjectSegment: ObjectVariable,
     StringSegment: StringVariable,
+    VersionedMemorySegment: VersionedMemoryVariable
 }
 
 
@@ -193,6 +197,7 @@ _segment_factory: Mapping[SegmentType, type[Segment]] = {
     SegmentType.FILE: FileSegment,
     SegmentType.BOOLEAN: BooleanSegment,
     SegmentType.OBJECT: ObjectSegment,
+    SegmentType.VERSIONED_MEMORY: VersionedMemorySegment,
     # Array types
     SegmentType.ARRAY_ANY: ArrayAnySegment,
     SegmentType.ARRAY_STRING: ArrayStringSegment,
@@ -259,6 +264,12 @@ def build_segment_with_type(segment_type: SegmentType, value: Any) -> Segment:
         else:
             raise TypeMismatchError(f"Type mismatch: expected {segment_type}, but got empty list")
 
+    if segment_type == SegmentType.VERSIONED_MEMORY:
+        return VersionedMemorySegment(
+            value_type=segment_type,
+            value=VersionedMemoryValue.model_validate(value)
+        )
+
     inferred_type = SegmentType.infer_segment_type(value)
     # Type compatibility checking
     if inferred_type is None:

api/fields/online_user_fields.py

@@ -0,0 +1,17 @@
+from flask_restx import fields
+
+online_user_partial_fields = {
+    "user_id": fields.String,
+    "username": fields.String,
+    "avatar": fields.String,
+    "sid": fields.String,
+}
+
+workflow_online_users_fields = {
+    "workflow_id": fields.String,
+    "users": fields.List(fields.Nested(online_user_partial_fields)),
+}
+
+online_user_list_fields = {
+    "data": fields.List(fields.Nested(workflow_online_users_fields)),
+}

api/fields/workflow_comment_fields.py

@@ -0,0 +1,96 @@
+from flask_restx import fields
+
+from libs.helper import AvatarUrlField, TimestampField
+
+# basic account fields for comments
+account_fields = {
+    "id": fields.String,
+    "name": fields.String,
+    "email": fields.String,
+    "avatar_url": AvatarUrlField,
+}
+
+# Comment mention fields
+workflow_comment_mention_fields = {
+    "mentioned_user_id": fields.String,
+    "mentioned_user_account": fields.Nested(account_fields, allow_null=True),
+    "reply_id": fields.String,
+}
+
+# Comment reply fields
+workflow_comment_reply_fields = {
+    "id": fields.String,
+    "content": fields.String,
+    "created_by": fields.String,
+    "created_by_account": fields.Nested(account_fields, allow_null=True),
+    "created_at": TimestampField,
+}
+
+# Basic comment fields (for list views)
+workflow_comment_basic_fields = {
+    "id": fields.String,
+    "position_x": fields.Float,
+    "position_y": fields.Float,
+    "content": fields.String,
+    "created_by": fields.String,
+    "created_by_account": fields.Nested(account_fields, allow_null=True),
+    "created_at": TimestampField,
+    "updated_at": TimestampField,
+    "resolved": fields.Boolean,
+    "resolved_at": TimestampField,
+    "resolved_by": fields.String,
+    "resolved_by_account": fields.Nested(account_fields, allow_null=True),
+    "reply_count": fields.Integer,
+    "mention_count": fields.Integer,
+    "participants": fields.List(fields.Nested(account_fields)),
+}
+
+# Detailed comment fields (for single comment view)
+workflow_comment_detail_fields = {
+    "id": fields.String,
+    "position_x": fields.Float,
+    "position_y": fields.Float,
+    "content": fields.String,
+    "created_by": fields.String,
+    "created_by_account": fields.Nested(account_fields, allow_null=True),
+    "created_at": TimestampField,
+    "updated_at": TimestampField,
+    "resolved": fields.Boolean,
+    "resolved_at": TimestampField,
+    "resolved_by": fields.String,
+    "resolved_by_account": fields.Nested(account_fields, allow_null=True),
+    "replies": fields.List(fields.Nested(workflow_comment_reply_fields)),
+    "mentions": fields.List(fields.Nested(workflow_comment_mention_fields)),
+}
+
+# Comment creation response fields (simplified)
+workflow_comment_create_fields = {
+    "id": fields.String,
+    "created_at": TimestampField,
+}
+
+# Comment update response fields (simplified)
+workflow_comment_update_fields = {
+    "id": fields.String,
+    "updated_at": TimestampField,
+}
+
+# Comment resolve response fields
+workflow_comment_resolve_fields = {
+    "id": fields.String,
+    "resolved": fields.Boolean,
+    "resolved_at": TimestampField,
+    "resolved_by": fields.String,
+}
+
+# Reply creation response fields (simplified)
+workflow_comment_reply_create_fields = {
+    "id": fields.String,
+    "created_at": TimestampField,
+}
+
+# Reply update response fields
+workflow_comment_reply_update_fields = {
+    "id": fields.String,
+    "updated_at": TimestampField,
+}

api/migrations/versions/2025_09_18_1726-227822d22895_add_workflow_comments_table.py

@@ -0,0 +1,90 @@
+"""Add workflow comments table
+
+Revision ID: 227822d22895
+Revises: 68519ad5cd18
+Create Date: 2025-08-22 17:26:15.255980
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '227822d22895'
+down_revision = '68519ad5cd18'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('workflow_comments',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+    sa.Column('app_id', models.types.StringUUID(), nullable=False),
+    sa.Column('position_x', sa.Float(), nullable=False),
+    sa.Column('position_y', sa.Float(), nullable=False),
+    sa.Column('content', sa.Text(), nullable=False),
+    sa.Column('created_by', models.types.StringUUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('resolved', sa.Boolean(), server_default=sa.text('false'), nullable=False),
+    sa.Column('resolved_at', sa.DateTime(), nullable=True),
+    sa.Column('resolved_by', models.types.StringUUID(), nullable=True),
+    sa.PrimaryKeyConstraint('id', name='workflow_comments_pkey')
+    )
+    with op.batch_alter_table('workflow_comments', schema=None) as batch_op:
+        batch_op.create_index('workflow_comments_app_idx', ['tenant_id', 'app_id'], unique=False)
+        batch_op.create_index('workflow_comments_created_at_idx', ['created_at'], unique=False)
+
+    op.create_table('workflow_comment_replies',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('comment_id', models.types.StringUUID(), nullable=False),
+    sa.Column('content', sa.Text(), nullable=False),
+    sa.Column('created_by', models.types.StringUUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.ForeignKeyConstraint(['comment_id'], ['workflow_comments.id'], name=op.f('workflow_comment_replies_comment_id_fkey'), ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='workflow_comment_replies_pkey')
+    )
+    with op.batch_alter_table('workflow_comment_replies', schema=None) as batch_op:
+        batch_op.create_index('comment_replies_comment_idx', ['comment_id'], unique=False)
+        batch_op.create_index('comment_replies_created_at_idx', ['created_at'], unique=False)
+
+    op.create_table('workflow_comment_mentions',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('comment_id', models.types.StringUUID(), nullable=False),
+    sa.Column('reply_id', models.types.StringUUID(), nullable=True),
+    sa.Column('mentioned_user_id', models.types.StringUUID(), nullable=False),
+    sa.ForeignKeyConstraint(['comment_id'], ['workflow_comments.id'], name=op.f('workflow_comment_mentions_comment_id_fkey'), ondelete='CASCADE'),
+    sa.ForeignKeyConstraint(['reply_id'], ['workflow_comment_replies.id'], name=op.f('workflow_comment_mentions_reply_id_fkey'), ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', name='workflow_comment_mentions_pkey')
+    )
+    with op.batch_alter_table('workflow_comment_mentions', schema=None) as batch_op:
+        batch_op.create_index('comment_mentions_comment_idx', ['comment_id'], unique=False)
+        batch_op.create_index('comment_mentions_reply_idx', ['reply_id'], unique=False)
+        batch_op.create_index('comment_mentions_user_idx', ['mentioned_user_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('workflow_comment_mentions', schema=None) as batch_op:
+        batch_op.drop_index('comment_mentions_user_idx')
+        batch_op.drop_index('comment_mentions_reply_idx')
+        batch_op.drop_index('comment_mentions_comment_idx')
+
+    op.drop_table('workflow_comment_mentions')
+    with op.batch_alter_table('workflow_comment_replies', schema=None) as batch_op:
+        batch_op.drop_index('comment_replies_created_at_idx')
+        batch_op.drop_index('comment_replies_comment_idx')
+
+    op.drop_table('workflow_comment_replies')
+    with op.batch_alter_table('workflow_comments', schema=None) as batch_op:
+        batch_op.drop_index('workflow_comments_created_at_idx')
+        batch_op.drop_index('workflow_comments_app_idx')
+
+    op.drop_table('workflow_comments')
+    # ### end Alembic commands ###

api/migrations/versions/2025_09_25_1520-58a70d22fdbd_add_table_credit_pool.py

@@ -0,0 +1,104 @@
+"""add table credit pool
+
+Revision ID: 58a70d22fdbd
+Revises: 68519ad5cd18
+Create Date: 2025-09-25 15:20:40.367078
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '58a70d22fdbd'
+down_revision = '68519ad5cd18'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('tenant_credit_pools',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+    sa.Column('pool_type', sa.String(length=40), nullable=False),
+    sa.Column('quota_limit', sa.BigInteger(), nullable=False),
+    sa.Column('quota_used', sa.BigInteger(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
+    )
+    with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op:
+        batch_op.create_index('tenant_credit_pool_pool_type_idx', ['pool_type'], unique=False)
+        batch_op.create_index('tenant_credit_pool_tenant_id_idx', ['tenant_id'], unique=False)
+    # Data migration: Move quota data from providers to tenant_credit_pools
+    migrate_quota_data()
+    
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op:
+        batch_op.drop_index('tenant_credit_pool_tenant_id_idx')
+        batch_op.drop_index('tenant_credit_pool_pool_type_idx')
+
+    op.drop_table('tenant_credit_pools')
+    # ### end Alembic commands ###
+
+
+def migrate_quota_data():
+    """
+    Migrate quota data from providers table to tenant_credit_pools table
+    for providers with quota_type='trial' or 'paid', provider_name='openai', provider_type='system'
+    """
+    # Create connection
+    bind = op.get_bind()
+    
+    # Define quota type mappings
+    quota_type_mappings = ['trial', 'paid']
+    
+    for quota_type in quota_type_mappings:
+        # Query providers that match the criteria
+        select_sql = sa.text("""
+            SELECT tenant_id, quota_limit, quota_used
+            FROM providers 
+            WHERE quota_type = :quota_type
+            AND provider_name = 'openai' 
+            AND provider_type = 'system'
+            AND quota_limit IS NOT NULL
+        """)
+        
+        result = bind.execute(select_sql, {"quota_type": quota_type})
+        providers_data = result.fetchall()
+        
+        # Insert data into tenant_credit_pools
+        for provider_data in providers_data:
+            tenant_id, quota_limit, quota_used = provider_data
+            
+            # Check if credit pool already exists for this tenant and pool type
+            check_sql = sa.text("""
+                SELECT COUNT(*) 
+                FROM tenant_credit_pools 
+                WHERE tenant_id = :tenant_id AND pool_type = :pool_type
+            """)
+            
+            existing_count = bind.execute(check_sql, {
+                "tenant_id": tenant_id,
+                "pool_type": quota_type
+            }).scalar()
+            
+            if existing_count == 0:
+                # Insert new credit pool record
+                insert_sql = sa.text("""
+                    INSERT INTO tenant_credit_pools (tenant_id, pool_type, quota_limit, quota_used, created_at, updated_at)
+                    VALUES (:tenant_id, :pool_type, :quota_limit, :quota_used, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+                """)
+                
+                bind.execute(insert_sql, {
+                    "tenant_id": tenant_id,
+                    "pool_type": quota_type,
+                    "quota_limit": quota_limit or 0,
+                    "quota_used": quota_used or 0
+                })

api/migrations/versions/2025_10_11_1442-3993fd9e9c2f_add_table_explore_banner_and_trial.py

@@ -0,0 +1,92 @@
+"""add table explore banner and trial
+
+Revision ID: 3993fd9e9c2f
+Revises: 68519ad5cd18
+Create Date: 2025-10-11 14:42:01.954865
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '3993fd9e9c2f'
+down_revision = '68519ad5cd18'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('account_trial_app_records',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('account_id', models.types.StringUUID(), nullable=False),
+    sa.Column('app_id', models.types.StringUUID(), nullable=False),
+    sa.Column('count', sa.Integer(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='user_trial_app_pkey'),
+    sa.UniqueConstraint('account_id', 'app_id', name='unique_account_trial_app_record')
+    )
+    with op.batch_alter_table('account_trial_app_records', schema=None) as batch_op:
+        batch_op.create_index('account_trial_app_record_account_id_idx', ['account_id'], unique=False)
+        batch_op.create_index('account_trial_app_record_app_id_idx', ['app_id'], unique=False)
+
+    op.create_table('exporle_banners',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('content', sa.JSON(), nullable=False),
+    sa.Column('link', sa.String(length=255), nullable=False),
+    sa.Column('sort', sa.Integer(), nullable=False),
+    sa.Column('status', sa.String(length=255), server_default=sa.text("'enabled'::character varying"), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('language', sa.String(length=255), server_default=sa.text("'en-US'::character varying"), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='exporler_banner_pkey')
+    )
+    op.create_table('trial_apps',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('app_id', models.types.StringUUID(), nullable=False),
+    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('trial_limit', sa.Integer(), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='trial_app_pkey'),
+    sa.UniqueConstraint('app_id', name='unique_trail_app_id')
+    )
+    with op.batch_alter_table('trial_apps', schema=None) as batch_op:
+        batch_op.create_index('trial_app_app_id_idx', ['app_id'], unique=False)
+        batch_op.create_index('trial_app_tenant_id_idx', ['tenant_id'], unique=False)
+
+    with op.batch_alter_table('datasource_providers', schema=None) as batch_op:
+        batch_op.alter_column('avatar_url',
+               existing_type=sa.TEXT(),
+               type_=sa.String(length=255),
+               existing_nullable=True)
+
+    with op.batch_alter_table('providers', schema=None) as batch_op:
+        batch_op.drop_column('credential_status')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('providers', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('credential_status', sa.VARCHAR(length=20), server_default=sa.text("'active'::character varying"), autoincrement=False, nullable=True))
+
+    with op.batch_alter_table('datasource_providers', schema=None) as batch_op:
+        batch_op.alter_column('avatar_url',
+               existing_type=sa.String(length=255),
+               type_=sa.TEXT(),
+               existing_nullable=True)
+
+    with op.batch_alter_table('trial_apps', schema=None) as batch_op:
+        batch_op.drop_index('trial_app_tenant_id_idx')
+        batch_op.drop_index('trial_app_app_id_idx')
+
+    op.drop_table('trial_apps')
+    op.drop_table('exporle_banners')
+    with op.batch_alter_table('account_trial_app_records', schema=None) as batch_op:
+        batch_op.drop_index('account_trial_app_record_app_id_idx')
+        batch_op.drop_index('account_trial_app_record_account_id_idx')
+
+    op.drop_table('account_trial_app_records')
+    # ### end Alembic commands ###

api/migrations/versions/2025_10_11_1529-d00b2b40ea3e_add_chatflow_memory_tables.py

@@ -0,0 +1,104 @@
+"""add_chatflow_memory_tables
+
+Revision ID: d00b2b40ea3e
+Revises: 68519ad5cd18
+Create Date: 2025-10-11 15:29:20.244675
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd00b2b40ea3e'
+down_revision = '68519ad5cd18'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('chatflow_conversations',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+    sa.Column('app_id', models.types.StringUUID(), nullable=False),
+    sa.Column('node_id', sa.Text(), nullable=True),
+    sa.Column('original_conversation_id', models.types.StringUUID(), nullable=True),
+    sa.Column('conversation_metadata', sa.Text(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='chatflow_conversations_pkey')
+    )
+    with op.batch_alter_table('chatflow_conversations', schema=None) as batch_op:
+        batch_op.create_index('chatflow_conversations_original_conversation_id_idx', ['tenant_id', 'app_id', 'node_id', 'original_conversation_id'], unique=False)
+
+    op.create_table('chatflow_memory_variables',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+    sa.Column('app_id', models.types.StringUUID(), nullable=True),
+    sa.Column('conversation_id', models.types.StringUUID(), nullable=True),
+    sa.Column('node_id', sa.Text(), nullable=True),
+    sa.Column('memory_id', sa.Text(), nullable=False),
+    sa.Column('value', sa.Text(), nullable=False),
+    sa.Column('name', sa.Text(), nullable=False),
+    sa.Column('scope', sa.String(length=10), nullable=False),
+    sa.Column('term', sa.String(length=20), nullable=False),
+    sa.Column('version', sa.Integer(), nullable=False),
+    sa.Column('created_by_role', sa.String(length=20), nullable=False),
+    sa.Column('created_by', models.types.StringUUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='chatflow_memory_variables_pkey')
+    )
+    with op.batch_alter_table('chatflow_memory_variables', schema=None) as batch_op:
+        batch_op.create_index('chatflow_memory_variables_memory_id_idx', ['tenant_id', 'app_id', 'node_id', 'memory_id'], unique=False)
+
+    op.create_table('chatflow_messages',
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('conversation_id', models.types.StringUUID(), nullable=False),
+    sa.Column('index', sa.Integer(), nullable=False),
+    sa.Column('version', sa.Integer(), nullable=False),
+    sa.Column('data', sa.Text(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='chatflow_messages_pkey')
+    )
+    with op.batch_alter_table('chatflow_messages', schema=None) as batch_op:
+        batch_op.create_index('chatflow_messages_version_idx', ['conversation_id', 'index', 'version'], unique=False)
+
+    with op.batch_alter_table('datasource_providers', schema=None) as batch_op:
+        batch_op.alter_column('avatar_url',
+               existing_type=sa.TEXT(),
+               type_=sa.String(length=255),
+               existing_nullable=True)
+
+    with op.batch_alter_table('providers', schema=None) as batch_op:
+        batch_op.drop_column('credential_status')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('providers', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('credential_status', sa.VARCHAR(length=20), server_default=sa.text("'active'::character varying"), autoincrement=False, nullable=True))
+
+    with op.batch_alter_table('datasource_providers', schema=None) as batch_op:
+        batch_op.alter_column('avatar_url',
+               existing_type=sa.String(length=255),
+               type_=sa.TEXT(),
+               existing_nullable=True)
+
+    with op.batch_alter_table('chatflow_messages', schema=None) as batch_op:
+        batch_op.drop_index('chatflow_messages_version_idx')
+
+    op.drop_table('chatflow_messages')
+    with op.batch_alter_table('chatflow_memory_variables', schema=None) as batch_op:
+        batch_op.drop_index('chatflow_memory_variables_memory_id_idx')
+
+    op.drop_table('chatflow_memory_variables')
+    with op.batch_alter_table('chatflow_conversations', schema=None) as batch_op:
+        batch_op.drop_index('chatflow_conversations_original_conversation_id_idx')
+
+    op.drop_table('chatflow_conversations')
+    # ### end Alembic commands ###

api/migrations/versions/2025_10_14_1618-d98acf217d43_add_app_mode_for_messsage.py

@@ -22,6 +22,55 @@ def upgrade():
         batch_op.add_column(sa.Column('app_mode', sa.String(length=255), nullable=True))
         batch_op.create_index('message_app_mode_idx', ['app_mode'], unique=False)
 
+    conn = op.get_bind()
+
+    # Strategy: Update in batches to minimize lock time
+    # For large tables (millions of rows), this prevents long-running transactions
+    batch_size = 10000
+
+    print("Starting backfill of app_mode from conversations...")
+
+    # Use a more efficient UPDATE with JOIN
+    # This query updates messages.app_mode from conversations.mode
+    # Using string formatting for LIMIT since it's a constant
+    update_query = f"""
+        UPDATE messages m
+        SET app_mode = c.mode
+        FROM conversations c
+        WHERE m.conversation_id = c.id
+          AND m.app_mode IS NULL
+          AND m.id IN (
+              SELECT id FROM messages
+              WHERE app_mode IS NULL
+              LIMIT {batch_size}
+          )
+    """
+
+    # Execute batched updates
+    total_updated = 0
+    iteration = 0
+    while True:
+        iteration += 1
+        result = conn.execute(sa.text(update_query))
+
+        # Check if result is None or has no rowcount
+        if result is None:
+            print("Warning: Query returned None, stopping backfill")
+            break
+
+        rows_updated = result.rowcount if hasattr(result, 'rowcount') else 0
+        total_updated += rows_updated
+
+        if rows_updated == 0:
+            break
+
+        print(f"Iteration {iteration}: Updated {rows_updated} messages (total: {total_updated})")
+
+        # For very large tables, add a small delay to reduce load
+        # Uncomment if needed: import time; time.sleep(0.1)
+
+    print(f"Backfill completed. Total messages updated: {total_updated}")
+
     # ### end Alembic commands ###
 
 

api/models/__init__.py

@@ -9,6 +9,12 @@ from .account import (
     TenantStatus,
 )
 from .api_based_extension import APIBasedExtension, APIBasedExtensionPoint
+from .chatflow_memory import ChatflowConversation, ChatflowMemoryVariable, ChatflowMessage
+from .comment import (
+    WorkflowComment,
+    WorkflowCommentMention,
+    WorkflowCommentReply,
+)
 from .dataset import (
     AppDatasetJoin,
     Dataset,
@@ -28,6 +34,7 @@ from .dataset import (
 )
 from .enums import CreatorUserRole, UserFrom, WorkflowRunTriggeredFrom
 from .model import (
+    AccountTrialAppRecord,
     ApiRequest,
     ApiToken,
     App,
@@ -40,6 +47,7 @@ from .model import (
     DatasetRetrieverResource,
     DifySetup,
     EndUser,
+    ExporleBanner,
     IconType,
     InstalledApp,
     Message,
@@ -53,7 +61,9 @@ from .model import (
     Site,
     Tag,
     TagBinding,
+    TenantCreditPool,
     TraceAppConfig,
+    TrialApp,
     UploadFile,
 )
 from .oauth import DatasourceOauthParamConfig, DatasourceProvider
@@ -99,6 +109,7 @@ __all__ = [
     "Account",
     "AccountIntegrate",
     "AccountStatus",
+    "AccountTrialAppRecord",
     "ApiRequest",
     "ApiToken",
     "ApiToolProvider",
@@ -112,6 +123,9 @@ __all__ = [
     "BuiltinToolProvider",
     "CeleryTask",
     "CeleryTaskSet",
+    "ChatflowConversation",
+    "ChatflowMemoryVariable",
+    "ChatflowMessage",
     "Conversation",
     "ConversationVariable",
     "CreatorUserRole",
@@ -132,6 +146,7 @@ __all__ = [
     "DocumentSegment",
     "Embedding",
     "EndUser",
+    "ExporleBanner",
     "ExternalKnowledgeApis",
     "ExternalKnowledgeBindings",
     "IconType",
@@ -160,6 +175,7 @@ __all__ = [
     "Tenant",
     "TenantAccountJoin",
     "TenantAccountRole",
+    "TenantCreditPool",
     "TenantDefaultModel",
     "TenantPreferredModelProvider",
     "TenantStatus",
@@ -169,12 +185,16 @@ __all__ = [
     "ToolLabelBinding",
     "ToolModelInvoke",
     "TraceAppConfig",
+    "TrialApp",
     "UploadFile",
     "UserFrom",
     "Whitelist",
     "Workflow",
     "WorkflowAppLog",
     "WorkflowAppLogCreatedFrom",
+    "WorkflowComment",
+    "WorkflowCommentMention",
+    "WorkflowCommentReply",
     "WorkflowNodeExecutionModel",
     "WorkflowNodeExecutionOffload",
     "WorkflowNodeExecutionTriggeredFrom",

api/models/chatflow_memory.py

@@ -0,0 +1,76 @@
+from datetime import datetime
+
+import sqlalchemy as sa
+from sqlalchemy import DateTime, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from .base import Base
+from .types import StringUUID
+
+
+class ChatflowMemoryVariable(Base):
+    __tablename__ = "chatflow_memory_variables"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="chatflow_memory_variables_pkey"),
+        sa.Index("chatflow_memory_variables_memory_id_idx", "tenant_id", "app_id", "node_id", "memory_id"),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    app_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
+    conversation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
+    node_id: Mapped[str | None] = mapped_column(sa.Text, nullable=True)
+    memory_id: Mapped[str] = mapped_column(sa.Text, nullable=False)
+    value: Mapped[str] = mapped_column(sa.Text, nullable=False)
+    name: Mapped[str] = mapped_column(sa.Text, nullable=False)
+    scope: Mapped[str] = mapped_column(sa.String(10), nullable=False)  # 'app' or 'node'
+    term: Mapped[str] = mapped_column(sa.String(20), nullable=False)  # 'session' or 'persistent'
+    version: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=1)
+    created_by_role: Mapped[str] = mapped_column(sa.String(20))  # 'end_user' or 'account`
+    created_by: Mapped[str] = mapped_column(StringUUID)
+
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )
+
+
+class ChatflowConversation(Base):
+    __tablename__ = "chatflow_conversations"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="chatflow_conversations_pkey"),
+        sa.Index(
+            "chatflow_conversations_original_conversation_id_idx",
+            "tenant_id", "app_id", "node_id", "original_conversation_id"
+        ),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    node_id: Mapped[str | None] = mapped_column(sa.Text, nullable=True)
+    original_conversation_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
+    conversation_metadata: Mapped[str] = mapped_column(sa.Text, nullable=False)  # JSON
+
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )
+
+
+class ChatflowMessage(Base):
+    __tablename__ = "chatflow_messages"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="chatflow_messages_pkey"),
+        sa.Index("chatflow_messages_version_idx", "conversation_id", "index", "version"),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
+    conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    index: Mapped[int] = mapped_column(sa.Integer, nullable=False)  # This index starts from 0
+    version: Mapped[int] = mapped_column(sa.Integer, nullable=False)
+    data: Mapped[str] = mapped_column(sa.Text, nullable=False)  # Serialized PromptMessage JSON
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )

api/models/comment.py

@@ -0,0 +1,189 @@
+"""Workflow comment models."""
+
+from datetime import datetime
+from typing import TYPE_CHECKING, Optional
+
+from sqlalchemy import Index, func
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from .account import Account
+from .base import Base
+from .engine import db
+from .types import StringUUID
+
+if TYPE_CHECKING:
+    pass
+
+
+class WorkflowComment(Base):
+    """Workflow comment model for canvas commenting functionality.
+
+    Comments are associated with apps rather than specific workflow versions,
+    since an app has only one draft workflow at a time and comments should persist
+    across workflow version changes.
+
+    Attributes:
+        id: Comment ID
+        tenant_id: Workspace ID
+        app_id: App ID (primary association, comments belong to apps)
+        position_x: X coordinate on canvas
+        position_y: Y coordinate on canvas
+        content: Comment content
+        created_by: Creator account ID
+        created_at: Creation time
+        updated_at: Last update time
+        resolved: Whether comment is resolved
+        resolved_at: Resolution time
+        resolved_by: Resolver account ID
+    """
+
+    __tablename__ = "workflow_comments"
+    __table_args__ = (
+        db.PrimaryKeyConstraint("id", name="workflow_comments_pkey"),
+        Index("workflow_comments_app_idx", "tenant_id", "app_id"),
+        Index("workflow_comments_created_at_idx", "created_at"),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    position_x: Mapped[float] = mapped_column(db.Float)
+    position_y: Mapped[float] = mapped_column(db.Float)
+    content: Mapped[str] = mapped_column(db.Text, nullable=False)
+    created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at: Mapped[datetime] = mapped_column(
+        db.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )
+    resolved: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    resolved_at: Mapped[Optional[datetime]] = mapped_column(db.DateTime)
+    resolved_by: Mapped[Optional[str]] = mapped_column(StringUUID)
+
+    # Relationships
+    replies: Mapped[list["WorkflowCommentReply"]] = relationship(
+        "WorkflowCommentReply", back_populates="comment", cascade="all, delete-orphan"
+    )
+    mentions: Mapped[list["WorkflowCommentMention"]] = relationship(
+        "WorkflowCommentMention", back_populates="comment", cascade="all, delete-orphan"
+    )
+
+    @property
+    def created_by_account(self):
+        """Get creator account."""
+        return db.session.get(Account, self.created_by)
+
+    @property
+    def resolved_by_account(self):
+        """Get resolver account."""
+        if self.resolved_by:
+            return db.session.get(Account, self.resolved_by)
+        return None
+
+    @property
+    def reply_count(self):
+        """Get reply count."""
+        return len(self.replies)
+
+    @property
+    def mention_count(self):
+        """Get mention count."""
+        return len(self.mentions)
+
+    @property
+    def participants(self):
+        """Get all participants (creator + repliers + mentioned users)."""
+        participant_ids = set()
+
+        # Add comment creator
+        participant_ids.add(self.created_by)
+
+        # Add reply creators
+        participant_ids.update(reply.created_by for reply in self.replies)
+
+        # Add mentioned users
+        participant_ids.update(mention.mentioned_user_id for mention in self.mentions)
+
+        # Get account objects
+        participants = []
+        for user_id in participant_ids:
+            account = db.session.get(Account, user_id)
+            if account:
+                participants.append(account)
+
+        return participants
+
+
+class WorkflowCommentReply(Base):
+    """Workflow comment reply model.
+
+    Attributes:
+        id: Reply ID
+        comment_id: Parent comment ID
+        content: Reply content
+        created_by: Creator account ID
+        created_at: Creation time
+    """
+
+    __tablename__ = "workflow_comment_replies"
+    __table_args__ = (
+        db.PrimaryKeyConstraint("id", name="workflow_comment_replies_pkey"),
+        Index("comment_replies_comment_idx", "comment_id"),
+        Index("comment_replies_created_at_idx", "created_at"),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    comment_id: Mapped[str] = mapped_column(
+        StringUUID, db.ForeignKey("workflow_comments.id", ondelete="CASCADE"), nullable=False
+    )
+    content: Mapped[str] = mapped_column(db.Text, nullable=False)
+    created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at: Mapped[datetime] = mapped_column(
+        db.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )
+    # Relationships
+    comment: Mapped["WorkflowComment"] = relationship("WorkflowComment", back_populates="replies")
+
+    @property
+    def created_by_account(self):
+        """Get creator account."""
+        return db.session.get(Account, self.created_by)
+
+
+class WorkflowCommentMention(Base):
+    """Workflow comment mention model.
+
+    Mentions are only for internal accounts since end users
+    cannot access workflow canvas and commenting features.
+
+    Attributes:
+        id: Mention ID
+        comment_id: Parent comment ID
+        mentioned_user_id: Mentioned account ID
+    """
+
+    __tablename__ = "workflow_comment_mentions"
+    __table_args__ = (
+        db.PrimaryKeyConstraint("id", name="workflow_comment_mentions_pkey"),
+        Index("comment_mentions_comment_idx", "comment_id"),
+        Index("comment_mentions_reply_idx", "reply_id"),
+        Index("comment_mentions_user_idx", "mentioned_user_id"),
+    )
+
+    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    comment_id: Mapped[str] = mapped_column(
+        StringUUID, db.ForeignKey("workflow_comments.id", ondelete="CASCADE"), nullable=False
+    )
+    reply_id: Mapped[Optional[str]] = mapped_column(
+        StringUUID, db.ForeignKey("workflow_comment_replies.id", ondelete="CASCADE"), nullable=True
+    )
+    mentioned_user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
+
+    # Relationships
+    comment: Mapped["WorkflowComment"] = relationship("WorkflowComment", back_populates="mentions")
+    reply: Mapped[Optional["WorkflowCommentReply"]] = relationship("WorkflowCommentReply")
+
+    @property
+    def mentioned_user_account(self):
+        """Get mentioned account."""
+        return db.session.get(Account, self.mentioned_user_id)

api/models/enums.py

@@ -23,6 +23,7 @@ class DraftVariableType(StrEnum):
     NODE = "node"
     SYS = "sys"
     CONVERSATION = "conversation"
+    MEMORY_BLOCK = "memory_block"
 
 
 class MessageStatus(StrEnum):

api/models/model.py

@@ -9,8 +9,8 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, cast
 
 import sqlalchemy as sa
 from flask import request
-from flask_login import UserMixin
-from sqlalchemy import Float, Index, PrimaryKeyConstraint, String, exists, func, select, text
+from flask_login import UserMixin  # type: ignore[import-untyped]
+from sqlalchemy import BigInteger, Float, Index, PrimaryKeyConstraint, String, exists, func, select, text
 from sqlalchemy.orm import Mapped, Session, mapped_column
 
 from configs import dify_config
@@ -582,6 +582,64 @@ class InstalledApp(Base):
         return tenant
 
 
+class TrialApp(Base):
+    __tablename__ = "trial_apps"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="trial_app_pkey"),
+        sa.Index("trial_app_app_id_idx", "app_id"),
+        sa.Index("trial_app_tenant_id_idx", "tenant_id"),
+        sa.UniqueConstraint("app_id", name="unique_trail_app_id"),
+    )
+
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
+    app_id = mapped_column(StringUUID, nullable=False)
+    tenant_id = mapped_column(StringUUID, nullable=False)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    trial_limit = mapped_column(sa.Integer, nullable=False, default=3)
+
+    @property
+    def app(self) -> App | None:
+        app = db.session.query(App).where(App.id == self.app_id).first()
+        return app
+
+
+class AccountTrialAppRecord(Base):
+    __tablename__ = "account_trial_app_records"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="user_trial_app_pkey"),
+        sa.Index("account_trial_app_record_account_id_idx", "account_id"),
+        sa.Index("account_trial_app_record_app_id_idx", "app_id"),
+        sa.UniqueConstraint("account_id", "app_id", name="unique_account_trial_app_record"),
+    )
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
+    account_id = mapped_column(StringUUID, nullable=False)
+    app_id = mapped_column(StringUUID, nullable=False)
+    count = mapped_column(sa.Integer, nullable=False, default=0)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+
+    @property
+    def app(self) -> App | None:
+        app = db.session.query(App).where(App.id == self.app_id).first()
+        return app
+
+    @property
+    def user(self) -> Account | None:
+        user = db.session.query(Account).where(Account.id == self.account_id).first()
+        return user
+
+
+class ExporleBanner(Base):
+    __tablename__ = "exporle_banners"
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="exporler_banner_pkey"),)
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
+    content = mapped_column(sa.JSON, nullable=False)
+    link = mapped_column(String(255), nullable=False)
+    sort = mapped_column(sa.Integer, nullable=False)
+    status = mapped_column(sa.String(255), nullable=False, server_default=sa.text("'enabled'::character varying"))
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    language = mapped_column(String(255), nullable=False, server_default=sa.text("'en-US'::character varying"))
+
+
 class OAuthProviderApp(Base):
     """
     Globally shared OAuth provider app information.
@@ -1957,3 +2015,29 @@ class TraceAppConfig(Base):
             "created_at": str(self.created_at) if self.created_at else None,
             "updated_at": str(self.updated_at) if self.updated_at else None,
         }
+
+
+class TenantCreditPool(Base):
+    __tablename__ = "tenant_credit_pools"
+    __table_args__ = (
+        sa.PrimaryKeyConstraint("id", name="tenant_credit_pool_pkey"),
+        sa.Index("tenant_credit_pool_tenant_id_idx", "tenant_id"),
+        sa.Index("tenant_credit_pool_pool_type_idx", "pool_type"),
+    )
+
+    id = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()"))
+    tenant_id = mapped_column(StringUUID, nullable=False)
+    pool_type = mapped_column(String(40), nullable=False, default="trial", server_default="trial")
+    quota_limit = mapped_column(BigInteger, nullable=False, default=0)
+    quota_used = mapped_column(BigInteger, nullable=False, default=0)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=text("CURRENT_TIMESTAMP"))
+    updated_at = mapped_column(
+        sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+    )
+
+    @property
+    def remaining_credits(self) -> int:
+        return max(0, self.quota_limit - self.quota_used)
+
+    def has_sufficient_credits(self, required_credits: int) -> bool:
+        return self.remaining_credits >= required_credits

api/models/workflow.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import uuid
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from enum import StrEnum
@@ -11,10 +12,13 @@ from sqlalchemy import DateTime, Select, exists, orm, select
 
 from core.file.constants import maybe_file_object
 from core.file.models import File
+from core.memory.entities import MemoryBlockSpec
 from core.variables import utils as variable_utils
+from core.variables.segments import VersionedMemoryValue
 from core.variables.variables import FloatVariable, IntegerVariable, StringVariable
 from core.workflow.constants import (
     CONVERSATION_VARIABLE_NODE_ID,
+    MEMORY_BLOCK_VARIABLE_NODE_ID,
     SYSTEM_VARIABLE_NODE_ID,
 )
 from core.workflow.enums import NodeType, WorkflowExecutionStatus
@@ -152,6 +156,9 @@ class Workflow(Base):
     _rag_pipeline_variables: Mapped[str] = mapped_column(
         "rag_pipeline_variables", db.Text, nullable=False, server_default="{}"
     )
+    _memory_blocks: Mapped[str] = mapped_column(
+        "memory_blocks", sa.Text, nullable=False, server_default="[]"
+    )
 
     VERSION_DRAFT = "draft"
 
@@ -169,6 +176,7 @@ class Workflow(Base):
         environment_variables: Sequence[Variable],
         conversation_variables: Sequence[Variable],
         rag_pipeline_variables: list[dict],
+        memory_blocks: Sequence[MemoryBlockSpec] | None = None,
         marked_name: str = "",
         marked_comment: str = "",
     ) -> "Workflow":
@@ -184,6 +192,7 @@ class Workflow(Base):
         workflow.environment_variables = environment_variables or []
         workflow.conversation_variables = conversation_variables or []
         workflow.rag_pipeline_variables = rag_pipeline_variables or []
+        workflow.memory_blocks = memory_blocks or []
         workflow.marked_name = marked_name
         workflow.marked_comment = marked_comment
         workflow.created_at = naive_utc_now()
@@ -343,7 +352,7 @@ class Workflow(Base):
 
         :return: hash
         """
-        entity = {"graph": self.graph_dict, "features": self.features_dict}
+        entity = {"graph": self.graph_dict}
 
         return helper.generate_text_hash(json.dumps(entity, sort_keys=True))
 
@@ -452,7 +461,7 @@ class Workflow(Base):
             "features": self.features_dict,
             "environment_variables": [var.model_dump(mode="json") for var in environment_variables],
             "conversation_variables": [var.model_dump(mode="json") for var in self.conversation_variables],
-            "rag_pipeline_variables": self.rag_pipeline_variables,
+            "memory_blocks": [block.model_dump(mode="json") for block in self.memory_blocks],
         }
         return result
 
@@ -490,6 +499,27 @@ class Workflow(Base):
             ensure_ascii=False,
         )
 
+    @property
+    def memory_blocks(self) -> Sequence[MemoryBlockSpec]:
+        """Memory blocks configuration stored in database"""
+        if self._memory_blocks is None or self._memory_blocks == "":
+            self._memory_blocks = "[]"
+
+        memory_blocks_list: list[dict[str, Any]] = json.loads(self._memory_blocks)
+        results = [MemoryBlockSpec.model_validate(config) for config in memory_blocks_list]
+        return results
+
+    @memory_blocks.setter
+    def memory_blocks(self, value: Sequence[MemoryBlockSpec]):
+        if not value:
+            self._memory_blocks = "[]"
+            return
+
+        self._memory_blocks = json.dumps(
+            [block.model_dump() for block in value],
+            ensure_ascii=False,
+        )
+
     @staticmethod
     def version_from_datetime(d: datetime) -> str:
         return str(d)
@@ -1521,6 +1551,31 @@ class WorkflowDraftVariable(Base):
         variable.editable = editable
         return variable
 
+    @staticmethod
+    def new_memory_block_variable(
+        *,
+        app_id: str,
+        node_id: str | None = None,
+        memory_id: str,
+        name: str,
+        value: VersionedMemoryValue,
+        description: str = "",
+    ) -> "WorkflowDraftVariable":
+        """Create a new memory block draft variable."""
+        return WorkflowDraftVariable(
+            id=str(uuid.uuid4()),
+            app_id=app_id,
+            node_id=MEMORY_BLOCK_VARIABLE_NODE_ID,
+            name=name,
+            value=value.model_dump_json(),
+            description=description,
+            selector=[MEMORY_BLOCK_VARIABLE_NODE_ID, memory_id] if node_id is None else
+            [MEMORY_BLOCK_VARIABLE_NODE_ID, memory_id, node_id],
+            value_type=SegmentType.VERSIONED_MEMORY,
+            visible=True,
+            editable=True,
+        )
+
     @property
     def edited(self):
         return self.last_edited_at is not None

api/pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     "flask-orjson~=2.0.0",
     "flask-sqlalchemy~=3.1.1",
     "gevent~=25.9.1",
+    "gevent-websocket~=0.10.1",
     "gmpy2~=2.2.1",
     "google-api-core==2.18.0",
     "google-api-python-client==2.90.0",
@@ -69,6 +70,7 @@ dependencies = [
     "pypdfium2==4.30.0",
     "python-docx~=1.1.0",
     "python-dotenv==1.0.1",
+    "python-socketio~=5.13.0",
     "pyyaml~=6.0.1",
     "readabilipy~=0.3.0",
     "redis[hiredis]~=6.1.0",

api/services/account_service.py

@@ -999,6 +999,11 @@ class TenantService:
 
         tenant.encrypt_public_key = generate_key_pair(tenant.id)
         db.session.commit()
+
+        from services.credit_pool_service import CreditPoolService
+
+        CreditPoolService.create_default_pool(tenant.id)
+
         return tenant
 
     @staticmethod

api/services/chatflow_history_service.py

@@ -0,0 +1,237 @@
+import json
+from collections.abc import MutableMapping, Sequence
+from typing import Literal, Optional, overload
+
+from sqlalchemy import Row, Select, and_, func, select
+from sqlalchemy.orm import Session
+
+from core.memory.entities import ChatflowConversationMetadata
+from core.model_runtime.entities.message_entities import (
+    PromptMessage,
+)
+from extensions.ext_database import db
+from models.chatflow_memory import ChatflowConversation, ChatflowMessage
+
+
+class ChatflowHistoryService:
+
+    @staticmethod
+    def get_visible_chat_history(
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None,
+        max_visible_count: Optional[int] = None
+    ) -> Sequence[PromptMessage]:
+        with Session(db.engine) as session:
+            chatflow_conv = ChatflowHistoryService._get_or_create_chatflow_conversation(
+                session, conversation_id, app_id, tenant_id, node_id, create_if_missing=False
+            )
+
+            if not chatflow_conv:
+                return []
+
+            metadata = ChatflowConversationMetadata.model_validate_json(chatflow_conv.conversation_metadata)
+            visible_count: int = max_visible_count or metadata.visible_count
+
+            stmt = select(ChatflowMessage).where(
+                ChatflowMessage.conversation_id == chatflow_conv.id
+            ).order_by(ChatflowMessage.index.asc(), ChatflowMessage.version.desc())
+            raw_messages: Sequence[Row[tuple[ChatflowMessage]]] = session.execute(stmt).all()
+            sorted_messages = ChatflowHistoryService._filter_latest_messages(
+                [it[0] for it in raw_messages]
+            )
+            visible_count = min(visible_count, len(sorted_messages))
+            visible_messages = sorted_messages[-visible_count:]
+            return [PromptMessage.model_validate_json(it.data) for it in visible_messages]
+
+    @staticmethod
+    def save_message(
+        prompt_message: PromptMessage,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None
+    ) -> None:
+        with Session(db.engine) as session:
+            chatflow_conv = ChatflowHistoryService._get_or_create_chatflow_conversation(
+                session, conversation_id, app_id, tenant_id, node_id, create_if_missing=True
+            )
+
+            # Get next index
+            max_index = session.execute(
+                select(func.max(ChatflowMessage.index)).where(
+                    ChatflowMessage.conversation_id == chatflow_conv.id
+                )
+            ).scalar() or -1
+            next_index = max_index + 1
+
+            # Save new message to append-only table
+            new_message = ChatflowMessage(
+                conversation_id=chatflow_conv.id,
+                index=next_index,
+                version=1,
+                data=json.dumps(prompt_message)
+            )
+            session.add(new_message)
+            session.commit()
+
+            # 添加：每次保存消息后简单增长visible_count
+            current_metadata = ChatflowConversationMetadata.model_validate_json(chatflow_conv.conversation_metadata)
+            new_visible_count = current_metadata.visible_count + 1
+            new_metadata = ChatflowConversationMetadata(visible_count=new_visible_count)
+            chatflow_conv.conversation_metadata = new_metadata.model_dump_json()
+
+    @staticmethod
+    def save_app_message(
+        prompt_message: PromptMessage,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str
+    ) -> None:
+        """Save PromptMessage to app-level chatflow conversation."""
+        ChatflowHistoryService.save_message(
+            prompt_message=prompt_message,
+            conversation_id=conversation_id,
+            app_id=app_id,
+            tenant_id=tenant_id,
+            node_id=None
+        )
+
+    @staticmethod
+    def save_node_message(
+        prompt_message: PromptMessage,
+        node_id: str,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str
+    ) -> None:
+        ChatflowHistoryService.save_message(
+            prompt_message=prompt_message,
+            conversation_id=conversation_id,
+            app_id=app_id,
+            tenant_id=tenant_id,
+            node_id=node_id
+        )
+
+    @staticmethod
+    def update_visible_count(
+        conversation_id: str,
+        node_id: Optional[str],
+        new_visible_count: int,
+        app_id: str,
+        tenant_id: str
+    ) -> None:
+        with Session(db.engine) as session:
+            chatflow_conv = ChatflowHistoryService._get_or_create_chatflow_conversation(
+                session, conversation_id, app_id, tenant_id, node_id, create_if_missing=True
+            )
+
+            # Only update visible_count in metadata, do not delete any data
+            new_metadata = ChatflowConversationMetadata(visible_count=new_visible_count)
+            chatflow_conv.conversation_metadata = new_metadata.model_dump_json()
+
+            session.commit()
+
+    @staticmethod
+    def get_conversation_metadata(
+        tenant_id: str,
+        app_id: str,
+        conversation_id: str,
+        node_id: Optional[str]
+    ) -> ChatflowConversationMetadata:
+        with Session(db.engine) as session:
+            chatflow_conv = ChatflowHistoryService._get_or_create_chatflow_conversation(
+                session, conversation_id, app_id, tenant_id, node_id, create_if_missing=False
+            )
+            if not chatflow_conv:
+                raise ValueError(f"Conversation not found: {conversation_id}")
+            return ChatflowConversationMetadata.model_validate_json(chatflow_conv.conversation_metadata)
+
+    @staticmethod
+    def _filter_latest_messages(raw_messages: Sequence[ChatflowMessage]) -> Sequence[ChatflowMessage]:
+        index_to_message: MutableMapping[int, ChatflowMessage] = {}
+        for msg in raw_messages:
+            index = msg.index
+            if index not in index_to_message or msg.version > index_to_message[index].version:
+                index_to_message[index] = msg
+
+        sorted_messages = sorted(index_to_message.values(), key=lambda m: m.index)
+        return sorted_messages
+
+    @overload
+    @staticmethod
+    def _get_or_create_chatflow_conversation(
+        session: Session,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None,
+        create_if_missing: Literal[True] = True
+    ) -> ChatflowConversation: ...
+
+    @overload
+    @staticmethod
+    def _get_or_create_chatflow_conversation(
+        session: Session,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None,
+        create_if_missing: Literal[False] = False
+    ) -> Optional[ChatflowConversation]: ...
+
+    @overload
+    @staticmethod
+    def _get_or_create_chatflow_conversation(
+        session: Session,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None,
+        create_if_missing: bool = False
+    ) -> Optional[ChatflowConversation]: ...
+
+    @staticmethod
+    def _get_or_create_chatflow_conversation(
+        session: Session,
+        conversation_id: str,
+        app_id: str,
+        tenant_id: str,
+        node_id: Optional[str] = None,
+        create_if_missing: bool = False
+    ) -> Optional[ChatflowConversation]:
+        """Get existing chatflow conversation or optionally create new one"""
+        stmt: Select[tuple[ChatflowConversation]] = select(ChatflowConversation).where(
+            and_(
+                ChatflowConversation.original_conversation_id == conversation_id,
+                ChatflowConversation.tenant_id == tenant_id,
+                ChatflowConversation.app_id == app_id
+            )
+        )
+
+        if node_id:
+            stmt = stmt.where(ChatflowConversation.node_id == node_id)
+        else:
+            stmt = stmt.where(ChatflowConversation.node_id.is_(None))
+
+        chatflow_conv: Row[tuple[ChatflowConversation]] | None = session.execute(stmt).first()
+
+        if chatflow_conv:
+            result: ChatflowConversation = chatflow_conv[0]  # Extract the ChatflowConversation object
+            return result
+        else:
+            if create_if_missing:
+                # Create a new chatflow conversation
+                default_metadata = ChatflowConversationMetadata(visible_count=0)
+                new_chatflow_conv = ChatflowConversation(
+                    tenant_id=tenant_id,
+                    app_id=app_id,
+                    node_id=node_id,
+                    original_conversation_id=conversation_id,
+                    conversation_metadata=default_metadata.model_dump_json(),
+                )
+                session.add(new_chatflow_conv)
+                session.flush()  # Obtain ID
+                return new_chatflow_conv
+            return None

api/services/chatflow_memory_service.py

@@ -0,0 +1,670 @@
+import logging
+import threading
+import time
+from collections.abc import Sequence
+from typing import Optional
+
+from sqlalchemy import and_, delete, select
+from sqlalchemy.orm import Session
+
+from core.llm_generator.llm_generator import LLMGenerator
+from core.memory.entities import (
+    MemoryBlock,
+    MemoryBlockSpec,
+    MemoryBlockWithConversation,
+    MemoryCreatedBy,
+    MemoryScheduleMode,
+    MemoryScope,
+    MemoryTerm,
+    MemoryValueData,
+)
+from core.memory.errors import MemorySyncTimeoutError
+from core.model_runtime.entities.message_entities import PromptMessage
+from core.variables.segments import VersionedMemoryValue
+from core.workflow.constants import MEMORY_BLOCK_VARIABLE_NODE_ID
+from core.workflow.runtime.variable_pool import VariablePool
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models import App, CreatorUserRole
+from models.chatflow_memory import ChatflowMemoryVariable
+from models.workflow import Workflow, WorkflowDraftVariable
+from services.chatflow_history_service import ChatflowHistoryService
+from services.workflow_draft_variable_service import WorkflowDraftVariableService
+from services.workflow_service import WorkflowService
+
+logger = logging.getLogger(__name__)
+
+
+class ChatflowMemoryService:
+    @staticmethod
+    def get_persistent_memories(
+        app: App, created_by: MemoryCreatedBy, version: int | None = None
+    ) -> Sequence[MemoryBlock]:
+        if created_by.account_id:
+            created_by_role = CreatorUserRole.ACCOUNT
+            created_by_id = created_by.account_id
+        else:
+            created_by_role = CreatorUserRole.END_USER
+            created_by_id = created_by.id
+        if version is None:
+            # If version not specified, get the latest version
+            stmt = (
+                select(ChatflowMemoryVariable)
+                .distinct(ChatflowMemoryVariable.memory_id)
+                .where(
+                    and_(
+                        ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                        ChatflowMemoryVariable.app_id == app.id,
+                        ChatflowMemoryVariable.conversation_id == None,
+                        ChatflowMemoryVariable.created_by_role == created_by_role,
+                        ChatflowMemoryVariable.created_by == created_by_id,
+                    )
+                )
+                .order_by(ChatflowMemoryVariable.version.desc())
+            )
+        else:
+            stmt = select(ChatflowMemoryVariable).where(
+                and_(
+                    ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                    ChatflowMemoryVariable.app_id == app.id,
+                    ChatflowMemoryVariable.conversation_id == None,
+                    ChatflowMemoryVariable.created_by_role == created_by_role,
+                    ChatflowMemoryVariable.created_by == created_by_id,
+                    ChatflowMemoryVariable.version == version,
+                )
+            )
+        with Session(db.engine) as session:
+            db_results = session.execute(stmt).all()
+        return ChatflowMemoryService._convert_to_memory_blocks(app, created_by, [result[0] for result in db_results])
+
+    @staticmethod
+    def get_session_memories(
+        app: App, created_by: MemoryCreatedBy, conversation_id: str, version: int | None = None
+    ) -> Sequence[MemoryBlock]:
+        if version is None:
+            # If version not specified, get the latest version
+            stmt = (
+                select(ChatflowMemoryVariable)
+                .distinct(ChatflowMemoryVariable.memory_id)
+                .where(
+                    and_(
+                        ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                        ChatflowMemoryVariable.app_id == app.id,
+                        ChatflowMemoryVariable.conversation_id == conversation_id,
+                    )
+                )
+                .order_by(ChatflowMemoryVariable.version.desc())
+            )
+        else:
+            stmt = select(ChatflowMemoryVariable).where(
+                and_(
+                    ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                    ChatflowMemoryVariable.app_id == app.id,
+                    ChatflowMemoryVariable.conversation_id == conversation_id,
+                    ChatflowMemoryVariable.version == version,
+                )
+            )
+        with Session(db.engine) as session:
+            db_results = session.execute(stmt).all()
+        return ChatflowMemoryService._convert_to_memory_blocks(app, created_by, [result[0] for result in db_results])
+
+    @staticmethod
+    def save_memory(memory: MemoryBlock, variable_pool: VariablePool, is_draft: bool) -> None:
+        key = f"{memory.node_id}.{memory.spec.id}" if memory.node_id else memory.spec.id
+        variable_pool.add([MEMORY_BLOCK_VARIABLE_NODE_ID, key], memory.value)
+        if memory.created_by.account_id:
+            created_by_role = CreatorUserRole.ACCOUNT
+            created_by = memory.created_by.account_id
+        else:
+            created_by_role = CreatorUserRole.END_USER
+            created_by = memory.created_by.id
+
+        with Session(db.engine) as session:
+            session.add(
+                ChatflowMemoryVariable(
+                    memory_id=memory.spec.id,
+                    tenant_id=memory.tenant_id,
+                    app_id=memory.app_id,
+                    node_id=memory.node_id,
+                    conversation_id=memory.conversation_id,
+                    name=memory.spec.name,
+                    value=MemoryValueData(value=memory.value, edited_by_user=memory.edited_by_user).model_dump_json(),
+                    term=memory.spec.term,
+                    scope=memory.spec.scope,
+                    version=memory.version,  # Use version from MemoryBlock directly
+                    created_by_role=created_by_role,
+                    created_by=created_by,
+                )
+            )
+            session.commit()
+
+        if is_draft:
+            with Session(bind=db.engine) as session:
+                draft_var_service = WorkflowDraftVariableService(session)
+                memory_selector = memory.spec.id if not memory.node_id else f"{memory.node_id}.{memory.spec.id}"
+                existing_vars = draft_var_service.get_draft_variables_by_selectors(
+                    app_id=memory.app_id, selectors=[[MEMORY_BLOCK_VARIABLE_NODE_ID, memory_selector]]
+                )
+                if existing_vars:
+                    draft_var = existing_vars[0]
+                    draft_var.value = (
+                        VersionedMemoryValue.model_validate_json(draft_var.value)
+                        .add_version(memory.value)
+                        .model_dump_json()
+                    )
+                else:
+                    draft_var = WorkflowDraftVariable.new_memory_block_variable(
+                        app_id=memory.app_id,
+                        memory_id=memory.spec.id,
+                        name=memory.spec.name,
+                        value=VersionedMemoryValue().add_version(memory.value),
+                        description=memory.spec.description,
+                    )
+                    session.add(draft_var)
+                session.commit()
+
+    @staticmethod
+    def get_memories_by_specs(
+        memory_block_specs: Sequence[MemoryBlockSpec],
+        tenant_id: str,
+        app_id: str,
+        created_by: MemoryCreatedBy,
+        conversation_id: Optional[str],
+        node_id: Optional[str],
+        is_draft: bool,
+    ) -> Sequence[MemoryBlock]:
+        return [
+            ChatflowMemoryService.get_memory_by_spec(
+                spec, tenant_id, app_id, created_by, conversation_id, node_id, is_draft
+            )
+            for spec in memory_block_specs
+        ]
+
+    @staticmethod
+    def get_memory_by_spec(
+        spec: MemoryBlockSpec,
+        tenant_id: str,
+        app_id: str,
+        created_by: MemoryCreatedBy,
+        conversation_id: Optional[str],
+        node_id: Optional[str],
+        is_draft: bool,
+    ) -> MemoryBlock:
+        with Session(db.engine) as session:
+            if is_draft:
+                draft_var_service = WorkflowDraftVariableService(session)
+                selector = (
+                    [MEMORY_BLOCK_VARIABLE_NODE_ID, f"{spec.id}.{node_id}"]
+                    if node_id
+                    else [MEMORY_BLOCK_VARIABLE_NODE_ID, spec.id]
+                )
+                draft_vars = draft_var_service.get_draft_variables_by_selectors(app_id=app_id, selectors=[selector])
+                if draft_vars:
+                    draft_var = draft_vars[0]
+                    return MemoryBlock(
+                        value=draft_var.get_value().text,
+                        tenant_id=tenant_id,
+                        app_id=app_id,
+                        conversation_id=conversation_id,
+                        node_id=node_id,
+                        spec=spec,
+                        created_by=created_by,
+                        version=1,
+                    )
+            stmt = (
+                select(ChatflowMemoryVariable)
+                .where(
+                    and_(
+                        ChatflowMemoryVariable.memory_id == spec.id,
+                        ChatflowMemoryVariable.tenant_id == tenant_id,
+                        ChatflowMemoryVariable.app_id == app_id,
+                        ChatflowMemoryVariable.node_id == (node_id if spec.scope == MemoryScope.NODE else None),
+                        ChatflowMemoryVariable.conversation_id
+                        == (conversation_id if spec.term == MemoryTerm.SESSION else None),
+                    )
+                )
+                .order_by(ChatflowMemoryVariable.version.desc())
+                .limit(1)
+            )
+            result = session.execute(stmt).scalar()
+            if result:
+                memory_value_data = MemoryValueData.model_validate_json(result.value)
+                return MemoryBlock(
+                    value=memory_value_data.value,
+                    tenant_id=tenant_id,
+                    app_id=app_id,
+                    conversation_id=conversation_id,
+                    node_id=node_id,
+                    spec=spec,
+                    edited_by_user=memory_value_data.edited_by_user,
+                    created_by=created_by,
+                    version=result.version,
+                )
+            return MemoryBlock(
+                tenant_id=tenant_id,
+                value=spec.template,
+                app_id=app_id,
+                conversation_id=conversation_id,
+                node_id=node_id,
+                spec=spec,
+                created_by=created_by,
+                version=1,
+            )
+
+    @staticmethod
+    def update_app_memory_if_needed(
+        workflow: Workflow,
+        conversation_id: str,
+        variable_pool: VariablePool,
+        created_by: MemoryCreatedBy,
+        is_draft: bool,
+    ):
+        visible_messages = ChatflowHistoryService.get_visible_chat_history(
+            conversation_id=conversation_id,
+            app_id=workflow.app_id,
+            tenant_id=workflow.tenant_id,
+            node_id=None,
+        )
+        sync_blocks: list[MemoryBlock] = []
+        async_blocks: list[MemoryBlock] = []
+        for memory_spec in workflow.memory_blocks:
+            if memory_spec.scope == MemoryScope.APP:
+                memory = ChatflowMemoryService.get_memory_by_spec(
+                    spec=memory_spec,
+                    tenant_id=workflow.tenant_id,
+                    app_id=workflow.app_id,
+                    conversation_id=conversation_id,
+                    node_id=None,
+                    is_draft=is_draft,
+                    created_by=created_by,
+                )
+                if ChatflowMemoryService._should_update_memory(memory, visible_messages):
+                    if memory.spec.schedule_mode == MemoryScheduleMode.SYNC:
+                        sync_blocks.append(memory)
+                    else:
+                        async_blocks.append(memory)
+
+        if not sync_blocks and not async_blocks:
+            return
+
+        # async mode: submit individual async tasks directly
+        for memory_block in async_blocks:
+            ChatflowMemoryService._app_submit_async_memory_update(
+                block=memory_block,
+                is_draft=is_draft,
+                variable_pool=variable_pool,
+                visible_messages=visible_messages,
+                conversation_id=conversation_id,
+            )
+
+        # sync mode: submit a batch update task
+        if sync_blocks:
+            ChatflowMemoryService._app_submit_sync_memory_batch_update(
+                sync_blocks=sync_blocks,
+                is_draft=is_draft,
+                conversation_id=conversation_id,
+                app_id=workflow.app_id,
+                visible_messages=visible_messages,
+                variable_pool=variable_pool,
+            )
+
+    @staticmethod
+    def update_node_memory_if_needed(
+        tenant_id: str,
+        app_id: str,
+        node_id: str,
+        created_by: MemoryCreatedBy,
+        conversation_id: str,
+        memory_block_spec: MemoryBlockSpec,
+        variable_pool: VariablePool,
+        is_draft: bool,
+    ) -> bool:
+        visible_messages = ChatflowHistoryService.get_visible_chat_history(
+            conversation_id=conversation_id,
+            app_id=app_id,
+            tenant_id=tenant_id,
+            node_id=node_id,
+        )
+        memory_block = ChatflowMemoryService.get_memory_by_spec(
+            spec=memory_block_spec,
+            tenant_id=tenant_id,
+            app_id=app_id,
+            conversation_id=conversation_id,
+            node_id=node_id,
+            is_draft=is_draft,
+            created_by=created_by,
+        )
+        if not ChatflowMemoryService._should_update_memory(memory_block=memory_block, visible_history=visible_messages):
+            return False
+
+        if memory_block_spec.schedule_mode == MemoryScheduleMode.SYNC:
+            # Node-level sync: blocking execution
+            ChatflowMemoryService._update_node_memory_sync(
+                visible_messages=visible_messages,
+                memory_block=memory_block,
+                variable_pool=variable_pool,
+                is_draft=is_draft,
+                conversation_id=conversation_id,
+            )
+        else:
+            # Node-level async: execute asynchronously
+            ChatflowMemoryService._update_node_memory_async(
+                memory_block=memory_block,
+                visible_messages=visible_messages,
+                variable_pool=variable_pool,
+                is_draft=is_draft,
+                conversation_id=conversation_id,
+            )
+        return True
+
+    @staticmethod
+    def wait_for_sync_memory_completion(workflow: Workflow, conversation_id: str):
+        """Wait for sync memory update to complete, maximum 50 seconds"""
+
+        memory_blocks = workflow.memory_blocks
+        sync_memory_blocks = [
+            block
+            for block in memory_blocks
+            if block.scope == MemoryScope.APP and block.schedule_mode == MemoryScheduleMode.SYNC
+        ]
+
+        if not sync_memory_blocks:
+            return
+
+        lock_key = _get_memory_sync_lock_key(workflow.app_id, conversation_id)
+
+        # Retry up to 10 times, wait 5 seconds each time, total 50 seconds
+        max_retries = 10
+        retry_interval = 5
+
+        for i in range(max_retries):
+            if not redis_client.exists(lock_key):
+                # Lock doesn't exist, can continue
+                return
+
+            if i < max_retries - 1:
+                # Still have retry attempts, wait
+                time.sleep(retry_interval)
+            else:
+                # Maximum retry attempts reached, raise exception
+                raise MemorySyncTimeoutError(app_id=workflow.app_id, conversation_id=conversation_id)
+
+    @staticmethod
+    def _convert_to_memory_blocks(
+        app: App, created_by: MemoryCreatedBy, raw_results: Sequence[ChatflowMemoryVariable]
+    ) -> Sequence[MemoryBlock]:
+        workflow = WorkflowService().get_published_workflow(app)
+        if not workflow:
+            return []
+        results = []
+        for chatflow_memory_variable in raw_results:
+            spec = next(
+                (spec for spec in workflow.memory_blocks if spec.id == chatflow_memory_variable.memory_id), None
+            )
+            if spec and chatflow_memory_variable.app_id:
+                memory_value_data = MemoryValueData.model_validate_json(chatflow_memory_variable.value)
+                results.append(
+                    MemoryBlock(
+                        spec=spec,
+                        tenant_id=chatflow_memory_variable.tenant_id,
+                        value=memory_value_data.value,
+                        app_id=chatflow_memory_variable.app_id,
+                        conversation_id=chatflow_memory_variable.conversation_id,
+                        node_id=chatflow_memory_variable.node_id,
+                        edited_by_user=memory_value_data.edited_by_user,
+                        created_by=created_by,
+                        version=chatflow_memory_variable.version,
+                    )
+                )
+        return results
+
+    @staticmethod
+    def _should_update_memory(memory_block: MemoryBlock, visible_history: Sequence[PromptMessage]) -> bool:
+        return len(visible_history) >= memory_block.spec.update_turns
+
+    @staticmethod
+    def _app_submit_async_memory_update(
+        block: MemoryBlock,
+        visible_messages: Sequence[PromptMessage],
+        variable_pool: VariablePool,
+        conversation_id: str,
+        is_draft: bool,
+    ):
+        thread = threading.Thread(
+            target=ChatflowMemoryService._perform_memory_update,
+            kwargs={
+                "memory_block": block,
+                "visible_messages": visible_messages,
+                "variable_pool": variable_pool,
+                "is_draft": is_draft,
+                "conversation_id": conversation_id,
+            },
+        )
+        thread.start()
+
+    @staticmethod
+    def _app_submit_sync_memory_batch_update(
+        sync_blocks: Sequence[MemoryBlock],
+        app_id: str,
+        conversation_id: str,
+        visible_messages: Sequence[PromptMessage],
+        variable_pool: VariablePool,
+        is_draft: bool,
+    ):
+        """Submit sync memory batch update task"""
+        thread = threading.Thread(
+            target=ChatflowMemoryService._batch_update_sync_memory,
+            kwargs={
+                "sync_blocks": sync_blocks,
+                "app_id": app_id,
+                "conversation_id": conversation_id,
+                "visible_messages": visible_messages,
+                "variable_pool": variable_pool,
+                "is_draft": is_draft,
+            },
+        )
+        thread.start()
+
+    @staticmethod
+    def _batch_update_sync_memory(
+        sync_blocks: Sequence[MemoryBlock],
+        app_id: str,
+        conversation_id: str,
+        visible_messages: Sequence[PromptMessage],
+        variable_pool: VariablePool,
+        is_draft: bool,
+    ):
+        try:
+            lock_key = _get_memory_sync_lock_key(app_id, conversation_id)
+            with redis_client.lock(lock_key, timeout=120):
+                threads = []
+                for block in sync_blocks:
+                    thread = threading.Thread(
+                        target=ChatflowMemoryService._perform_memory_update,
+                        kwargs={
+                            "memory_block": block,
+                            "visible_messages": visible_messages,
+                            "variable_pool": variable_pool,
+                            "is_draft": is_draft,
+                            "conversation_id": conversation_id,
+                        },
+                    )
+                    threads.append(thread)
+                for thread in threads:
+                    thread.start()
+                for thread in threads:
+                    thread.join()
+        except Exception as e:
+            logger.exception("Error batch updating memory", exc_info=e)
+
+    @staticmethod
+    def _update_node_memory_sync(
+        memory_block: MemoryBlock,
+        visible_messages: Sequence[PromptMessage],
+        variable_pool: VariablePool,
+        conversation_id: str,
+        is_draft: bool,
+    ):
+        ChatflowMemoryService._perform_memory_update(
+            memory_block=memory_block,
+            visible_messages=visible_messages,
+            variable_pool=variable_pool,
+            is_draft=is_draft,
+            conversation_id=conversation_id,
+        )
+
+    @staticmethod
+    def _update_node_memory_async(
+        memory_block: MemoryBlock,
+        visible_messages: Sequence[PromptMessage],
+        variable_pool: VariablePool,
+        conversation_id: str,
+        is_draft: bool = False,
+    ):
+        thread = threading.Thread(
+            target=ChatflowMemoryService._perform_memory_update,
+            kwargs={
+                "memory_block": memory_block,
+                "visible_messages": visible_messages,
+                "variable_pool": variable_pool,
+                "is_draft": is_draft,
+                "conversation_id": conversation_id,
+            },
+            daemon=True,
+        )
+        thread.start()
+
+    @staticmethod
+    def _perform_memory_update(
+        memory_block: MemoryBlock,
+        variable_pool: VariablePool,
+        conversation_id: str,
+        visible_messages: Sequence[PromptMessage],
+        is_draft: bool,
+    ):
+        updated_value = LLMGenerator.update_memory_block(
+            tenant_id=memory_block.tenant_id,
+            visible_history=ChatflowMemoryService._format_chat_history(visible_messages),
+            variable_pool=variable_pool,
+            memory_block=memory_block,
+            memory_spec=memory_block.spec,
+        )
+        updated_memory = MemoryBlock(
+            tenant_id=memory_block.tenant_id,
+            value=updated_value,
+            spec=memory_block.spec,
+            app_id=memory_block.app_id,
+            conversation_id=conversation_id,
+            node_id=memory_block.node_id,
+            edited_by_user=False,
+            created_by=memory_block.created_by,
+            version=memory_block.version + 1,  # Increment version for business logic update
+        )
+        ChatflowMemoryService.save_memory(updated_memory, variable_pool, is_draft)
+
+        ChatflowHistoryService.update_visible_count(
+            conversation_id=conversation_id,
+            node_id=memory_block.node_id,
+            new_visible_count=memory_block.spec.preserved_turns,
+            app_id=memory_block.app_id,
+            tenant_id=memory_block.tenant_id,
+        )
+
+    @staticmethod
+    def delete_memory(app: App, memory_id: str, created_by: MemoryCreatedBy):
+        workflow = WorkflowService().get_published_workflow(app)
+        if not workflow:
+            raise ValueError("Workflow not found")
+
+        memory_spec = next((it for it in workflow.memory_blocks if it.id == memory_id), None)
+        if not memory_spec or not memory_spec.end_user_editable:
+            raise ValueError("Memory not found or not deletable")
+
+        if created_by.account_id:
+            created_by_role = CreatorUserRole.ACCOUNT
+            created_by_id = created_by.account_id
+        else:
+            created_by_role = CreatorUserRole.END_USER
+            created_by_id = created_by.id
+
+        with Session(db.engine) as session:
+            stmt = delete(ChatflowMemoryVariable).where(
+                and_(
+                    ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                    ChatflowMemoryVariable.app_id == app.id,
+                    ChatflowMemoryVariable.memory_id == memory_id,
+                    ChatflowMemoryVariable.created_by_role == created_by_role,
+                    ChatflowMemoryVariable.created_by == created_by_id,
+                )
+            )
+            session.execute(stmt)
+            session.commit()
+
+    @staticmethod
+    def delete_all_user_memories(app: App, created_by: MemoryCreatedBy):
+        if created_by.account_id:
+            created_by_role = CreatorUserRole.ACCOUNT
+            created_by_id = created_by.account_id
+        else:
+            created_by_role = CreatorUserRole.END_USER
+            created_by_id = created_by.id
+
+        with Session(db.engine) as session:
+            stmt = delete(ChatflowMemoryVariable).where(
+                and_(
+                    ChatflowMemoryVariable.tenant_id == app.tenant_id,
+                    ChatflowMemoryVariable.app_id == app.id,
+                    ChatflowMemoryVariable.created_by_role == created_by_role,
+                    ChatflowMemoryVariable.created_by == created_by_id,
+                )
+            )
+            session.execute(stmt)
+            session.commit()
+
+    @staticmethod
+    def get_persistent_memories_with_conversation(
+        app: App, created_by: MemoryCreatedBy, conversation_id: str, version: int | None = None
+    ) -> Sequence[MemoryBlockWithConversation]:
+        """Get persistent memories with conversation metadata (always None for persistent)"""
+        memory_blocks = ChatflowMemoryService.get_persistent_memories(app, created_by, version)
+        return [
+            MemoryBlockWithConversation.from_memory_block(
+                block,
+                ChatflowHistoryService.get_conversation_metadata(app.tenant_id, app.id, conversation_id, block.node_id),
+            )
+            for block in memory_blocks
+        ]
+
+    @staticmethod
+    def get_session_memories_with_conversation(
+        app: App, created_by: MemoryCreatedBy, conversation_id: str, version: int | None = None
+    ) -> Sequence[MemoryBlockWithConversation]:
+        """Get session memories with conversation metadata"""
+        memory_blocks = ChatflowMemoryService.get_session_memories(app, created_by, conversation_id, version)
+        return [
+            MemoryBlockWithConversation.from_memory_block(
+                block,
+                ChatflowHistoryService.get_conversation_metadata(app.tenant_id, app.id, conversation_id, block.node_id),
+            )
+            for block in memory_blocks
+        ]
+
+    @staticmethod
+    def _format_chat_history(messages: Sequence[PromptMessage]) -> Sequence[tuple[str, str]]:
+        result = []
+        for message in messages:
+            result.append((str(message.role.value), message.get_text_content()))
+        return result
+
+
+def _get_memory_sync_lock_key(app_id: str, conversation_id: str) -> str:
+    """Generate Redis lock key for memory sync updates
+
+    Args:
+        app_id: Application ID
+        conversation_id: Conversation ID
+
+    Returns:
+        Formatted lock key
+    """
+    return f"memory_sync_update:{app_id}:{conversation_id}"

api/services/credit_pool_service.py

@@ -0,0 +1,67 @@
+import logging
+
+from sqlalchemy import update
+from sqlalchemy.orm import Session
+
+from configs import dify_config
+from core.errors.error import QuotaExceededError
+from extensions.ext_database import db
+from models import TenantCreditPool
+
+logger = logging.getLogger(__name__)
+
+
+class CreditPoolService:
+    @classmethod
+    def create_default_pool(cls, tenant_id: str) -> TenantCreditPool:
+        """create default credit pool for new tenant"""
+        credit_pool = TenantCreditPool(
+            tenant_id=tenant_id, quota_limit=dify_config.HOSTED_POOL_CREDITS, quota_used=0, pool_type="trial"
+        )
+        db.session.add(credit_pool)
+        db.session.commit()
+        return credit_pool
+
+    @classmethod
+    def get_pool(cls, tenant_id: str, pool_type: str = "trial") -> TenantCreditPool | None:
+        """get tenant credit pool"""
+        return (
+            db.session.query(TenantCreditPool)
+            .filter_by(
+                tenant_id=tenant_id,
+                pool_type=pool_type,
+            )
+            .first()
+        )
+
+    @classmethod
+    def check_and_deduct_credits(
+        cls,
+        tenant_id: str,
+        credits_required: int,
+        pool_type: str = "trial",
+    ):
+        """check and deduct credits"""
+
+        pool = cls.get_pool(tenant_id, pool_type)
+        if not pool:
+            raise QuotaExceededError("Credit pool not found")
+
+        if pool.remaining_credits < credits_required:
+            raise QuotaExceededError(
+                f"Insufficient credits. Required: {credits_required}, Available: {pool.remaining_credits}"
+            )
+        try:
+            with Session(db.engine) as session:
+                update_values = {"quota_used": pool.quota_used + credits_required}
+
+                where_conditions = [
+                    TenantCreditPool.pool_type == pool_type,
+                    TenantCreditPool.tenant_id == tenant_id,
+                    TenantCreditPool.quota_used + credits_required <= TenantCreditPool.quota_limit,
+                ]
+                stmt = update(TenantCreditPool).where(*where_conditions).values(**update_values)
+                session.execute(stmt)
+                session.commit()
+        except Exception:
+            raise QuotaExceededError("Failed to deduct credits")

api/services/dataset_service.py

@@ -50,6 +50,7 @@ from models.model import UploadFile
 from models.provider_ids import ModelProviderID
 from models.source import DataSourceOauthBinding
 from models.workflow import Workflow
+from services.document_indexing_task_proxy import DocumentIndexingTaskProxy
 from services.entities.knowledge_entities.knowledge_entities import (
     ChildChunkUpdateArgs,
     KnowledgeConfig,
@@ -79,7 +80,6 @@ from tasks.deal_dataset_vector_index_task import deal_dataset_vector_index_task
 from tasks.delete_segment_from_index_task import delete_segment_from_index_task
 from tasks.disable_segment_from_index_task import disable_segment_from_index_task
 from tasks.disable_segments_from_index_task import disable_segments_from_index_task
-from tasks.document_indexing_task import document_indexing_task
 from tasks.document_indexing_update_task import document_indexing_update_task
 from tasks.duplicate_document_indexing_task import duplicate_document_indexing_task
 from tasks.enable_segments_to_index_task import enable_segments_to_index_task
@@ -1694,7 +1694,7 @@ class DocumentService:
 
                 # trigger async task
                 if document_ids:
-                    document_indexing_task.delay(dataset.id, document_ids)
+                    DocumentIndexingTaskProxy(dataset.tenant_id, dataset.id, document_ids).delay()
                 if duplicate_document_ids:
                     duplicate_document_indexing_task.delay(dataset.id, duplicate_document_ids)
 

api/services/document_indexing_task_proxy.py

@@ -0,0 +1,82 @@
+import logging
+from collections.abc import Callable
+from dataclasses import asdict
+from functools import cached_property
+
+from core.entities.document_task import DocumentTask
+from core.rag.pipeline.queue import TenantSelfTaskQueue
+from services.feature_service import FeatureService
+from tasks.document_indexing_task import normal_document_indexing_task, priority_document_indexing_task
+
+logger = logging.getLogger(__name__)
+
+
+class DocumentIndexingTaskProxy:
+    def __init__(self, tenant_id: str, dataset_id: str, document_ids: list[str]):
+        self.tenant_id = tenant_id
+        self.dataset_id = dataset_id
+        self.document_ids = document_ids
+        self.tenant_self_task_queue = TenantSelfTaskQueue(tenant_id, "document_indexing")
+
+    @cached_property
+    def features(self):
+        return FeatureService.get_features(self.tenant_id)
+
+    def _send_to_direct_queue(self, task_func: Callable):
+        logger.info("send dataset %s to direct queue", self.dataset_id)
+        task_func.delay(  # type: ignore
+            tenant_id=self.tenant_id,
+            dataset_id=self.dataset_id,
+            document_ids=self.document_ids
+        )
+
+    def _send_to_tenant_queue(self, task_func: Callable):
+        logger.info("send dataset %s to tenant queue", self.dataset_id)
+        if self.tenant_self_task_queue.get_task_key():
+            # Add to waiting queue using List operations (lpush)
+            self.tenant_self_task_queue.push_tasks([
+                asdict(
+                    DocumentTask(tenant_id=self.tenant_id, dataset_id=self.dataset_id, document_ids=self.document_ids)
+                )
+            ])
+            logger.info("push tasks: %s - %s", self.dataset_id, self.document_ids)
+        else:
+            # Set flag and execute task
+            self.tenant_self_task_queue.set_task_waiting_time()
+            task_func.delay(  # type: ignore
+                tenant_id=self.tenant_id,
+                dataset_id=self.dataset_id,
+                document_ids=self.document_ids
+            )
+            logger.info("init tasks: %s - %s", self.dataset_id, self.document_ids)
+
+    def _send_to_default_tenant_queue(self):
+        self._send_to_tenant_queue(normal_document_indexing_task)
+    
+    def _send_to_priority_tenant_queue(self):
+        self._send_to_tenant_queue(priority_document_indexing_task)
+
+    def _send_to_priority_direct_queue(self):
+        self._send_to_direct_queue(priority_document_indexing_task)
+
+    def _dispatch(self):
+        logger.info(
+            "dispatch args: %s - %s - %s",
+            self.tenant_id,
+            self.features.billing.enabled,
+            self.features.billing.subscription.plan
+        )
+        # dispatch to different indexing queue with tenant isolation when billing enabled
+        if self.features.billing.enabled:
+            if self.features.billing.subscription.plan == "sandbox":
+                # dispatch to normal pipeline queue with tenant self sub queue for sandbox plan
+                self._send_to_default_tenant_queue()
+            else:
+                # dispatch to priority pipeline queue with tenant self sub queue for other plans
+                self._send_to_priority_tenant_queue()
+        else:
+            # dispatch to priority queue without tenant isolation for others, e.g.: self-hosted or enterprise
+            self._send_to_priority_direct_queue()
+
+    def delay(self):
+        self._dispatch()

api/services/feature_service.py

@@ -132,6 +132,7 @@ class FeatureModel(BaseModel):
     # pydantic configs
     model_config = ConfigDict(protected_namespaces=())
     knowledge_pipeline: KnowledgePipeline = KnowledgePipeline()
+    next_credit_reset_date: int = 0
 
 
 class KnowledgeRateLimitModel(BaseModel):
@@ -152,6 +153,7 @@ class SystemFeatureModel(BaseModel):
     enable_email_code_login: bool = False
     enable_email_password_login: bool = True
     enable_social_oauth_login: bool = False
+    enable_collaboration_mode: bool = False
     is_allow_register: bool = False
     is_allow_create_workspace: bool = False
     is_email_setup: bool = False
@@ -161,6 +163,8 @@ class SystemFeatureModel(BaseModel):
     plugin_installation_permission: PluginInstallationPermissionModel = PluginInstallationPermissionModel()
     enable_change_email: bool = True
     plugin_manager: PluginManagerModel = PluginManagerModel()
+    enable_trial_app: bool = False
+    enable_explore_banner: bool = False
 
 
 class FeatureService:
@@ -213,9 +217,12 @@ class FeatureService:
         system_features.enable_email_code_login = dify_config.ENABLE_EMAIL_CODE_LOGIN
         system_features.enable_email_password_login = dify_config.ENABLE_EMAIL_PASSWORD_LOGIN
         system_features.enable_social_oauth_login = dify_config.ENABLE_SOCIAL_OAUTH_LOGIN
+        system_features.enable_collaboration_mode = dify_config.ENABLE_COLLABORATION_MODE
         system_features.is_allow_register = dify_config.ALLOW_REGISTER
         system_features.is_allow_create_workspace = dify_config.ALLOW_CREATE_WORKSPACE
         system_features.is_email_setup = dify_config.MAIL_TYPE is not None and dify_config.MAIL_TYPE != ""
+        system_features.enable_trial_app = dify_config.ENABLE_TRIAL_APP
+        system_features.enable_explore_banner = dify_config.ENABLE_EXPLORE_BANNER
 
     @classmethod
     def _fulfill_params_from_env(cls, features: FeatureModel):
@@ -280,6 +287,9 @@ class FeatureService:
 
         if "knowledge_pipeline_publish_enabled" in billing_info:
             features.knowledge_pipeline.publish_enabled = billing_info["knowledge_pipeline_publish_enabled"]
+    
+        if "next_credit_reset_date" in billing_info:
+            features.next_credit_reset_date = billing_info["next_credit_reset_date"]
 
     @classmethod
     def _fulfill_params_from_enterprise(cls, features: SystemFeatureModel):

api/services/rag_pipeline/rag_pipeline_task_proxy.py

@@ -0,0 +1,101 @@
+import json
+import logging
+from collections.abc import Callable
+from functools import cached_property
+
+from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
+from core.rag.pipeline.queue import TenantSelfTaskQueue
+from extensions.ext_database import db
+from services.feature_service import FeatureService
+from services.file_service import FileService
+from tasks.rag_pipeline.priority_rag_pipeline_run_task import priority_rag_pipeline_run_task
+from tasks.rag_pipeline.rag_pipeline_run_task import rag_pipeline_run_task
+
+logger = logging.getLogger(__name__)
+
+
+class RagPipelineTaskProxy:
+    def __init__(
+        self, dataset_tenant_id: str, user_id: str, rag_pipeline_invoke_entities: list[RagPipelineInvokeEntity]
+    ):
+        self.dataset_tenant_id = dataset_tenant_id
+        self.user_id = user_id
+        self.rag_pipeline_invoke_entities = rag_pipeline_invoke_entities
+        self.tenant_self_pipeline_task_queue = TenantSelfTaskQueue(dataset_tenant_id, "pipeline")
+
+    @cached_property
+    def features(self):
+        return FeatureService.get_features(self.dataset_tenant_id)
+    
+    def _upload_invoke_entities(self) -> str:
+        text = [item.model_dump() for item in self.rag_pipeline_invoke_entities]
+        name = "rag_pipeline_invoke_entities.json"
+        # Convert list to proper JSON string
+        json_text = json.dumps(text)
+        upload_file = FileService(db.engine).upload_text(json_text, name, self.user_id, self.dataset_tenant_id)
+        return upload_file.id
+    
+    def _send_to_direct_queue(self, upload_file_id: str, task_func: Callable):
+        logger.info("send file %s to direct queue", upload_file_id)
+        task_func.delay(  # type: ignore
+            rag_pipeline_invoke_entities_file_id=upload_file_id,
+            tenant_id=self.dataset_tenant_id,
+        )
+
+    def _send_to_tenant_queue(self, upload_file_id: str, task_func: Callable):
+        logger.info("send file %s to tenant queue", upload_file_id)
+        if self.tenant_self_pipeline_task_queue.get_task_key():
+            # Add to waiting queue using List operations (lpush)
+            self.tenant_self_pipeline_task_queue.push_tasks([upload_file_id])
+            logger.info("push tasks: %s", upload_file_id)
+        else:
+            # Set flag and execute task
+            self.tenant_self_pipeline_task_queue.set_task_waiting_time()
+            task_func.delay(  # type: ignore
+                rag_pipeline_invoke_entities_file_id=upload_file_id,
+                tenant_id=self.dataset_tenant_id,
+            )
+            logger.info("init tasks: %s", upload_file_id)
+
+    def _send_to_default_tenant_queue(self, upload_file_id: str):
+        self._send_to_tenant_queue(upload_file_id, rag_pipeline_run_task)
+
+    def _send_to_priority_tenant_queue(self, upload_file_id: str):
+        self._send_to_tenant_queue(upload_file_id, priority_rag_pipeline_run_task)
+
+    def _send_to_priority_direct_queue(self, upload_file_id: str):
+        self._send_to_direct_queue(upload_file_id, priority_rag_pipeline_run_task)
+
+    def _dispatch(self):
+        upload_file_id = self._upload_invoke_entities()
+        if not upload_file_id:
+            raise ValueError("upload_file_id is empty")
+
+        logger.info(
+            "dispatch args: %s - %s - %s",
+            self.dataset_tenant_id,
+            self.features.billing.enabled,
+            self.features.billing.subscription.plan
+        )
+
+        # dispatch to different pipeline queue with tenant isolation when billing enabled
+        if self.features.billing.enabled:
+            if self.features.billing.subscription.plan == "sandbox":
+                # dispatch to normal pipeline queue with tenant isolation for sandbox plan
+                self._send_to_default_tenant_queue(upload_file_id)
+            else:
+                # dispatch to priority pipeline queue with tenant isolation for other plans
+                self._send_to_priority_tenant_queue(upload_file_id)
+        else:
+            # dispatch to priority pipeline queue without tenant isolation for others, e.g.: self-hosted or enterprise
+            self._send_to_priority_direct_queue(upload_file_id)
+
+    def delay(self):
+        if not self.rag_pipeline_invoke_entities:
+            logger.warning(
+                "Received empty rag pipeline invoke entities, no tasks delivered: %s %s",
+                self.dataset_tenant_id,
+                self.user_id
+            )
+            return
+        self._dispatch()

api/services/recommended_app_service.py

@@ -1,4 +1,7 @@
 from configs import dify_config
+from extensions.ext_database import db
+from models.model import AccountTrialAppRecord, TrialApp
+from services.feature_service import FeatureService
 from services.recommend_app.recommend_app_factory import RecommendAppRetrievalFactory
 
 
@@ -20,6 +23,15 @@ class RecommendedAppService:
                 )
             )
 
+        if FeatureService.get_system_features().enable_trial_app:
+            apps = result["recommended_apps"]
+            for app in apps:
+                app_id = app["app_id"]
+                trial_app_model = db.session.query(TrialApp).where(TrialApp.app_id == app_id).first()
+                if trial_app_model:
+                    app["can_trial"] = True
+                else:
+                    app["can_trial"] = False
         return result
 
     @classmethod
@@ -32,4 +44,29 @@ class RecommendedAppService:
         mode = dify_config.HOSTED_FETCH_APP_TEMPLATES_MODE
         retrieval_instance = RecommendAppRetrievalFactory.get_recommend_app_factory(mode)()
         result: dict = retrieval_instance.get_recommend_app_detail(app_id)
+        if FeatureService.get_system_features().enable_trial_app:
+            app_id = result["id"]
+            trial_app_model = db.session.query(TrialApp).where(TrialApp.app_id == app_id).first()
+            if trial_app_model:
+                result["can_trial"] = True
+            else:
+                result["can_trial"] = False
         return result
+
+    @classmethod
+    def add_trial_app_record(cls, app_id: str, account_id: str):
+        """
+        Add trial app record.
+        :param app_id: app id
+        :return:
+        """
+        account_trial_app_record = db.session.query(AccountTrialAppRecord).where(
+            AccountTrialAppRecord.app_id == app_id,
+            AccountTrialAppRecord.account_id == account_id
+        ).first()
+        if account_trial_app_record:
+            account_trial_app_record.count += 1
+            db.session.commit()
+        else:
+            db.session.add(AccountTrialAppRecord(app_id=app_id, count=1, account_id=account_id))
+            db.session.commit()

api/services/tools/tools_transform_service.py

@@ -7,6 +7,7 @@ from pydantic import ValidationError
 from yarl import URL
 
 from configs import dify_config
+from core.entities.mcp_provider import MCPConfiguration
 from core.helper.provider_cache import ToolProviderCredentialsCache
 from core.mcp.types import Tool as MCPTool
 from core.plugin.entities.plugin_daemon import PluginDatasourceProviderEntity
@@ -239,7 +240,6 @@ class ToolTransformService:
         user_name: str | None = None,
         include_sensitive: bool = True,
     ) -> ToolProviderApiEntity:
-        from core.entities.mcp_provider import MCPConfiguration
 
         # Use provided user_name to avoid N+1 query, fallback to load_user() if not provided
         if user_name is None:

api/services/workflow_comment_service.py

@@ -0,0 +1,311 @@
+import logging
+from typing import Optional
+
+from sqlalchemy import desc, select
+from sqlalchemy.orm import Session, selectinload
+from werkzeug.exceptions import Forbidden, NotFound
+
+from extensions.ext_database import db
+from libs.datetime_utils import naive_utc_now
+from libs.helper import uuid_value
+from models import WorkflowComment, WorkflowCommentMention, WorkflowCommentReply
+
+logger = logging.getLogger(__name__)
+
+
+class WorkflowCommentService:
+    """Service for managing workflow comments."""
+
+    @staticmethod
+    def _validate_content(content: str) -> None:
+        if len(content.strip()) == 0:
+            raise ValueError("Comment content cannot be empty")
+
+        if len(content) > 1000:
+            raise ValueError("Comment content cannot exceed 1000 characters")
+
+    @staticmethod
+    def get_comments(tenant_id: str, app_id: str) -> list[WorkflowComment]:
+        """Get all comments for a workflow."""
+        with Session(db.engine) as session:
+            # Get all comments with eager loading
+            stmt = (
+                select(WorkflowComment)
+                .options(selectinload(WorkflowComment.replies), selectinload(WorkflowComment.mentions))
+                .where(WorkflowComment.tenant_id == tenant_id, WorkflowComment.app_id == app_id)
+                .order_by(desc(WorkflowComment.created_at))
+            )
+
+            comments = session.scalars(stmt).all()
+            return comments
+
+    @staticmethod
+    def get_comment(tenant_id: str, app_id: str, comment_id: str, session: Session = None) -> WorkflowComment:
+        """Get a specific comment."""
+
+        def _get_comment(session: Session) -> WorkflowComment:
+            stmt = (
+                select(WorkflowComment)
+                .options(selectinload(WorkflowComment.replies), selectinload(WorkflowComment.mentions))
+                .where(
+                    WorkflowComment.id == comment_id,
+                    WorkflowComment.tenant_id == tenant_id,
+                    WorkflowComment.app_id == app_id,
+                )
+            )
+            comment = session.scalar(stmt)
+
+            if not comment:
+                raise NotFound("Comment not found")
+
+            return comment
+
+        if session is not None:
+            return _get_comment(session)
+        else:
+            with Session(db.engine, expire_on_commit=False) as session:
+                return _get_comment(session)
+
+    @staticmethod
+    def create_comment(
+        tenant_id: str,
+        app_id: str,
+        created_by: str,
+        content: str,
+        position_x: float,
+        position_y: float,
+        mentioned_user_ids: Optional[list[str]] = None,
+    ) -> WorkflowComment:
+        """Create a new workflow comment."""
+        WorkflowCommentService._validate_content(content)
+
+        with Session(db.engine) as session:
+            comment = WorkflowComment(
+                tenant_id=tenant_id,
+                app_id=app_id,
+                position_x=position_x,
+                position_y=position_y,
+                content=content,
+                created_by=created_by,
+            )
+
+            session.add(comment)
+            session.flush()  # Get the comment ID for mentions
+
+            # Create mentions if specified
+            mentioned_user_ids = mentioned_user_ids or []
+            for user_id in mentioned_user_ids:
+                if isinstance(user_id, str) and uuid_value(user_id):
+                    mention = WorkflowCommentMention(
+                        comment_id=comment.id,
+                        reply_id=None,  # This is a comment mention, not reply mention
+                        mentioned_user_id=user_id,
+                    )
+                    session.add(mention)
+
+            session.commit()
+
+            # Return only what we need - id and created_at
+            return {"id": comment.id, "created_at": comment.created_at}
+
+    @staticmethod
+    def update_comment(
+        tenant_id: str,
+        app_id: str,
+        comment_id: str,
+        user_id: str,
+        content: str,
+        position_x: Optional[float] = None,
+        position_y: Optional[float] = None,
+        mentioned_user_ids: Optional[list[str]] = None,
+    ) -> dict:
+        """Update a workflow comment."""
+        WorkflowCommentService._validate_content(content)
+
+        with Session(db.engine, expire_on_commit=False) as session:
+            # Get comment with validation
+            stmt = select(WorkflowComment).where(
+                WorkflowComment.id == comment_id,
+                WorkflowComment.tenant_id == tenant_id,
+                WorkflowComment.app_id == app_id,
+            )
+            comment = session.scalar(stmt)
+
+            if not comment:
+                raise NotFound("Comment not found")
+
+            # Only the creator can update the comment
+            if comment.created_by != user_id:
+                raise Forbidden("Only the comment creator can update it")
+
+            # Update comment fields
+            comment.content = content
+            if position_x is not None:
+                comment.position_x = position_x
+            if position_y is not None:
+                comment.position_y = position_y
+
+            # Update mentions - first remove existing mentions for this comment only (not replies)
+            existing_mentions = session.scalars(
+                select(WorkflowCommentMention).where(
+                    WorkflowCommentMention.comment_id == comment.id,
+                    WorkflowCommentMention.reply_id.is_(None),  # Only comment mentions, not reply mentions
+                )
+            ).all()
+            for mention in existing_mentions:
+                session.delete(mention)
+
+            # Add new mentions
+            mentioned_user_ids = mentioned_user_ids or []
+            for user_id_str in mentioned_user_ids:
+                if isinstance(user_id_str, str) and uuid_value(user_id_str):
+                    mention = WorkflowCommentMention(
+                        comment_id=comment.id,
+                        reply_id=None,  # This is a comment mention
+                        mentioned_user_id=user_id_str,
+                    )
+                    session.add(mention)
+
+            session.commit()
+
+            return {"id": comment.id, "updated_at": comment.updated_at}
+
+    @staticmethod
+    def delete_comment(tenant_id: str, app_id: str, comment_id: str, user_id: str) -> None:
+        """Delete a workflow comment."""
+        with Session(db.engine, expire_on_commit=False) as session:
+            comment = WorkflowCommentService.get_comment(tenant_id, app_id, comment_id, session)
+
+            # Only the creator can delete the comment
+            if comment.created_by != user_id:
+                raise Forbidden("Only the comment creator can delete it")
+
+            # Delete associated mentions (both comment and reply mentions)
+            mentions = session.scalars(
+                select(WorkflowCommentMention).where(WorkflowCommentMention.comment_id == comment_id)
+            ).all()
+            for mention in mentions:
+                session.delete(mention)
+
+            # Delete associated replies
+            replies = session.scalars(
+                select(WorkflowCommentReply).where(WorkflowCommentReply.comment_id == comment_id)
+            ).all()
+            for reply in replies:
+                session.delete(reply)
+
+            session.delete(comment)
+            session.commit()
+
+    @staticmethod
+    def resolve_comment(tenant_id: str, app_id: str, comment_id: str, user_id: str) -> WorkflowComment:
+        """Resolve a workflow comment."""
+        with Session(db.engine, expire_on_commit=False) as session:
+            comment = WorkflowCommentService.get_comment(tenant_id, app_id, comment_id, session)
+            if comment.resolved:
+                return comment
+
+            comment.resolved = True
+            comment.resolved_at = naive_utc_now()
+            comment.resolved_by = user_id
+            session.commit()
+
+        return comment
+
+    @staticmethod
+    def create_reply(
+        comment_id: str, content: str, created_by: str, mentioned_user_ids: Optional[list[str]] = None
+    ) -> dict:
+        """Add a reply to a workflow comment."""
+        WorkflowCommentService._validate_content(content)
+
+        with Session(db.engine, expire_on_commit=False) as session:
+            # Check if comment exists
+            comment = session.get(WorkflowComment, comment_id)
+            if not comment:
+                raise NotFound("Comment not found")
+
+            reply = WorkflowCommentReply(comment_id=comment_id, content=content, created_by=created_by)
+
+            session.add(reply)
+            session.flush()  # Get the reply ID for mentions
+
+            # Create mentions if specified
+            mentioned_user_ids = mentioned_user_ids or []
+            for user_id in mentioned_user_ids:
+                if isinstance(user_id, str) and uuid_value(user_id):
+                    # Create mention linking to specific reply
+                    mention = WorkflowCommentMention(
+                        comment_id=comment_id, reply_id=reply.id, mentioned_user_id=user_id
+                    )
+                    session.add(mention)
+
+            session.commit()
+
+            return {"id": reply.id, "created_at": reply.created_at}
+
+    @staticmethod
+    def update_reply(
+        reply_id: str, user_id: str, content: str, mentioned_user_ids: Optional[list[str]] = None
+    ) -> WorkflowCommentReply:
+        """Update a comment reply."""
+        WorkflowCommentService._validate_content(content)
+
+        with Session(db.engine, expire_on_commit=False) as session:
+            reply = session.get(WorkflowCommentReply, reply_id)
+            if not reply:
+                raise NotFound("Reply not found")
+
+            # Only the creator can update the reply
+            if reply.created_by != user_id:
+                raise Forbidden("Only the reply creator can update it")
+
+            reply.content = content
+
+            # Update mentions - first remove existing mentions for this reply
+            existing_mentions = session.scalars(
+                select(WorkflowCommentMention).where(WorkflowCommentMention.reply_id == reply.id)
+            ).all()
+            for mention in existing_mentions:
+                session.delete(mention)
+
+            # Add mentions
+            mentioned_user_ids = mentioned_user_ids or []
+            for user_id_str in mentioned_user_ids:
+                if isinstance(user_id_str, str) and uuid_value(user_id_str):
+                    mention = WorkflowCommentMention(
+                        comment_id=reply.comment_id, reply_id=reply.id, mentioned_user_id=user_id_str
+                    )
+                    session.add(mention)
+
+            session.commit()
+            session.refresh(reply)  # Refresh to get updated timestamp
+
+            return {"id": reply.id, "updated_at": reply.updated_at}
+
+    @staticmethod
+    def delete_reply(reply_id: str, user_id: str) -> None:
+        """Delete a comment reply."""
+        with Session(db.engine, expire_on_commit=False) as session:
+            reply = session.get(WorkflowCommentReply, reply_id)
+            if not reply:
+                raise NotFound("Reply not found")
+
+            # Only the creator can delete the reply
+            if reply.created_by != user_id:
+                raise Forbidden("Only the reply creator can delete it")
+
+            # Delete associated mentions first
+            mentions = session.scalars(
+                select(WorkflowCommentMention).where(WorkflowCommentMention.reply_id == reply_id)
+            ).all()
+            for mention in mentions:
+                session.delete(mention)
+
+            session.delete(reply)
+            session.commit()
+
+    @staticmethod
+    def validate_comment_access(comment_id: str, tenant_id: str, app_id: str) -> WorkflowComment:
+        """Validate that a comment belongs to the specified tenant and app."""
+        return WorkflowCommentService.get_comment(tenant_id, app_id, comment_id)

api/services/workflow_service.py

@@ -11,6 +11,7 @@ from core.app.app_config.entities import VariableEntityType
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
 from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
 from core.file import File
+from core.memory.entities import MemoryBlockSpec, MemoryCreatedBy, MemoryScope
 from core.repositories import DifyCoreRepositoryFactory
 from core.variables import Variable
 from core.variables.variables import VariableUnion
@@ -197,15 +198,18 @@ class WorkflowService:
         account: Account,
         environment_variables: Sequence[Variable],
         conversation_variables: Sequence[Variable],
+        memory_blocks: Sequence[MemoryBlockSpec] | None = None,
+        force_upload: bool = False,
     ) -> Workflow:
         """
         Sync draft workflow
+        :param force_upload: Skip hash validation when True (for restore operations)
         :raises WorkflowHashNotEqualError
         """
         # fetch draft workflow by app_model
         workflow = self.get_draft_workflow(app_model=app_model)
 
-        if workflow and workflow.unique_hash != unique_hash:
+        if workflow and workflow.unique_hash != unique_hash and not force_upload:
             raise WorkflowHashNotEqualError()
 
         # validate features structure
@@ -224,6 +228,7 @@ class WorkflowService:
                 environment_variables=environment_variables,
                 conversation_variables=conversation_variables,
             )
+            workflow.memory_blocks = memory_blocks or []
             db.session.add(workflow)
         # update draft workflow if found
         else:
@@ -233,6 +238,7 @@ class WorkflowService:
             workflow.updated_at = naive_utc_now()
             workflow.environment_variables = environment_variables
             workflow.conversation_variables = conversation_variables
+            workflow.memory_blocks = memory_blocks or []
 
         # commit db session changes
         db.session.commit()
@@ -243,6 +249,78 @@ class WorkflowService:
         # return draft workflow
         return workflow
 
+    def update_draft_workflow_environment_variables(
+        self,
+        *,
+        app_model: App,
+        environment_variables: Sequence[Variable],
+        account: Account,
+    ):
+        """
+        Update draft workflow environment variables
+        """
+        # fetch draft workflow by app_model
+        workflow = self.get_draft_workflow(app_model=app_model)
+
+        if not workflow:
+            raise ValueError("No draft workflow found.")
+
+        workflow.environment_variables = environment_variables
+        workflow.updated_by = account.id
+        workflow.updated_at = naive_utc_now()
+
+        # commit db session changes
+        db.session.commit()
+
+    def update_draft_workflow_conversation_variables(
+        self,
+        *,
+        app_model: App,
+        conversation_variables: Sequence[Variable],
+        account: Account,
+    ):
+        """
+        Update draft workflow conversation variables
+        """
+        # fetch draft workflow by app_model
+        workflow = self.get_draft_workflow(app_model=app_model)
+
+        if not workflow:
+            raise ValueError("No draft workflow found.")
+
+        workflow.conversation_variables = conversation_variables
+        workflow.updated_by = account.id
+        workflow.updated_at = naive_utc_now()
+
+        # commit db session changes
+        db.session.commit()
+
+    def update_draft_workflow_features(
+        self,
+        *,
+        app_model: App,
+        features: dict,
+        account: Account,
+    ):
+        """
+        Update draft workflow features
+        """
+        # fetch draft workflow by app_model
+        workflow = self.get_draft_workflow(app_model=app_model)
+
+        if not workflow:
+            raise ValueError("No draft workflow found.")
+
+        # validate features structure
+        self.validate_features_structure(app_model=app_model, features=features)
+
+        workflow.features = json.dumps(features)
+        workflow.updated_by = account.id
+        workflow.updated_at = naive_utc_now()
+
+        # commit db session changes
+        db.session.commit()
+
     def publish_workflow(
         self,
         *,
@@ -280,6 +358,7 @@ class WorkflowService:
             marked_name=marked_name,
             marked_comment=marked_comment,
             rag_pipeline_variables=draft_workflow.rag_pipeline_variables,
+            memory_blocks=draft_workflow.memory_blocks,
             features=draft_workflow.features,
         )
 
@@ -636,17 +715,10 @@ class WorkflowService:
                     tenant_id=draft_workflow.tenant_id, start_node_data=start_data, user_inputs=user_inputs
                 )
                 # init variable pool
-                variable_pool = _setup_variable_pool(
-                    query=query,
-                    files=files or [],
-                    user_id=account.id,
-                    user_inputs=user_inputs,
-                    workflow=draft_workflow,
-                    # NOTE(QuantumGhost): We rely on `DraftVarLoader` to load conversation variables.
-                    conversation_variables=[],
-                    node_type=node_type,
-                    conversation_id=conversation_id,
-                )
+                variable_pool = _setup_variable_pool(query=query, files=files or [], user_id=account.id,
+                                                     user_inputs=user_inputs, workflow=draft_workflow,
+                                                     node_type=node_type, conversation_id=conversation_id,
+                                                     conversation_variables=[], is_draft=True)
 
         else:
             variable_pool = VariablePool(
@@ -995,6 +1067,7 @@ def _setup_variable_pool(
     node_type: NodeType,
     conversation_id: str,
     conversation_variables: list[Variable],
+    is_draft: bool
 ):
     # Only inject system variables for START node type.
     if node_type == NodeType.START:
@@ -1013,7 +1086,6 @@ def _setup_variable_pool(
             system_variable.dialogue_count = 1
     else:
         system_variable = SystemVariable.empty()
-
     # init variable pool
     variable_pool = VariablePool(
         system_variables=system_variable,
@@ -1022,6 +1094,12 @@ def _setup_variable_pool(
         # Based on the definition of `VariableUnion`,
         # `list[Variable]` can be safely used as `list[VariableUnion]` since they are compatible.
         conversation_variables=cast(list[VariableUnion], conversation_variables),  #
+        memory_blocks=_fetch_memory_blocks(
+            workflow,
+            MemoryCreatedBy(account_id=user_id),
+            conversation_id,
+            is_draft=is_draft
+        ),
     )
 
     return variable_pool
@@ -1058,3 +1136,30 @@ def _rebuild_single_file(tenant_id: str, value: Any, variable_entity_type: Varia
         return build_from_mappings(mappings=value, tenant_id=tenant_id)
     else:
         raise Exception("unreachable")
+
+
+def _fetch_memory_blocks(
+    workflow: Workflow,
+    created_by: MemoryCreatedBy,
+    conversation_id: str,
+    is_draft: bool
+) -> Mapping[str, str]:
+    memory_blocks = {}
+    memory_block_specs = workflow.memory_blocks
+    from services.chatflow_memory_service import ChatflowMemoryService
+    memories = ChatflowMemoryService.get_memories_by_specs(
+        memory_block_specs=memory_block_specs,
+        tenant_id=workflow.tenant_id,
+        app_id=workflow.app_id,
+        node_id=None,
+        conversation_id=conversation_id,
+        is_draft=is_draft,
+        created_by=created_by,
+    )
+    for memory in memories:
+        if memory.spec.scope == MemoryScope.APP:
+            memory_blocks[memory.spec.id] = memory.value
+        else:  # NODE scope
+            memory_blocks[f"{memory.node_id}.{memory.spec.id}"] = memory.value
+
+    return memory_blocks

api/services/workspace_service.py

@@ -31,7 +31,8 @@ class WorkspaceService:
         assert tenant_account_join is not None, "TenantAccountJoin not found"
         tenant_info["role"] = tenant_account_join.role
 
-        can_replace_logo = FeatureService.get_features(tenant.id).can_replace_logo
+        feature = FeatureService.get_features(tenant.id)
+        can_replace_logo = feature.can_replace_logo
 
         if can_replace_logo and TenantService.has_roles(tenant, [TenantAccountRole.OWNER, TenantAccountRole.ADMIN]):
             base_url = dify_config.FILES_URL
@@ -46,5 +47,20 @@ class WorkspaceService:
                 "remove_webapp_brand": remove_webapp_brand,
                 "replace_webapp_logo": replace_webapp_logo,
             }
+        if dify_config.EDITION == "CLOUD":
+
+            tenant_info["next_credit_reset_date"] = feature.next_credit_reset_date
+
+            from services.credit_pool_service import CreditPoolService
+
+            paid_pool = CreditPoolService.get_pool(tenant_id=tenant.id, pool_type="paid")
+            if paid_pool:
+                tenant_info["trial_credits"] = paid_pool.quota_limit
+                tenant_info["trial_credits_used"] = paid_pool.quota_used
+            else:
+                trial_pool = CreditPoolService.get_pool(tenant_id=tenant.id, pool_type="trial")
+                if trial_pool:
+                    tenant_info["trial_credits"] = trial_pool.quota_limit
+                    tenant_info["trial_credits_used"] = trial_pool.quota_used
 
         return tenant_info

api/tasks/document_indexing_task.py

@@ -1,11 +1,14 @@
 import logging
 import time
+from collections.abc import Callable
 
 import click
 from celery import shared_task
 
 from configs import dify_config
+from core.entities.document_task import DocumentTask
 from core.indexing_runner import DocumentIsPausedError, IndexingRunner
+from core.rag.pipeline.queue import TenantSelfTaskQueue
 from enums.cloud_plan import CloudPlan
 from extensions.ext_database import db
 from libs.datetime_utils import naive_utc_now
@@ -22,8 +25,24 @@ def document_indexing_task(dataset_id: str, document_ids: list):
     :param dataset_id:
     :param document_ids:
 
+    .. warning:: TO BE DEPRECATED
+        This function will be deprecated and removed in a future version.
+        Use normal_document_indexing_task or priority_document_indexing_task instead.
+
     Usage: document_indexing_task.delay(dataset_id, document_ids)
     """
+    logger.warning("document indexing legacy mode received: %s - %s", dataset_id, document_ids)
+    _document_indexing(dataset_id, document_ids)
+
+
+def _document_indexing(dataset_id: str, document_ids: list):
+    """
+    Process document for tasks
+    :param dataset_id:
+    :param document_ids:
+
+    Usage: _document_indexing(dataset_id, document_ids)
+    """
     documents = []
     start_at = time.perf_counter()
 
@@ -87,3 +106,61 @@ def document_indexing_task(dataset_id: str, document_ids: list):
         logger.exception("Document indexing task failed, dataset_id: %s", dataset_id)
     finally:
         db.session.close()
+
+
+def _document_indexing_with_tenant_queue(tenant_id: str, dataset_id: str, document_ids: list, task_func: Callable):
+    try:
+        _document_indexing(dataset_id, document_ids)
+    except Exception:
+        logger.exception("Error processing document indexing %s for tenant %s: %s", dataset_id, tenant_id)
+    finally:
+        tenant_self_task_queue = TenantSelfTaskQueue(tenant_id, "document_indexing")
+
+        # Check if there are waiting tasks in the queue
+        # Use rpop to get the next task from the queue (FIFO order)
+        next_tasks = tenant_self_task_queue.pull_tasks(count=dify_config.TENANT_SELF_TASK_QUEUE_PULL_SIZE)
+
+        logger.info("document indexing tenant isolation queue next tasks: %s", next_tasks)
+
+        if next_tasks:
+            for next_task in next_tasks:
+                document_task = DocumentTask(**next_task)
+                # Process the next waiting task
+                # Keep the flag set to indicate a task is running
+                tenant_self_task_queue.set_task_waiting_time()
+                task_func.delay(  # type: ignore
+                    tenant_id=document_task.tenant_id,
+                    dataset_id=document_task.dataset_id,
+                    document_ids=document_task.document_ids,
+                )
+        else:
+            # No more waiting tasks, clear the flag
+            tenant_self_task_queue.delete_task_key()
+
+
+@shared_task(queue="dataset")
+def normal_document_indexing_task(tenant_id: str, dataset_id: str, document_ids: list):
+    """
+    Async process document
+    :param tenant_id:
+    :param dataset_id:
+    :param document_ids:
+
+    Usage: normal_document_indexing_task.delay(tenant_id, dataset_id, document_ids)
+    """
+    logger.info("normal document indexing task received: %s - %s - %s", tenant_id, dataset_id, document_ids)
+    _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, normal_document_indexing_task)
+
+
+@shared_task(queue="priority_dataset")
+def priority_document_indexing_task(tenant_id: str, dataset_id: str, document_ids: list):
+    """
+    Priority async process document
+    :param tenant_id:
+    :param dataset_id:
+    :param document_ids:
+
+    Usage: priority_document_indexing_task.delay(tenant_id, dataset_id, document_ids)
+    """
+    logger.info("priority document indexing task received: %s - %s - %s", tenant_id, dataset_id, document_ids)
+    _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, priority_document_indexing_task)

api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py

@@ -12,8 +12,10 @@ from celery import shared_task  # type: ignore
 from flask import current_app, g
 from sqlalchemy.orm import Session, sessionmaker
 
+from configs import dify_config
 from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity
 from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
+from core.rag.pipeline.queue import TenantSelfTaskQueue
 from core.repositories.factory import DifyCoreRepositoryFactory
 from extensions.ext_database import db
 from models import Account, Tenant
@@ -22,6 +24,8 @@ from models.enums import WorkflowRunTriggeredFrom
 from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom
 from services.file_service import FileService
 
+logger = logging.getLogger(__name__)
+
 
 @shared_task(queue="priority_pipeline")
 def priority_rag_pipeline_run_task(
@@ -69,6 +73,27 @@ def priority_rag_pipeline_run_task(
         logging.exception(click.style(f"Error running rag pipeline, tenant_id: {tenant_id}", fg="red"))
         raise
     finally:
+        tenant_self_pipeline_task_queue = TenantSelfTaskQueue(tenant_id, "pipeline")
+
+        # Check if there are waiting tasks in the queue
+        # Use rpop to get the next task from the queue (FIFO order)
+        next_file_ids = tenant_self_pipeline_task_queue.pull_tasks(count=dify_config.TENANT_SELF_TASK_QUEUE_PULL_SIZE)
+        logger.info("priority rag pipeline tenant isolation queue next files: %s", next_file_ids)
+
+        if next_file_ids:
+            for next_file_id in next_file_ids:
+                # Process the next waiting task
+                # Keep the flag set to indicate a task is running
+                tenant_self_pipeline_task_queue.set_task_waiting_time()
+                priority_rag_pipeline_run_task.delay(  # type: ignore
+                    rag_pipeline_invoke_entities_file_id=next_file_id.decode("utf-8")
+                    if isinstance(next_file_id, bytes)
+                    else next_file_id,
+                    tenant_id=tenant_id,
+                )
+        else:
+            # No more waiting tasks, clear the flag
+            tenant_self_pipeline_task_queue.delete_task_key()
         file_service = FileService(db.engine)
         file_service.delete_file(rag_pipeline_invoke_entities_file_id)
         db.session.close()

api/tasks/rag_pipeline/rag_pipeline_run_task.py

@@ -12,17 +12,20 @@ from celery import shared_task  # type: ignore
 from flask import current_app, g
 from sqlalchemy.orm import Session, sessionmaker
 
+from configs import dify_config
 from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity
 from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
+from core.rag.pipeline.queue import TenantSelfTaskQueue
 from core.repositories.factory import DifyCoreRepositoryFactory
 from extensions.ext_database import db
-from extensions.ext_redis import redis_client
 from models import Account, Tenant
 from models.dataset import Pipeline
 from models.enums import WorkflowRunTriggeredFrom
 from models.workflow import Workflow, WorkflowNodeExecutionTriggeredFrom
 from services.file_service import FileService
 
+logger = logging.getLogger(__name__)
+
 
 @shared_task(queue="pipeline")
 def rag_pipeline_run_task(
@@ -70,26 +73,27 @@ def rag_pipeline_run_task(
         logging.exception(click.style(f"Error running rag pipeline, tenant_id: {tenant_id}", fg="red"))
         raise
     finally:
-        tenant_self_pipeline_task_queue = f"tenant_self_pipeline_task_queue:{tenant_id}"
-        tenant_pipeline_task_key = f"tenant_pipeline_task:{tenant_id}"
+        tenant_self_pipeline_task_queue = TenantSelfTaskQueue(tenant_id, "pipeline")
 
         # Check if there are waiting tasks in the queue
         # Use rpop to get the next task from the queue (FIFO order)
-        next_file_id = redis_client.rpop(tenant_self_pipeline_task_queue)
+        next_file_ids = tenant_self_pipeline_task_queue.pull_tasks(count=dify_config.TENANT_SELF_TASK_QUEUE_PULL_SIZE)
+        logger.info("rag pipeline tenant isolation queue next files: %s", next_file_ids)
 
-        if next_file_id:
-            # Process the next waiting task
-            # Keep the flag set to indicate a task is running
-            redis_client.setex(tenant_pipeline_task_key, 60 * 60, 1)
-            rag_pipeline_run_task.delay(  # type: ignore
-                rag_pipeline_invoke_entities_file_id=next_file_id.decode("utf-8")
-                if isinstance(next_file_id, bytes)
-                else next_file_id,
-                tenant_id=tenant_id,
-            )
+        if next_file_ids:
+            for next_file_id in next_file_ids:
+                # Process the next waiting task
+                # Keep the flag set to indicate a task is running
+                tenant_self_pipeline_task_queue.set_task_waiting_time()
+                rag_pipeline_run_task.delay(  # type: ignore
+                    rag_pipeline_invoke_entities_file_id=next_file_id.decode("utf-8")
+                    if isinstance(next_file_id, bytes)
+                    else next_file_id,
+                    tenant_id=tenant_id,
+                )
         else:
             # No more waiting tasks, clear the flag
-            redis_client.delete(tenant_pipeline_task_key)
+            tenant_self_pipeline_task_queue.delete_task_key()
         file_service = FileService(db.engine)
         file_service.delete_file(rag_pipeline_invoke_entities_file_id)
         db.session.close()

api/tests/test_containers_integration_tests/core/rag/__init__.py

@@ -0,0 +1 @@
+# Test containers integration tests for RAG pipeline

api/tests/test_containers_integration_tests/core/rag/pipeline/__init__.py

@@ -0,0 +1 @@
+# Test containers integration tests for RAG pipeline queue

api/tests/test_containers_integration_tests/core/rag/pipeline/test_queue_integration.py

@@ -0,0 +1,663 @@
+"""
+Integration tests for TenantSelfTaskQueue using testcontainers.
+
+These tests verify the Redis-based task queue functionality with real Redis instances,
+testing tenant isolation, task serialization, and queue operations in a realistic environment.
+Includes compatibility tests for migrating from legacy string-only queues.
+
+All tests use generic naming to avoid coupling to specific business implementations.
+"""
+
+import time
+from dataclasses import dataclass
+from typing import Any
+from uuid import uuid4
+
+import pytest
+from faker import Faker
+
+from core.rag.pipeline.queue import TASK_WRAPPER_PREFIX, TaskWrapper, TenantSelfTaskQueue
+from extensions.ext_redis import redis_client
+from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
+
+
+@dataclass
+class TestTask:
+    """Test task data structure for testing complex object serialization."""
+    task_id: str
+    tenant_id: str
+    data: dict[str, Any]
+    metadata: dict[str, Any]
+
+
+class TestTenantSelfTaskQueueIntegration:
+    """Integration tests for TenantSelfTaskQueue using testcontainers."""
+
+    @pytest.fixture
+    def fake(self):
+        """Faker instance for generating test data."""
+        return Faker()
+
+    @pytest.fixture
+    def test_tenant_and_account(self, db_session_with_containers, fake):
+        """Create test tenant and account for testing."""
+        # Create account
+        account = Account(
+            email=fake.email(),
+            name=fake.name(),
+            interface_language="en-US",
+            status="active",
+        )
+        db_session_with_containers.add(account)
+        db_session_with_containers.commit()
+
+        # Create tenant
+        tenant = Tenant(
+            name=fake.company(),
+            status="normal",
+        )
+        db_session_with_containers.add(tenant)
+        db_session_with_containers.commit()
+
+        # Create tenant-account join
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER,
+            current=True,
+        )
+        db_session_with_containers.add(join)
+        db_session_with_containers.commit()
+
+        return tenant, account
+
+    @pytest.fixture
+    def test_queue(self, test_tenant_and_account):
+        """Create a generic test queue for testing."""
+        tenant, _ = test_tenant_and_account
+        return TenantSelfTaskQueue(tenant.id, "test_queue")
+
+    @pytest.fixture
+    def secondary_queue(self, test_tenant_and_account):
+        """Create a secondary test queue for testing isolation."""
+        tenant, _ = test_tenant_and_account
+        return TenantSelfTaskQueue(tenant.id, "secondary_queue")
+
+    def test_queue_initialization(self, test_tenant_and_account):
+        """Test queue initialization with correct key generation."""
+        tenant, _ = test_tenant_and_account
+        queue = TenantSelfTaskQueue(tenant.id, "test-key")
+        
+        assert queue.tenant_id == tenant.id
+        assert queue.unique_key == "test-key"
+        assert queue.queue == f"tenant_self_test-key_task_queue:{tenant.id}"
+        assert queue.task_key == f"tenant_test-key_task:{tenant.id}"
+        assert queue.DEFAULT_TASK_TTL == 60 * 60
+
+    def test_tenant_isolation(self, test_tenant_and_account, db_session_with_containers, fake):
+        """Test that different tenants have isolated queues."""
+        tenant1, _ = test_tenant_and_account
+        
+        # Create second tenant
+        tenant2 = Tenant(
+            name=fake.company(),
+            status="normal",
+        )
+        db_session_with_containers.add(tenant2)
+        db_session_with_containers.commit()
+        
+        queue1 = TenantSelfTaskQueue(tenant1.id, "same-key")
+        queue2 = TenantSelfTaskQueue(tenant2.id, "same-key")
+        
+        assert queue1.queue != queue2.queue
+        assert queue1.task_key != queue2.task_key
+        assert queue1.queue == f"tenant_self_same-key_task_queue:{tenant1.id}"
+        assert queue2.queue == f"tenant_self_same-key_task_queue:{tenant2.id}"
+
+    def test_key_isolation(self, test_tenant_and_account):
+        """Test that different keys have isolated queues."""
+        tenant, _ = test_tenant_and_account
+        queue1 = TenantSelfTaskQueue(tenant.id, "key1")
+        queue2 = TenantSelfTaskQueue(tenant.id, "key2")
+        
+        assert queue1.queue != queue2.queue
+        assert queue1.task_key != queue2.task_key
+        assert queue1.queue == f"tenant_self_key1_task_queue:{tenant.id}"
+        assert queue2.queue == f"tenant_self_key2_task_queue:{tenant.id}"
+
+    def test_task_key_operations(self, test_queue):
+        """Test task key operations (get, set, delete)."""
+        # Initially no task key should exist
+        assert test_queue.get_task_key() is None
+        
+        # Set task waiting time with default TTL
+        test_queue.set_task_waiting_time()
+        task_key = test_queue.get_task_key()
+        # Redis returns bytes, convert to string for comparison
+        assert task_key in (b"1", "1")
+        
+        # Set task waiting time with custom TTL
+        custom_ttl = 30
+        test_queue.set_task_waiting_time(custom_ttl)
+        task_key = test_queue.get_task_key()
+        assert task_key in (b"1", "1")
+        
+        # Delete task key
+        test_queue.delete_task_key()
+        assert test_queue.get_task_key() is None
+
+    def test_push_and_pull_string_tasks(self, test_queue):
+        """Test pushing and pulling string tasks."""
+        tasks = ["task1", "task2", "task3"]
+        
+        # Push tasks
+        test_queue.push_tasks(tasks)
+        
+        # Pull tasks one by one (FIFO order)
+        pulled_tasks = []
+        for _ in range(3):
+            task = test_queue.get_next_task()
+            if task:
+                pulled_tasks.append(task)
+        
+        # Should get tasks in FIFO order (lpush + rpop = FIFO)
+        assert pulled_tasks == ["task1", "task2", "task3"]
+
+    def test_push_and_pull_multiple_tasks(self, test_queue):
+        """Test pushing and pulling multiple tasks at once."""
+        tasks = ["task1", "task2", "task3", "task4", "task5"]
+        
+        # Push tasks
+        test_queue.push_tasks(tasks)
+        
+        # Pull multiple tasks
+        pulled_tasks = test_queue.pull_tasks(3)
+        assert len(pulled_tasks) == 3
+        assert pulled_tasks == ["task1", "task2", "task3"]
+        
+        # Pull remaining tasks
+        remaining_tasks = test_queue.pull_tasks(5)
+        assert len(remaining_tasks) == 2
+        assert remaining_tasks == ["task4", "task5"]
+
+    def test_push_and_pull_complex_objects(self, test_queue, fake):
+        """Test pushing and pulling complex object tasks."""
+        # Create complex task objects as dictionaries (not dataclass instances)
+        tasks = [
+            {
+                "task_id": str(uuid4()),
+                "tenant_id": test_queue.tenant_id,
+                "data": {
+                    "file_id": str(uuid4()),
+                    "content": fake.text(),
+                    "metadata": {"size": fake.random_int(1000, 10000)}
+                },
+                "metadata": {
+                    "created_at": fake.iso8601(),
+                    "tags": fake.words(3)
+                }
+            },
+            {
+                "task_id": str(uuid4()),
+                "tenant_id": test_queue.tenant_id,
+                "data": {
+                    "file_id": str(uuid4()),
+                    "content": "测试中文内容",
+                    "metadata": {"size": fake.random_int(1000, 10000)}
+                },
+                "metadata": {
+                    "created_at": fake.iso8601(),
+                    "tags": ["中文", "测试", "emoji🚀"]
+                }
+            }
+        ]
+        
+        # Push complex tasks
+        test_queue.push_tasks(tasks)
+        
+        # Pull tasks
+        pulled_tasks = test_queue.pull_tasks(2)
+        assert len(pulled_tasks) == 2
+        
+        # Verify deserialized tasks match original (FIFO order)
+        for i, pulled_task in enumerate(pulled_tasks):
+            original_task = tasks[i]  # FIFO order
+            assert isinstance(pulled_task, dict)
+            assert pulled_task["task_id"] == original_task["task_id"]
+            assert pulled_task["tenant_id"] == original_task["tenant_id"]
+            assert pulled_task["data"] == original_task["data"]
+            assert pulled_task["metadata"] == original_task["metadata"]
+
+    def test_mixed_task_types(self, test_queue, fake):
+        """Test pushing and pulling mixed string and object tasks."""
+        string_task = "simple_string_task"
+        object_task = {
+            "task_id": str(uuid4()),
+            "dataset_id": str(uuid4()),
+            "document_ids": [str(uuid4()) for _ in range(3)]
+        }
+        
+        tasks = [string_task, object_task, "another_string"]
+        
+        # Push mixed tasks
+        test_queue.push_tasks(tasks)
+        
+        # Pull all tasks
+        pulled_tasks = test_queue.pull_tasks(3)
+        assert len(pulled_tasks) == 3
+        
+        # Verify types and content
+        assert pulled_tasks[0] == string_task
+        assert isinstance(pulled_tasks[1], dict)
+        assert pulled_tasks[1] == object_task
+        assert pulled_tasks[2] == "another_string"
+
+    def test_empty_queue_operations(self, test_queue):
+        """Test operations on empty queue."""
+        # Pull from empty queue
+        tasks = test_queue.pull_tasks(5)
+        assert tasks == []
+        
+        # Get next task from empty queue
+        task = test_queue.get_next_task()
+        assert task is None
+        
+        # Pull zero or negative count
+        assert test_queue.pull_tasks(0) == []
+        assert test_queue.pull_tasks(-1) == []
+
+    def test_task_ttl_expiration(self, test_queue):
+        """Test task key TTL expiration."""
+        # Set task with short TTL
+        short_ttl = 2
+        test_queue.set_task_waiting_time(short_ttl)
+        
+        # Verify task key exists
+        assert test_queue.get_task_key() == b"1" or test_queue.get_task_key() == "1"
+        
+        # Wait for TTL to expire
+        time.sleep(short_ttl + 1)
+        
+        # Verify task key has expired
+        assert test_queue.get_task_key() is None
+
+    def test_large_task_batch(self, test_queue, fake):
+        """Test handling large batches of tasks."""
+        # Create large batch of tasks
+        large_batch = []
+        for i in range(100):
+            task = {
+                "task_id": str(uuid4()),
+                "index": i,
+                "data": fake.text(max_nb_chars=100),
+                "metadata": {"batch_id": str(uuid4())}
+            }
+            large_batch.append(task)
+        
+        # Push large batch
+        test_queue.push_tasks(large_batch)
+        
+        # Pull all tasks
+        pulled_tasks = test_queue.pull_tasks(100)
+        assert len(pulled_tasks) == 100
+        
+        # Verify all tasks were retrieved correctly (FIFO order)
+        for i, task in enumerate(pulled_tasks):
+            assert isinstance(task, dict)
+            assert task["index"] == i  # FIFO order
+
+    def test_queue_operations_isolation(self, test_tenant_and_account, fake):
+        """Test concurrent operations on different queues."""
+        tenant, _ = test_tenant_and_account
+        
+        # Create multiple queues for the same tenant
+        queue1 = TenantSelfTaskQueue(tenant.id, "queue1")
+        queue2 = TenantSelfTaskQueue(tenant.id, "queue2")
+        
+        # Push tasks to different queues
+        queue1.push_tasks(["task1_queue1", "task2_queue1"])
+        queue2.push_tasks(["task1_queue2", "task2_queue2"])
+        
+        # Verify queues are isolated
+        tasks1 = queue1.pull_tasks(2)
+        tasks2 = queue2.pull_tasks(2)
+        
+        assert tasks1 == ["task1_queue1", "task2_queue1"]
+        assert tasks2 == ["task1_queue2", "task2_queue2"]
+        assert tasks1 != tasks2
+
+    def test_task_wrapper_serialization_roundtrip(self, test_queue, fake):
+        """Test TaskWrapper serialization and deserialization roundtrip."""
+        # Create complex nested data
+        complex_data = {
+            "id": str(uuid4()),
+            "nested": {
+                "deep": {
+                    "value": "test",
+                    "numbers": [1, 2, 3, 4, 5],
+                    "unicode": "测试中文",
+                    "emoji": "🚀"
+                }
+            },
+            "metadata": {
+                "created_at": fake.iso8601(),
+                "tags": ["tag1", "tag2", "tag3"]
+            }
+        }
+        
+        # Create wrapper and serialize
+        wrapper = TaskWrapper(complex_data)
+        serialized = wrapper.serialize()
+        
+        # Verify serialization
+        assert isinstance(serialized, str)
+        assert "测试中文" in serialized
+        assert "🚀" in serialized
+        
+        # Deserialize and verify
+        deserialized_wrapper = TaskWrapper.deserialize(serialized)
+        assert deserialized_wrapper.data == complex_data
+
+    def test_error_handling_invalid_json(self, test_queue):
+        """Test error handling for invalid JSON in wrapped tasks."""
+        # Manually create invalid wrapped task
+        invalid_wrapped_task = f"{TASK_WRAPPER_PREFIX}invalid json data"
+        
+        # Push invalid task directly to Redis
+        redis_client.lpush(test_queue.queue, invalid_wrapped_task)
+        
+        # Pull task - should fall back to string
+        task = test_queue.get_next_task()
+        assert task == invalid_wrapped_task
+
+    def test_real_world_processing_scenario(self, test_queue, fake):
+        """Test realistic task processing scenario."""
+        # Simulate various task types
+        tasks = []
+        for i in range(5):
+            task = {
+                "tenant_id": test_queue.tenant_id,
+                "resource_id": str(uuid4()),
+                "resource_ids": [str(uuid4()) for _ in range(fake.random_int(1, 5))]
+            }
+            tasks.append(task)
+        
+        # Push all tasks
+        test_queue.push_tasks(tasks)
+        
+        # Simulate processing tasks one by one
+        processed_tasks = []
+        while True:
+            task = test_queue.get_next_task()
+            if task is None:
+                break
+            
+            processed_tasks.append(task)
+            
+            # Simulate task processing time
+            time.sleep(0.01)
+        
+        # Verify all tasks were processed
+        assert len(processed_tasks) == 5
+        
+        # Verify task content
+        for task in processed_tasks:
+            assert isinstance(task, dict)
+            assert "tenant_id" in task
+            assert "resource_id" in task
+            assert "resource_ids" in task
+            assert task["tenant_id"] == test_queue.tenant_id
+
+    def test_real_world_batch_processing_scenario(self, test_queue, fake):
+        """Test realistic batch processing scenario."""
+        # Simulate batch processing tasks
+        batch_tasks = []
+        for i in range(3):
+            task = {
+                "file_id": str(uuid4()),
+                "tenant_id": test_queue.tenant_id,
+                "user_id": str(uuid4()),
+                "processing_config": {
+                    "model": fake.random_element(["model_a", "model_b", "model_c"]),
+                    "temperature": fake.random.uniform(0.1, 1.0),
+                    "max_tokens": fake.random_int(1000, 4000)
+                },
+                "metadata": {
+                    "source": fake.random_element(["upload", "api", "webhook"]),
+                    "priority": fake.random_element(["low", "normal", "high"])
+                }
+            }
+            batch_tasks.append(task)
+        
+        # Push tasks
+        test_queue.push_tasks(batch_tasks)
+        
+        # Process tasks in batches
+        batch_size = 2
+        processed_tasks = []
+        
+        while True:
+            batch = test_queue.pull_tasks(batch_size)
+            if not batch:
+                break
+            
+            processed_tasks.extend(batch)
+        
+        # Verify all tasks were processed
+        assert len(processed_tasks) == 3
+        
+        # Verify task structure
+        for task in processed_tasks:
+            assert isinstance(task, dict)
+            assert "file_id" in task
+            assert "tenant_id" in task
+            assert "processing_config" in task
+            assert "metadata" in task
+            assert task["tenant_id"] == test_queue.tenant_id
+
+
+class TestTenantSelfTaskQueueCompatibility:
+    """Compatibility tests for migrating from legacy string-only queues."""
+
+    @pytest.fixture
+    def fake(self):
+        """Faker instance for generating test data."""
+        return Faker()
+
+    @pytest.fixture
+    def test_tenant_and_account(self, db_session_with_containers, fake):
+        """Create test tenant and account for testing."""
+        # Create account
+        account = Account(
+            email=fake.email(),
+            name=fake.name(),
+            interface_language="en-US",
+            status="active",
+        )
+        db_session_with_containers.add(account)
+        db_session_with_containers.commit()
+
+        # Create tenant
+        tenant = Tenant(
+            name=fake.company(),
+            status="normal",
+        )
+        db_session_with_containers.add(tenant)
+        db_session_with_containers.commit()
+
+        # Create tenant-account join
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER,
+            current=True,
+        )
+        db_session_with_containers.add(join)
+        db_session_with_containers.commit()
+
+        return tenant, account
+
+    def test_legacy_string_queue_compatibility(self, test_tenant_and_account, fake):
+        """
+        Test compatibility with legacy queues containing only string data.
+        
+        This simulates the scenario where Redis queues already contain string data
+        from the old architecture, and we need to ensure the new code can read them.
+        """
+        tenant, _ = test_tenant_and_account
+        queue = TenantSelfTaskQueue(tenant.id, "legacy_queue")
+        
+        # Simulate legacy string data in Redis queue (using old format)
+        legacy_strings = [
+            "legacy_task_1",
+            "legacy_task_2", 
+            "legacy_task_3",
+            "legacy_task_4",
+            "legacy_task_5"
+        ]
+        
+        # Manually push legacy strings directly to Redis (simulating old system)
+        for legacy_string in legacy_strings:
+            redis_client.lpush(queue.queue, legacy_string)
+        
+        # Verify new code can read legacy string data
+        pulled_tasks = queue.pull_tasks(5)
+        assert len(pulled_tasks) == 5
+        
+        # Verify all tasks are strings (not wrapped)
+        for task in pulled_tasks:
+            assert isinstance(task, str)
+            assert task.startswith("legacy_task_")
+        
+        # Verify order (FIFO from Redis list)
+        expected_order = ["legacy_task_1", "legacy_task_2", "legacy_task_3", "legacy_task_4", "legacy_task_5"]
+        assert pulled_tasks == expected_order
+
+    def test_legacy_queue_migration_scenario(self, test_tenant_and_account, fake):
+        """
+        Test complete migration scenario from legacy to new system.
+        
+        This simulates the real-world scenario where:
+        1. Legacy system has string data in Redis
+        2. New system starts processing the same queue
+        3. Both legacy and new tasks coexist during migration
+        4. New system can handle both formats seamlessly
+        """
+        tenant, _ = test_tenant_and_account
+        queue = TenantSelfTaskQueue(tenant.id, "migration_queue")
+        
+        # Phase 1: Legacy system has data
+        legacy_tasks = [f"legacy_resource_{i}" for i in range(1, 6)]
+        redis_client.lpush(queue.queue, *legacy_tasks)
+        
+        # Phase 2: New system starts processing legacy data
+        processed_legacy = []
+        while True:
+            task = queue.get_next_task()
+            if task is None:
+                break
+            processed_legacy.append(task)
+        
+        # Verify legacy data was processed correctly
+        assert len(processed_legacy) == 5
+        for task in processed_legacy:
+            assert isinstance(task, str)
+            assert task.startswith("legacy_resource_")
+        
+        # Phase 3: New system adds new tasks (mixed types)
+        new_string_tasks = ["new_resource_1", "new_resource_2"]
+        new_object_tasks = [
+            {
+                "resource_id": str(uuid4()),
+                "tenant_id": tenant.id,
+                "processing_type": "new_system",
+                "metadata": {"version": "2.0", "features": ["ai", "ml"]}
+            },
+            {
+                "resource_id": str(uuid4()),
+                "tenant_id": tenant.id,
+                "processing_type": "new_system",
+                "metadata": {"version": "2.0", "features": ["ai", "ml"]}
+            }
+        ]
+        
+        # Push new tasks using new system
+        queue.push_tasks(new_string_tasks)
+        queue.push_tasks(new_object_tasks)
+        
+        # Phase 4: Process all new tasks
+        processed_new = []
+        while True:
+            task = queue.get_next_task()
+            if task is None:
+                break
+            processed_new.append(task)
+        
+        # Verify new tasks were processed correctly
+        assert len(processed_new) == 4
+        
+        string_tasks = [task for task in processed_new if isinstance(task, str)]
+        object_tasks = [task for task in processed_new if isinstance(task, dict)]
+        
+        assert len(string_tasks) == 2
+        assert len(object_tasks) == 2
+        
+        # Verify string tasks
+        for task in string_tasks:
+            assert task.startswith("new_resource_")
+        
+        # Verify object tasks
+        for task in object_tasks:
+            assert isinstance(task, dict)
+            assert "resource_id" in task
+            assert "tenant_id" in task
+            assert task["tenant_id"] == tenant.id
+            assert task["processing_type"] == "new_system"
+
+    def test_legacy_queue_error_recovery(self, test_tenant_and_account, fake):
+        """
+        Test error recovery when legacy queue contains malformed data.
+        
+        This ensures the new system can gracefully handle corrupted or
+        malformed legacy data without crashing.
+        """
+        tenant, _ = test_tenant_and_account
+        queue = TenantSelfTaskQueue(tenant.id, "error_recovery_queue")
+        
+        # Create mix of valid and malformed legacy data
+        mixed_legacy_data = [
+            "valid_legacy_task_1",
+            "valid_legacy_task_2",
+            "malformed_data_without_prefix",  # This should be treated as string
+            "valid_legacy_task_3",
+            f"{TASK_WRAPPER_PREFIX}invalid_json_data",  # This should fall back to string
+            "valid_legacy_task_4"
+        ]
+        
+        # Manually push mixed data directly to Redis
+        redis_client.lpush(queue.queue, *mixed_legacy_data)
+        
+        # Process all tasks
+        processed_tasks = []
+        while True:
+            task = queue.get_next_task()
+            if task is None:
+                break
+            processed_tasks.append(task)
+        
+        # Verify all tasks were processed (no crashes)
+        assert len(processed_tasks) == 6
+        
+        # Verify all tasks are strings (malformed data falls back to string)
+        for task in processed_tasks:
+            assert isinstance(task, str)
+        
+        # Verify valid tasks are preserved
+        valid_tasks = [task for task in processed_tasks if task.startswith("valid_legacy_task_")]
+        assert len(valid_tasks) == 4
+        
+        # Verify malformed data is handled gracefully
+        malformed_tasks = [task for task in processed_tasks if not task.startswith("valid_legacy_task_")]
+        assert len(malformed_tasks) == 2
+        assert "malformed_data_without_prefix" in malformed_tasks
+        assert f"{TASK_WRAPPER_PREFIX}invalid_json_data" in malformed_tasks

api/tests/test_containers_integration_tests/services/test_feature_service.py

@@ -268,6 +268,7 @@ class TestFeatureService:
             mock_config.ENABLE_EMAIL_CODE_LOGIN = True
             mock_config.ENABLE_EMAIL_PASSWORD_LOGIN = True
             mock_config.ENABLE_SOCIAL_OAUTH_LOGIN = False
+            mock_config.ENABLE_COLLABORATION_MODE = True
             mock_config.ALLOW_REGISTER = False
             mock_config.ALLOW_CREATE_WORKSPACE = False
             mock_config.MAIL_TYPE = "smtp"
@@ -292,6 +293,7 @@ class TestFeatureService:
         # Verify authentication settings
         assert result.enable_email_code_login is True
         assert result.enable_email_password_login is False
+        assert result.enable_collaboration_mode is True
         assert result.is_allow_register is False
         assert result.is_allow_create_workspace is False
 
@@ -341,6 +343,7 @@ class TestFeatureService:
             mock_config.ENABLE_EMAIL_CODE_LOGIN = True
             mock_config.ENABLE_EMAIL_PASSWORD_LOGIN = True
             mock_config.ENABLE_SOCIAL_OAUTH_LOGIN = False
+            mock_config.ENABLE_COLLABORATION_MODE = False
             mock_config.ALLOW_REGISTER = True
             mock_config.ALLOW_CREATE_WORKSPACE = True
             mock_config.MAIL_TYPE = "smtp"
@@ -362,6 +365,7 @@ class TestFeatureService:
             assert result.enable_email_code_login is True
             assert result.enable_email_password_login is True
             assert result.enable_social_oauth_login is False
+            assert result.enable_collaboration_mode is False
             assert result.is_allow_register is True
             assert result.is_allow_create_workspace is True
             assert result.is_email_setup is True

api/tests/test_containers_integration_tests/tasks/test_document_indexing_task.py

@@ -1,17 +1,33 @@
+from dataclasses import asdict
 from unittest.mock import MagicMock, patch
 
 import pytest
 from faker import Faker
 
+from core.entities.document_task import DocumentTask
 from enums.cloud_plan import CloudPlan
 from extensions.ext_database import db
 from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
 from models.dataset import Dataset, Document
-from tasks.document_indexing_task import document_indexing_task
+from tasks.document_indexing_task import (
+    _document_indexing,  # Core function
+    _document_indexing_with_tenant_queue,  # Tenant queue wrapper function
+    document_indexing_task,  # Deprecated old interface
+    normal_document_indexing_task,  # New normal task
+    priority_document_indexing_task,  # New priority task
+)
 
 
-class TestDocumentIndexingTask:
-    """Integration tests for document_indexing_task using testcontainers."""
+class TestDocumentIndexingTasks:
+    """Integration tests for document indexing tasks using testcontainers.
+
+    This test class covers:
+    - Core _document_indexing function
+    - Deprecated document_indexing_task function
+    - New normal_document_indexing_task function
+    - New priority_document_indexing_task function
+    - Tenant queue wrapper _document_indexing_with_tenant_queue function
+    """
 
     @pytest.fixture
     def mock_external_service_dependencies(self):
@@ -224,7 +240,7 @@ class TestDocumentIndexingTask:
         document_ids = [doc.id for doc in documents]
 
         # Act: Execute the task
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify the expected outcomes
         # Verify indexing runner was called correctly
@@ -232,10 +248,11 @@ class TestDocumentIndexingTask:
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify documents were updated to parsing status
-        for document in documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
 
         # Verify the run method was called with correct documents
         call_args = mock_external_service_dependencies["indexing_runner_instance"].run.call_args
@@ -261,7 +278,7 @@ class TestDocumentIndexingTask:
         document_ids = [fake.uuid4() for _ in range(3)]
 
         # Act: Execute the task with non-existent dataset
-        document_indexing_task(non_existent_dataset_id, document_ids)
+        _document_indexing(non_existent_dataset_id, document_ids)
 
         # Assert: Verify no processing occurred
         mock_external_service_dependencies["indexing_runner"].assert_not_called()
@@ -291,17 +308,18 @@ class TestDocumentIndexingTask:
         all_document_ids = existing_document_ids + non_existent_document_ids
 
         # Act: Execute the task with mixed document IDs
-        document_indexing_task(dataset.id, all_document_ids)
+        _document_indexing(dataset.id, all_document_ids)
 
         # Assert: Verify only existing documents were processed
         mock_external_service_dependencies["indexing_runner"].assert_called_once()
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify only existing documents were updated
-        for document in documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in existing_document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
 
         # Verify the run method was called with only existing documents
         call_args = mock_external_service_dependencies["indexing_runner_instance"].run.call_args
@@ -333,7 +351,7 @@ class TestDocumentIndexingTask:
         )
 
         # Act: Execute the task
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify exception was handled gracefully
         # The task should complete without raising exceptions
@@ -341,10 +359,11 @@ class TestDocumentIndexingTask:
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify documents were still updated to parsing status before the exception
-        for document in documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing close the session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
 
     def test_document_indexing_task_mixed_document_states(
         self, db_session_with_containers, mock_external_service_dependencies
@@ -407,17 +426,18 @@ class TestDocumentIndexingTask:
         document_ids = [doc.id for doc in all_documents]
 
         # Act: Execute the task with mixed document states
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify processing
         mock_external_service_dependencies["indexing_runner"].assert_called_once()
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify all documents were updated to parsing status
-        for document in all_documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
 
         # Verify the run method was called with all documents
         call_args = mock_external_service_dependencies["indexing_runner_instance"].run.call_args
@@ -470,15 +490,16 @@ class TestDocumentIndexingTask:
         document_ids = [doc.id for doc in all_documents]
 
         # Act: Execute the task with too many documents for sandbox plan
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify error handling
-        for document in all_documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "error"
-            assert document.error is not None
-            assert "batch upload" in document.error
-            assert document.stopped_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "error"
+            assert updated_document.error is not None
+            assert "batch upload" in updated_document.error
+            assert updated_document.stopped_at is not None
 
         # Verify no indexing runner was called
         mock_external_service_dependencies["indexing_runner"].assert_not_called()
@@ -503,17 +524,18 @@ class TestDocumentIndexingTask:
         document_ids = [doc.id for doc in documents]
 
         # Act: Execute the task with billing disabled
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify successful processing
         mock_external_service_dependencies["indexing_runner"].assert_called_once()
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify documents were updated to parsing status
-        for document in documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
 
     def test_document_indexing_task_document_is_paused_error(
         self, db_session_with_containers, mock_external_service_dependencies
@@ -541,7 +563,7 @@ class TestDocumentIndexingTask:
         )
 
         # Act: Execute the task
-        document_indexing_task(dataset.id, document_ids)
+        _document_indexing(dataset.id, document_ids)
 
         # Assert: Verify exception was handled gracefully
         # The task should complete without raising exceptions
@@ -549,7 +571,314 @@ class TestDocumentIndexingTask:
         mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
 
         # Verify documents were still updated to parsing status before the exception
-        for document in documents:
-            db.session.refresh(document)
-            assert document.indexing_status == "parsing"
-            assert document.processing_started_at is not None
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
+
+    # ==================== NEW TESTS FOR REFACTORED FUNCTIONS ====================
+    def test_old_document_indexing_task_success(self, db_session_with_containers, mock_external_service_dependencies):
+        """
+        Test document_indexing_task basic functionality.
+
+        This test verifies:
+        - Task function calls the wrapper correctly
+        - Basic parameter passing works
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        document_ids = [doc.id for doc in documents]
+
+        # Act: Execute the deprecated task (it only takes 2 parameters)
+        document_indexing_task(dataset.id, document_ids)
+
+        # Assert: Verify processing occurred (core logic is tested in _document_indexing tests)
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+    def test_normal_document_indexing_task_success(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test normal_document_indexing_task basic functionality.
+
+        This test verifies:
+        - Task function calls the wrapper correctly
+        - Basic parameter passing works
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        document_ids = [doc.id for doc in documents]
+        tenant_id = dataset.tenant_id
+
+        # Act: Execute the new normal task
+        normal_document_indexing_task(tenant_id, dataset.id, document_ids)
+
+        # Assert: Verify processing occurred (core logic is tested in _document_indexing tests)
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+    def test_priority_document_indexing_task_success(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test priority_document_indexing_task basic functionality.
+
+        This test verifies:
+        - Task function calls the wrapper correctly
+        - Basic parameter passing works
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        document_ids = [doc.id for doc in documents]
+        tenant_id = dataset.tenant_id
+
+        # Act: Execute the new priority task
+        priority_document_indexing_task(tenant_id, dataset.id, document_ids)
+
+        # Assert: Verify processing occurred (core logic is tested in _document_indexing tests)
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+    def test_document_indexing_with_tenant_queue_success(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test _document_indexing_with_tenant_queue function with no waiting tasks.
+
+        This test verifies:
+        - Core indexing logic execution (same as _document_indexing)
+        - Tenant queue cleanup when no waiting tasks
+        - Task function parameter passing
+        - Queue management after processing
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=2
+        )
+        document_ids = [doc.id for doc in documents]
+        tenant_id = dataset.tenant_id
+
+        # Mock the task function
+        from unittest.mock import MagicMock
+        mock_task_func = MagicMock()
+
+        # Act: Execute the wrapper function
+        _document_indexing_with_tenant_queue(tenant_id, dataset.id, document_ids, mock_task_func)
+
+        # Assert: Verify core processing occurred (same as _document_indexing)
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+        # Verify documents were updated (same as _document_indexing)
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
+
+        # Verify the run method was called with correct documents
+        call_args = mock_external_service_dependencies["indexing_runner_instance"].run.call_args
+        assert call_args is not None
+        processed_documents = call_args[0][0]
+        assert len(processed_documents) == 2
+
+        # Verify task function was not called (no waiting tasks)
+        mock_task_func.delay.assert_not_called()
+
+    def test_document_indexing_with_tenant_queue_with_waiting_tasks(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test _document_indexing_with_tenant_queue function with waiting tasks in queue using real Redis.
+
+        This test verifies:
+        - Core indexing logic execution
+        - Real Redis-based tenant queue processing of waiting tasks
+        - Task function calls for waiting tasks
+        - Queue management with multiple tasks using actual Redis operations
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        document_ids = [doc.id for doc in documents]
+        tenant_id = dataset.tenant_id
+        dateset_id = dataset.id
+
+        # Mock the task function
+        from unittest.mock import MagicMock
+        mock_task_func = MagicMock()
+
+        # Use real Redis for TenantSelfTaskQueue
+        from core.rag.pipeline.queue import TenantSelfTaskQueue
+
+        # Create real queue instance
+        queue = TenantSelfTaskQueue(tenant_id, "document_indexing")
+
+        # Add waiting tasks to the real Redis queue
+        waiting_tasks = [
+            DocumentTask(tenant_id=tenant_id, dataset_id=dataset.id, document_ids=["waiting-doc-1"]),
+            DocumentTask(tenant_id=tenant_id, dataset_id=dataset.id, document_ids=["waiting-doc-2"])
+        ]
+        # Convert DocumentTask objects to dictionaries for serialization
+        waiting_task_dicts = [asdict(task) for task in waiting_tasks]
+        queue.push_tasks(waiting_task_dicts)
+
+        # Act: Execute the wrapper function
+        _document_indexing_with_tenant_queue(tenant_id, dataset.id, document_ids, mock_task_func)
+
+        # Assert: Verify core processing occurred
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+        # Verify task function was called for each waiting task
+        assert mock_task_func.delay.call_count == 1
+
+        # Verify correct parameters for each call
+        calls = mock_task_func.delay.call_args_list
+        assert calls[0][1] == {"tenant_id": tenant_id, "dataset_id": dateset_id, "document_ids": ["waiting-doc-1"]}
+
+        # Verify queue is empty after processing (tasks were pulled)
+        remaining_tasks = queue.pull_tasks(count=10)  # Pull more than we added
+        assert len(remaining_tasks) == 1
+
+    def test_document_indexing_with_tenant_queue_error_handling(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test error handling in _document_indexing_with_tenant_queue using real Redis.
+
+        This test verifies:
+        - Exception handling during core processing
+        - Tenant queue cleanup even on errors using real Redis
+        - Proper error logging
+        - Function completes without raising exceptions
+        - Queue management continues despite core processing errors
+        """
+        # Arrange: Create test data
+        dataset, documents = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        document_ids = [doc.id for doc in documents]
+        tenant_id = dataset.tenant_id
+        dateset_id = dataset.id
+
+        # Mock IndexingRunner to raise an exception
+        mock_external_service_dependencies["indexing_runner_instance"].run.side_effect = Exception("Test error")
+
+        # Mock the task function
+        from unittest.mock import MagicMock
+        mock_task_func = MagicMock()
+
+        # Use real Redis for TenantSelfTaskQueue
+        from core.rag.pipeline.queue import TenantSelfTaskQueue
+
+        # Create real queue instance
+        queue = TenantSelfTaskQueue(tenant_id, "document_indexing")
+
+        # Add waiting task to the real Redis queue
+        waiting_task = DocumentTask(tenant_id=tenant_id, dataset_id=dataset.id, document_ids=["waiting-doc-1"])
+        queue.push_tasks([asdict(waiting_task)])
+
+        # Act: Execute the wrapper function
+        _document_indexing_with_tenant_queue(tenant_id, dataset.id, document_ids, mock_task_func)
+
+        # Assert: Verify error was handled gracefully
+        # The function should not raise exceptions
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+        # Verify documents were still updated to parsing status before the exception
+        # Re-query documents from database since _document_indexing uses a different session
+        for doc_id in document_ids:
+            updated_document = db.session.query(Document).filter(Document.id == doc_id).first()
+            assert updated_document.indexing_status == "parsing"
+            assert updated_document.processing_started_at is not None
+
+        # Verify waiting task was still processed despite core processing error
+        mock_task_func.delay.assert_called_once()
+
+        # Verify correct parameters for the call
+        call = mock_task_func.delay.call_args
+        assert call[1] == {"tenant_id": tenant_id, "dataset_id": dateset_id, "document_ids": ["waiting-doc-1"]}
+
+        # Verify queue is empty after processing (task was pulled)
+        remaining_tasks = queue.pull_tasks(count=10)
+        assert len(remaining_tasks) == 0
+
+    def test_document_indexing_with_tenant_queue_tenant_isolation(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test tenant isolation in _document_indexing_with_tenant_queue using real Redis.
+
+        This test verifies:
+        - Different tenants have isolated queues
+        - Tasks from one tenant don't affect another tenant's queue
+        - Queue operations are properly scoped to tenant
+        """
+        # Arrange: Create test data for two different tenants
+        dataset1, documents1 = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+        dataset2, documents2 = self._create_test_dataset_and_documents(
+            db_session_with_containers, mock_external_service_dependencies, document_count=1
+        )
+
+        tenant1_id = dataset1.tenant_id
+        tenant2_id = dataset2.tenant_id
+        dataset1_id = dataset1.id
+        dataset2_id = dataset2.id
+        document_ids1 = [doc.id for doc in documents1]
+        document_ids2 = [doc.id for doc in documents2]
+
+        # Mock the task function
+        from unittest.mock import MagicMock
+        mock_task_func = MagicMock()
+
+        # Use real Redis for TenantSelfTaskQueue
+        from core.rag.pipeline.queue import TenantSelfTaskQueue
+
+        # Create queue instances for both tenants
+        queue1 = TenantSelfTaskQueue(tenant1_id, "document_indexing")
+        queue2 = TenantSelfTaskQueue(tenant2_id, "document_indexing")
+
+        # Add waiting tasks to both queues
+        waiting_task1 = DocumentTask(tenant_id=tenant1_id, dataset_id=dataset1.id, document_ids=["tenant1-doc-1"])
+        waiting_task2 = DocumentTask(tenant_id=tenant2_id, dataset_id=dataset2.id, document_ids=["tenant2-doc-1"])
+
+        queue1.push_tasks([asdict(waiting_task1)])
+        queue2.push_tasks([asdict(waiting_task2)])
+
+        # Act: Execute the wrapper function for tenant1 only
+        _document_indexing_with_tenant_queue(tenant1_id, dataset1.id, document_ids1, mock_task_func)
+
+        # Assert: Verify core processing occurred for tenant1
+        mock_external_service_dependencies["indexing_runner"].assert_called_once()
+        mock_external_service_dependencies["indexing_runner_instance"].run.assert_called_once()
+
+        # Verify only tenant1's waiting task was processed
+        mock_task_func.delay.assert_called_once()
+        call = mock_task_func.delay.call_args
+        assert call[1] == {"tenant_id": tenant1_id, "dataset_id": dataset1_id, "document_ids": ["tenant1-doc-1"]}
+
+        # Verify tenant1's queue is empty
+        remaining_tasks1 = queue1.pull_tasks(count=10)
+        assert len(remaining_tasks1) == 0
+
+        # Verify tenant2's queue still has its task (isolation)
+        remaining_tasks2 = queue2.pull_tasks(count=10)
+        assert len(remaining_tasks2) == 1
+
+        # Verify queue keys are different
+        assert queue1.queue != queue2.queue
+        assert queue1.task_key != queue2.task_key
+

api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py

@@ -0,0 +1,939 @@
+import json
+import uuid
+from unittest.mock import patch
+
+import pytest
+from faker import Faker
+
+from core.app.entities.app_invoke_entities import InvokeFrom, RagPipelineGenerateEntity
+from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
+from core.rag.pipeline.queue import TenantSelfTaskQueue
+from extensions.ext_database import db
+from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
+from models.dataset import Pipeline
+from models.workflow import Workflow
+from tasks.rag_pipeline.priority_rag_pipeline_run_task import (
+    priority_rag_pipeline_run_task,
+    run_single_rag_pipeline_task,
+)
+from tasks.rag_pipeline.rag_pipeline_run_task import rag_pipeline_run_task
+
+
+class TestRagPipelineRunTasks:
+    """Integration tests for RAG pipeline run tasks using testcontainers.
+    
+    This test class covers:
+    - priority_rag_pipeline_run_task function
+    - rag_pipeline_run_task function
+    - run_single_rag_pipeline_task function
+    - Real Redis-based TenantSelfTaskQueue operations
+    - PipelineGenerator._generate method mocking and parameter validation
+    - File operations and cleanup
+    - Error handling and queue management
+    """
+
+    @pytest.fixture
+    def mock_pipeline_generator(self):
+        """Mock PipelineGenerator._generate method."""
+        with patch("core.app.apps.pipeline.pipeline_generator.PipelineGenerator._generate") as mock_generate:
+            # Mock the _generate method to return a simple response
+            mock_generate.return_value = {
+                "answer": "Test response",
+                "metadata": {"test": "data"}
+            }
+            yield mock_generate
+
+    @pytest.fixture
+    def mock_file_service(self):
+        """Mock FileService for file operations."""
+        with (
+            patch("services.file_service.FileService.get_file_content") as mock_get_content,
+            patch("services.file_service.FileService.delete_file") as mock_delete_file,
+        ):
+            yield {
+                "get_content": mock_get_content,
+                "delete_file": mock_delete_file,
+            }
+
+    def _create_test_pipeline_and_workflow(self, db_session_with_containers):
+        """
+        Helper method to create test pipeline and workflow for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+
+        Returns:
+            tuple: (account, tenant, pipeline, workflow) - Created entities
+        """
+        fake = Faker()
+
+        # Create account and tenant
+        account = Account(
+            email=fake.email(),
+            name=fake.name(),
+            interface_language="en-US",
+            status="active",
+        )
+        db.session.add(account)
+        db.session.commit()
+
+        tenant = Tenant(
+            name=fake.company(),
+            status="normal",
+        )
+        db.session.add(tenant)
+        db.session.commit()
+
+        # Create tenant-account join
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER,
+            current=True,
+        )
+        db.session.add(join)
+        db.session.commit()
+
+        # Create workflow
+        workflow = Workflow(
+            id=str(uuid.uuid4()),
+            tenant_id=tenant.id,
+            app_id=str(uuid.uuid4()),
+            type="workflow",
+            version="draft",
+            graph="{}",
+            features="{}",
+            marked_name=fake.company(),
+            marked_comment=fake.text(max_nb_chars=100),
+            created_by=account.id,
+            environment_variables=[],
+            conversation_variables=[],
+            rag_pipeline_variables=[],
+        )
+        db.session.add(workflow)
+        db.session.commit()
+
+        # Create pipeline
+        pipeline = Pipeline(
+            id=str(uuid.uuid4()),
+            tenant_id=tenant.id,
+            workflow_id=workflow.id,
+            name=fake.company(),
+            description=fake.text(max_nb_chars=100),
+            created_by=account.id,
+        )
+        db.session.add(pipeline)
+        db.session.commit()
+
+        # Refresh entities to ensure they're properly loaded
+        db.session.refresh(account)
+        db.session.refresh(tenant)
+        db.session.refresh(workflow)
+        db.session.refresh(pipeline)
+
+        return account, tenant, pipeline, workflow
+
+    def _create_rag_pipeline_invoke_entities(self, account, tenant, pipeline, workflow, count=2):
+        """
+        Helper method to create RAG pipeline invoke entities for testing.
+
+        Args:
+            account: Account instance
+            tenant: Tenant instance
+            pipeline: Pipeline instance
+            workflow: Workflow instance
+            count: Number of entities to create
+
+        Returns:
+            list: List of RagPipelineInvokeEntity instances
+        """
+        fake = Faker()
+        entities = []
+
+        for i in range(count):
+            # Create application generate entity
+            app_config = {
+                "app_id": str(uuid.uuid4()),
+                "app_name": fake.company(),
+                "mode": "workflow",
+                "workflow_id": workflow.id,
+                "tenant_id": tenant.id,
+                "app_mode": "workflow",
+            }
+
+            application_generate_entity = {
+                "task_id": str(uuid.uuid4()),
+                "app_config": app_config,
+                "inputs": {"query": f"Test query {i}"},
+                "files": [],
+                "user_id": account.id,
+                "stream": False,
+                "invoke_from": "published",
+                "workflow_execution_id": str(uuid.uuid4()),
+                "pipeline_config": {
+                    "app_id": str(uuid.uuid4()),
+                    "app_name": fake.company(),
+                    "mode": "workflow",
+                    "workflow_id": workflow.id,
+                    "tenant_id": tenant.id,
+                    "app_mode": "workflow",
+                },
+                "datasource_type": "upload_file",
+                "datasource_info": {},
+                "dataset_id": str(uuid.uuid4()),
+                "batch": "test_batch",
+            }
+
+            entity = RagPipelineInvokeEntity(
+                pipeline_id=pipeline.id,
+                application_generate_entity=application_generate_entity,
+                user_id=account.id,
+                tenant_id=tenant.id,
+                workflow_id=workflow.id,
+                streaming=False,
+                workflow_execution_id=str(uuid.uuid4()),
+                workflow_thread_pool_id=str(uuid.uuid4()),
+            )
+            entities.append(entity)
+
+        return entities
+
+    def _create_file_content_for_entities(self, entities):
+        """
+        Helper method to create file content for RAG pipeline invoke entities.
+
+        Args:
+            entities: List of RagPipelineInvokeEntity instances
+
+        Returns:
+            str: JSON string containing serialized entities
+        """
+        entities_data = [entity.model_dump() for entity in entities]
+        return json.dumps(entities_data)
+
+    def test_priority_rag_pipeline_run_task_success(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test successful priority RAG pipeline run task execution.
+
+        This test verifies:
+        - Task execution with multiple RAG pipeline invoke entities
+        - File content retrieval and parsing
+        - PipelineGenerator._generate method calls with correct parameters
+        - Thread pool execution
+        - File cleanup after execution
+        - Queue management with no waiting tasks
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=2)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Act: Execute the priority task
+        priority_rag_pipeline_run_task(file_id, tenant.id)
+
+        # Assert: Verify expected outcomes
+        # Verify file operations
+        mock_file_service["get_content"].assert_called_once_with(file_id)
+        mock_file_service["delete_file"].assert_called_once_with(file_id)
+
+        # Verify PipelineGenerator._generate was called for each entity
+        assert mock_pipeline_generator.call_count == 2
+
+        # Verify call parameters for each entity
+        calls = mock_pipeline_generator.call_args_list
+        for call in calls:
+            call_kwargs = call[1]  # Get keyword arguments
+            assert call_kwargs["pipeline"].id == pipeline.id
+            assert call_kwargs["workflow_id"] == workflow.id
+            assert call_kwargs["user"].id == account.id
+            assert call_kwargs["invoke_from"] == InvokeFrom.PUBLISHED
+            assert call_kwargs["streaming"] == False
+            assert isinstance(call_kwargs["application_generate_entity"], RagPipelineGenerateEntity)
+
+    def test_rag_pipeline_run_task_success(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test successful regular RAG pipeline run task execution.
+
+        This test verifies:
+        - Task execution with multiple RAG pipeline invoke entities
+        - File content retrieval and parsing
+        - PipelineGenerator._generate method calls with correct parameters
+        - Thread pool execution
+        - File cleanup after execution
+        - Queue management with no waiting tasks
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=3)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Act: Execute the regular task
+        rag_pipeline_run_task(file_id, tenant.id)
+
+        # Assert: Verify expected outcomes
+        # Verify file operations
+        mock_file_service["get_content"].assert_called_once_with(file_id)
+        mock_file_service["delete_file"].assert_called_once_with(file_id)
+
+        # Verify PipelineGenerator._generate was called for each entity
+        assert mock_pipeline_generator.call_count == 3
+
+        # Verify call parameters for each entity
+        calls = mock_pipeline_generator.call_args_list
+        for call in calls:
+            call_kwargs = call[1]  # Get keyword arguments
+            assert call_kwargs["pipeline"].id == pipeline.id
+            assert call_kwargs["workflow_id"] == workflow.id
+            assert call_kwargs["user"].id == account.id
+            assert call_kwargs["invoke_from"] == InvokeFrom.PUBLISHED
+            assert call_kwargs["streaming"] == False
+            assert isinstance(call_kwargs["application_generate_entity"], RagPipelineGenerateEntity)
+
+    def test_priority_rag_pipeline_run_task_with_waiting_tasks(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test priority RAG pipeline run task with waiting tasks in queue using real Redis.
+
+        This test verifies:
+        - Core task execution
+        - Real Redis-based tenant queue processing of waiting tasks
+        - Task function calls for waiting tasks
+        - Queue management with multiple tasks using actual Redis operations
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting tasks to the real Redis queue
+        waiting_file_ids = [str(uuid.uuid4()) for _ in range(2)]
+        queue.push_tasks(waiting_file_ids)
+
+        # Mock the task function calls
+        with patch(
+            "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task.delay"
+        ) as mock_delay:
+            # Act: Execute the priority task
+            priority_rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify core processing occurred
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_file_service["delete_file"].assert_called_once_with(file_id)
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify waiting tasks were processed, pull 1 task a time by default
+            assert mock_delay.call_count == 1
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_ids[0]
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue still has remaining tasks (only 1 was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 1  # 2 original - 1 pulled = 1 remaining
+
+    def test_rag_pipeline_run_task_legacy_compatibility(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test regular RAG pipeline run task with legacy Redis queue format for backward compatibility.
+
+        This test simulates the scenario where:
+        - Old code writes file IDs directly to Redis list using lpush
+        - New worker processes these legacy queue entries
+        - Ensures backward compatibility during deployment transition
+
+        Legacy format: redis_client.lpush(tenant_self_pipeline_task_queue, upload_file.id)
+        New format: TenantSelfTaskQueue.push_tasks([file_id])
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Simulate legacy Redis queue format - direct file IDs in Redis list
+        from extensions.ext_redis import redis_client
+        
+        # Legacy queue key format (old code)
+        legacy_queue_key = f"tenant_self_pipeline_task_queue:{tenant.id}"
+        legacy_task_key = f"tenant_pipeline_task:{tenant.id}"
+        
+        # Add legacy format data to Redis (simulating old code behavior)
+        legacy_file_ids = [str(uuid.uuid4()) for _ in range(3)]
+        for file_id_legacy in legacy_file_ids:
+            redis_client.lpush(legacy_queue_key, file_id_legacy)
+        
+        # Set the task key to indicate there are waiting tasks (legacy behavior)
+        redis_client.set(legacy_task_key, 1, ex=60 * 60)
+
+        # Mock the task function calls
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+            # Act: Execute the priority task with new code but legacy queue data
+            rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify core processing occurred
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_file_service["delete_file"].assert_called_once_with(file_id)
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify waiting tasks were processed, pull 1 task a time by default
+            assert mock_delay.call_count == 1
+
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == legacy_file_ids[0]
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify that new code can process legacy queue entries
+            # The new TenantSelfTaskQueue should be able to read from the legacy format
+            queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+
+            # Verify queue still has remaining tasks (only 1 was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 2  # 3 original - 1 pulled = 2 remaining
+
+        # Cleanup: Remove legacy test data
+        redis_client.delete(legacy_queue_key)
+        redis_client.delete(legacy_task_key)
+
+    def test_rag_pipeline_run_task_with_waiting_tasks(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test regular RAG pipeline run task with waiting tasks in queue using real Redis.
+
+        This test verifies:
+        - Core task execution
+        - Real Redis-based tenant queue processing of waiting tasks
+        - Task function calls for waiting tasks
+        - Queue management with multiple tasks using actual Redis operations
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting tasks to the real Redis queue
+        waiting_file_ids = [str(uuid.uuid4()) for _ in range(3)]
+        queue.push_tasks(waiting_file_ids)
+
+        # Mock the task function calls
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+            # Act: Execute the regular task
+            rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify core processing occurred
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_file_service["delete_file"].assert_called_once_with(file_id)
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify waiting tasks were processed, pull 1 task a time by default
+            assert mock_delay.call_count == 1
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_ids[0]
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue still has remaining tasks (only 1 was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 2  # 3 original - 1 pulled = 2 remaining
+
+    def test_priority_rag_pipeline_run_task_error_handling(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test error handling in priority RAG pipeline run task using real Redis.
+
+        This test verifies:
+        - Exception handling during core processing
+        - Tenant queue cleanup even on errors using real Redis
+        - Proper error logging
+        - Function completes without raising exceptions
+        - Queue management continues despite core processing errors
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Mock PipelineGenerator to raise an exception
+        mock_pipeline_generator.side_effect = Exception("Pipeline generation failed")
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting task to the real Redis queue
+        waiting_file_id = str(uuid.uuid4())
+        queue.push_tasks([waiting_file_id])
+
+        # Mock the task function calls
+        with patch(
+            "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task.delay"
+        ) as mock_delay:
+            # Act: Execute the priority task (should not raise exception)
+            priority_rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify error was handled gracefully
+            # The function should not raise exceptions
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_file_service["delete_file"].assert_called_once_with(file_id)
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify waiting task was still processed despite core processing error
+            mock_delay.assert_called_once()
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue is empty after processing (task was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 0
+
+    def test_rag_pipeline_run_task_error_handling(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test error handling in regular RAG pipeline run task using real Redis.
+
+        This test verifies:
+        - Exception handling during core processing
+        - Tenant queue cleanup even on errors using real Redis
+        - Proper error logging
+        - Function completes without raising exceptions
+        - Queue management continues despite core processing errors
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        file_content = self._create_file_content_for_entities(entities)
+        
+        # Mock file service
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].return_value = file_content
+
+        # Mock PipelineGenerator to raise an exception
+        mock_pipeline_generator.side_effect = Exception("Pipeline generation failed")
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting task to the real Redis queue
+        waiting_file_id = str(uuid.uuid4())
+        queue.push_tasks([waiting_file_id])
+
+        # Mock the task function calls
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+            # Act: Execute the regular task (should not raise exception)
+            rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify error was handled gracefully
+            # The function should not raise exceptions
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_file_service["delete_file"].assert_called_once_with(file_id)
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify waiting task was still processed despite core processing error
+            mock_delay.assert_called_once()
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue is empty after processing (task was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 0
+
+    def test_priority_rag_pipeline_run_task_tenant_isolation(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test tenant isolation in priority RAG pipeline run task using real Redis.
+
+        This test verifies:
+        - Different tenants have isolated queues
+        - Tasks from one tenant don't affect another tenant's queue
+        - Queue operations are properly scoped to tenant
+        """
+        # Arrange: Create test data for two different tenants
+        account1, tenant1, pipeline1, workflow1 = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        account2, tenant2, pipeline2, workflow2 = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        
+        entities1 = self._create_rag_pipeline_invoke_entities(account1, tenant1, pipeline1, workflow1, count=1)
+        entities2 = self._create_rag_pipeline_invoke_entities(account2, tenant2, pipeline2, workflow2, count=1)
+        
+        file_content1 = self._create_file_content_for_entities(entities1)
+        file_content2 = self._create_file_content_for_entities(entities2)
+        
+        # Mock file service
+        file_id1 = str(uuid.uuid4())
+        file_id2 = str(uuid.uuid4())
+        mock_file_service["get_content"].side_effect = [file_content1, file_content2]
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue1 = TenantSelfTaskQueue(tenant1.id, "pipeline")
+        queue2 = TenantSelfTaskQueue(tenant2.id, "pipeline")
+        
+        # Add waiting tasks to both queues
+        waiting_file_id1 = str(uuid.uuid4())
+        waiting_file_id2 = str(uuid.uuid4())
+        
+        queue1.push_tasks([waiting_file_id1])
+        queue2.push_tasks([waiting_file_id2])
+
+        # Mock the task function calls
+        with patch(
+            "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task.delay"
+        ) as mock_delay:
+            # Act: Execute the priority task for tenant1 only
+            priority_rag_pipeline_run_task(file_id1, tenant1.id)
+
+            # Assert: Verify core processing occurred for tenant1
+            assert mock_file_service["get_content"].call_count == 1
+            assert mock_file_service["delete_file"].call_count == 1
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify only tenant1's waiting task was processed
+            mock_delay.assert_called_once()
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id1
+            assert call_kwargs.get('tenant_id') == tenant1.id
+
+            # Verify tenant1's queue is empty
+            remaining_tasks1 = queue1.pull_tasks(count=10)
+            assert len(remaining_tasks1) == 0
+
+            # Verify tenant2's queue still has its task (isolation)
+            remaining_tasks2 = queue2.pull_tasks(count=10)
+            assert len(remaining_tasks2) == 1
+
+            # Verify queue keys are different
+            assert queue1.queue != queue2.queue
+            assert queue1.task_key != queue2.task_key
+
+    def test_rag_pipeline_run_task_tenant_isolation(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test tenant isolation in regular RAG pipeline run task using real Redis.
+
+        This test verifies:
+        - Different tenants have isolated queues
+        - Tasks from one tenant don't affect another tenant's queue
+        - Queue operations are properly scoped to tenant
+        """
+        # Arrange: Create test data for two different tenants
+        account1, tenant1, pipeline1, workflow1 = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        account2, tenant2, pipeline2, workflow2 = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        
+        entities1 = self._create_rag_pipeline_invoke_entities(account1, tenant1, pipeline1, workflow1, count=1)
+        entities2 = self._create_rag_pipeline_invoke_entities(account2, tenant2, pipeline2, workflow2, count=1)
+        
+        file_content1 = self._create_file_content_for_entities(entities1)
+        file_content2 = self._create_file_content_for_entities(entities2)
+        
+        # Mock file service
+        file_id1 = str(uuid.uuid4())
+        file_id2 = str(uuid.uuid4())
+        mock_file_service["get_content"].side_effect = [file_content1, file_content2]
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue1 = TenantSelfTaskQueue(tenant1.id, "pipeline")
+        queue2 = TenantSelfTaskQueue(tenant2.id, "pipeline")
+        
+        # Add waiting tasks to both queues
+        waiting_file_id1 = str(uuid.uuid4())
+        waiting_file_id2 = str(uuid.uuid4())
+        
+        queue1.push_tasks([waiting_file_id1])
+        queue2.push_tasks([waiting_file_id2])
+
+        # Mock the task function calls
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+            # Act: Execute the regular task for tenant1 only
+            rag_pipeline_run_task(file_id1, tenant1.id)
+
+            # Assert: Verify core processing occurred for tenant1
+            assert mock_file_service["get_content"].call_count == 1
+            assert mock_file_service["delete_file"].call_count == 1
+            assert mock_pipeline_generator.call_count == 1
+
+            # Verify only tenant1's waiting task was processed
+            mock_delay.assert_called_once()
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id1
+            assert call_kwargs.get('tenant_id') == tenant1.id
+
+            # Verify tenant1's queue is empty
+            remaining_tasks1 = queue1.pull_tasks(count=10)
+            assert len(remaining_tasks1) == 0
+
+            # Verify tenant2's queue still has its task (isolation)
+            remaining_tasks2 = queue2.pull_tasks(count=10)
+            assert len(remaining_tasks2) == 1
+
+            # Verify queue keys are different
+            assert queue1.queue != queue2.queue
+            assert queue1.task_key != queue2.task_key
+
+    def test_run_single_rag_pipeline_task_success(
+        self, db_session_with_containers, mock_pipeline_generator, flask_app_with_containers
+    ):
+        """
+        Test successful run_single_rag_pipeline_task execution.
+
+        This test verifies:
+        - Single RAG pipeline task execution within Flask app context
+        - Entity validation and database queries
+        - PipelineGenerator._generate method call with correct parameters
+        - Proper Flask context handling
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        entities = self._create_rag_pipeline_invoke_entities(account, tenant, pipeline, workflow, count=1)
+        entity_data = entities[0].model_dump()
+
+        # Act: Execute the single task
+        with flask_app_with_containers.app_context():
+            run_single_rag_pipeline_task(entity_data, flask_app_with_containers)
+
+        # Assert: Verify expected outcomes
+        # Verify PipelineGenerator._generate was called
+        assert mock_pipeline_generator.call_count == 1
+
+        # Verify call parameters
+        call = mock_pipeline_generator.call_args
+        call_kwargs = call[1]  # Get keyword arguments
+        assert call_kwargs["pipeline"].id == pipeline.id
+        assert call_kwargs["workflow_id"] == workflow.id
+        assert call_kwargs["user"].id == account.id
+        assert call_kwargs["invoke_from"] == InvokeFrom.PUBLISHED
+        assert call_kwargs["streaming"] == False
+        assert isinstance(call_kwargs["application_generate_entity"], RagPipelineGenerateEntity)
+
+    def test_run_single_rag_pipeline_task_entity_validation_error(
+        self, db_session_with_containers, mock_pipeline_generator, flask_app_with_containers
+    ):
+        """
+        Test run_single_rag_pipeline_task with invalid entity data.
+
+        This test verifies:
+        - Proper error handling for invalid entity data
+        - Exception logging
+        - Function raises ValueError for missing entities
+        """
+        # Arrange: Create entity data with valid UUIDs but non-existent entities
+        fake = Faker()
+        invalid_entity_data = {
+            "pipeline_id": str(uuid.uuid4()),
+            "application_generate_entity": {
+                "app_config": {
+                    "app_id": str(uuid.uuid4()),
+                    "app_name": "Test App",
+                    "mode": "workflow",
+                    "workflow_id": str(uuid.uuid4()),
+                },
+                "inputs": {"query": "Test query"},
+                "query": "Test query",
+                "response_mode": "blocking",
+                "user": str(uuid.uuid4()),
+                "files": [],
+                "conversation_id": str(uuid.uuid4()),
+            },
+            "user_id": str(uuid.uuid4()),
+            "tenant_id": str(uuid.uuid4()),
+            "workflow_id": str(uuid.uuid4()),
+            "streaming": False,
+            "workflow_execution_id": str(uuid.uuid4()),
+            "workflow_thread_pool_id": str(uuid.uuid4()),
+        }
+
+        # Act & Assert: Execute the single task with non-existent entities (should raise ValueError)
+        with flask_app_with_containers.app_context():
+            with pytest.raises(ValueError, match="Account .* not found"):
+                run_single_rag_pipeline_task(invalid_entity_data, flask_app_with_containers)
+
+        # Assert: Pipeline generator should not be called
+        mock_pipeline_generator.assert_not_called()
+
+    def test_run_single_rag_pipeline_task_database_entity_not_found(
+        self, db_session_with_containers, mock_pipeline_generator, flask_app_with_containers
+    ):
+        """
+        Test run_single_rag_pipeline_task with non-existent database entities.
+
+        This test verifies:
+        - Proper error handling for missing database entities
+        - Exception logging
+        - Function raises ValueError for missing entities
+        """
+        # Arrange: Create test data with non-existent IDs
+        fake = Faker()
+        entity_data = {
+            "pipeline_id": str(uuid.uuid4()),
+            "application_generate_entity": {
+                "app_config": {
+                    "app_id": str(uuid.uuid4()),
+                    "app_name": "Test App",
+                    "mode": "workflow",
+                    "workflow_id": str(uuid.uuid4()),
+                },
+                "inputs": {"query": "Test query"},
+                "query": "Test query",
+                "response_mode": "blocking",
+                "user": str(uuid.uuid4()),
+                "files": [],
+                "conversation_id": str(uuid.uuid4()),
+            },
+            "user_id": str(uuid.uuid4()),
+            "tenant_id": str(uuid.uuid4()),
+            "workflow_id": str(uuid.uuid4()),
+            "streaming": False,
+            "workflow_execution_id": str(uuid.uuid4()),
+            "workflow_thread_pool_id": str(uuid.uuid4()),
+        }
+
+        # Act & Assert: Execute the single task with non-existent entities (should raise ValueError)
+        with flask_app_with_containers.app_context():
+            with pytest.raises(ValueError, match="Account .* not found"):
+                run_single_rag_pipeline_task(entity_data, flask_app_with_containers)
+
+        # Assert: Pipeline generator should not be called
+        mock_pipeline_generator.assert_not_called()
+
+    def test_priority_rag_pipeline_run_task_file_not_found(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test priority RAG pipeline run task with non-existent file.
+
+        This test verifies:
+        - Proper error handling for missing files
+        - Exception logging
+        - Function raises Exception for file errors
+        - Queue management continues despite file errors
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        
+        # Mock file service to raise exception
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].side_effect = Exception("File not found")
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting task to the real Redis queue
+        waiting_file_id = str(uuid.uuid4())
+        queue.push_tasks([waiting_file_id])
+
+        # Mock the task function calls
+        with patch(
+            "tasks.rag_pipeline.priority_rag_pipeline_run_task.priority_rag_pipeline_run_task.delay"
+        ) as mock_delay:
+            # Act & Assert: Execute the priority task (should raise Exception)
+            with pytest.raises(Exception, match="File not found"):
+                priority_rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify error was handled gracefully
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_pipeline_generator.assert_not_called()
+
+            # Verify waiting task was still processed despite file error
+            mock_delay.assert_called_once()
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue is empty after processing (task was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 0
+
+    def test_rag_pipeline_run_task_file_not_found(
+        self, db_session_with_containers, mock_pipeline_generator, mock_file_service
+    ):
+        """
+        Test regular RAG pipeline run task with non-existent file.
+
+        This test verifies:
+        - Proper error handling for missing files
+        - Exception logging
+        - Function raises Exception for file errors
+        - Queue management continues despite file errors
+        """
+        # Arrange: Create test data
+        account, tenant, pipeline, workflow = self._create_test_pipeline_and_workflow(db_session_with_containers)
+        
+        # Mock file service to raise exception
+        file_id = str(uuid.uuid4())
+        mock_file_service["get_content"].side_effect = Exception("File not found")
+
+        # Use real Redis for TenantSelfTaskQueue
+        queue = TenantSelfTaskQueue(tenant.id, "pipeline")
+        
+        # Add waiting task to the real Redis queue
+        waiting_file_id = str(uuid.uuid4())
+        queue.push_tasks([waiting_file_id])
+
+        # Mock the task function calls
+        with patch("tasks.rag_pipeline.rag_pipeline_run_task.rag_pipeline_run_task.delay") as mock_delay:
+            # Act & Assert: Execute the regular task (should raise Exception)
+            with pytest.raises(Exception, match="File not found"):
+                rag_pipeline_run_task(file_id, tenant.id)
+
+            # Assert: Verify error was handled gracefully
+            mock_file_service["get_content"].assert_called_once_with(file_id)
+            mock_pipeline_generator.assert_not_called()
+
+            # Verify waiting task was still processed despite file error
+            mock_delay.assert_called_once()
+            
+            # Verify correct parameters for the call
+            call_kwargs = mock_delay.call_args[1] if mock_delay.call_args else {}
+            assert call_kwargs.get('rag_pipeline_invoke_entities_file_id') == waiting_file_id
+            assert call_kwargs.get('tenant_id') == tenant.id
+
+            # Verify queue is empty after processing (task was pulled)
+            remaining_tasks = queue.pull_tasks(count=10)
+            assert len(remaining_tasks) == 0

api/tests/unit_tests/core/rag/pipeline/test_queue.py

@@ -0,0 +1,369 @@
+"""
+Unit tests for TenantSelfTaskQueue.
+
+These tests verify the Redis-based task queue functionality for tenant-specific
+task management with proper serialization and deserialization.
+"""
+
+import json
+from unittest.mock import MagicMock, patch
+from uuid import uuid4
+
+import pytest
+
+from core.rag.pipeline.queue import TASK_WRAPPER_PREFIX, TaskWrapper, TenantSelfTaskQueue
+
+
+class TestTaskWrapper:
+    """Test cases for TaskWrapper serialization/deserialization."""
+
+    def test_serialize_simple_data(self):
+        """Test serialization of simple data types."""
+        data = {"key": "value", "number": 42, "list": [1, 2, 3]}
+        wrapper = TaskWrapper(data)
+        
+        serialized = wrapper.serialize()
+        assert isinstance(serialized, str)
+        
+        # Verify it's valid JSON
+        parsed = json.loads(serialized)
+        assert parsed == data
+
+    def test_serialize_complex_data(self):
+        """Test serialization of complex nested data."""
+        data = {
+            "nested": {
+                "deep": {
+                    "value": "test",
+                    "numbers": [1, 2, 3, 4, 5]
+                }
+            },
+            "unicode": "测试中文",
+            "special_chars": "!@#$%^&*()"
+        }
+        wrapper = TaskWrapper(data)
+        
+        serialized = wrapper.serialize()
+        parsed = json.loads(serialized)
+        assert parsed == data
+
+    def test_deserialize_valid_data(self):
+        """Test deserialization of valid JSON data."""
+        original_data = {"key": "value", "number": 42}
+        serialized = json.dumps(original_data, ensure_ascii=False)
+        
+        wrapper = TaskWrapper.deserialize(serialized)
+        assert wrapper.data == original_data
+
+    def test_deserialize_invalid_json(self):
+        """Test deserialization handles invalid JSON gracefully."""
+        invalid_json = "{invalid json}"
+        
+        with pytest.raises(json.JSONDecodeError):
+            TaskWrapper.deserialize(invalid_json)
+
+    def test_serialize_ensure_ascii_false(self):
+        """Test that serialization preserves Unicode characters."""
+        data = {"chinese": "中文测试", "emoji": "🚀"}
+        wrapper = TaskWrapper(data)
+        
+        serialized = wrapper.serialize()
+        assert "中文测试" in serialized
+        assert "🚀" in serialized
+
+
+class TestTenantSelfTaskQueue:
+    """Test cases for TenantSelfTaskQueue functionality."""
+
+    @pytest.fixture
+    def mock_redis_client(self):
+        """Mock Redis client for testing."""
+        mock_redis = MagicMock()
+        return mock_redis
+
+    @pytest.fixture
+    def sample_queue(self, mock_redis_client):
+        """Create a sample TenantSelfTaskQueue instance."""
+        return TenantSelfTaskQueue("tenant-123", "test-key")
+
+    def test_initialization(self, sample_queue):
+        """Test queue initialization with correct key generation."""
+        assert sample_queue.tenant_id == "tenant-123"
+        assert sample_queue.unique_key == "test-key"
+        assert sample_queue.queue == "tenant_self_test-key_task_queue:tenant-123"
+        assert sample_queue.task_key == "tenant_test-key_task:tenant-123"
+        assert sample_queue.DEFAULT_TASK_TTL == 60 * 60
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_get_task_key_exists(self, mock_redis, sample_queue):
+        """Test getting task key when it exists."""
+        mock_redis.get.return_value = "1"
+        
+        result = sample_queue.get_task_key()
+        
+        assert result == "1"
+        mock_redis.get.assert_called_once_with("tenant_test-key_task:tenant-123")
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_get_task_key_not_exists(self, mock_redis, sample_queue):
+        """Test getting task key when it doesn't exist."""
+        mock_redis.get.return_value = None
+        
+        result = sample_queue.get_task_key()
+        
+        assert result is None
+        mock_redis.get.assert_called_once_with("tenant_test-key_task:tenant-123")
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_set_task_waiting_time_default_ttl(self, mock_redis, sample_queue):
+        """Test setting task waiting flag with default TTL."""
+        sample_queue.set_task_waiting_time()
+        
+        mock_redis.setex.assert_called_once_with(
+            "tenant_test-key_task:tenant-123", 
+            3600,  # DEFAULT_TASK_TTL
+            1
+        )
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_set_task_waiting_time_custom_ttl(self, mock_redis, sample_queue):
+        """Test setting task waiting flag with custom TTL."""
+        custom_ttl = 1800
+        sample_queue.set_task_waiting_time(custom_ttl)
+        
+        mock_redis.setex.assert_called_once_with(
+            "tenant_test-key_task:tenant-123", 
+            custom_ttl, 
+            1
+        )
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_delete_task_key(self, mock_redis, sample_queue):
+        """Test deleting task key."""
+        sample_queue.delete_task_key()
+        
+        mock_redis.delete.assert_called_once_with("tenant_test-key_task:tenant-123")
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_push_tasks_string_list(self, mock_redis, sample_queue):
+        """Test pushing string tasks directly."""
+        tasks = ["task1", "task2", "task3"]
+        
+        sample_queue.push_tasks(tasks)
+        
+        mock_redis.lpush.assert_called_once_with(
+            "tenant_self_test-key_task_queue:tenant-123",
+            "task1", "task2", "task3"
+        )
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_push_tasks_mixed_types(self, mock_redis, sample_queue):
+        """Test pushing mixed string and object tasks."""
+        tasks = [
+            "string_task",
+            {"object_task": "data", "id": 123},
+            "another_string"
+        ]
+        
+        sample_queue.push_tasks(tasks)
+        
+        # Verify lpush was called
+        mock_redis.lpush.assert_called_once()
+        call_args = mock_redis.lpush.call_args
+        
+        # Check queue name
+        assert call_args[0][0] == "tenant_self_test-key_task_queue:tenant-123"
+        
+        # Check serialized tasks
+        serialized_tasks = call_args[0][1:]
+        assert len(serialized_tasks) == 3
+        assert serialized_tasks[0] == "string_task"
+        assert serialized_tasks[2] == "another_string"
+        
+        # Check object task is wrapped
+        assert serialized_tasks[1].startswith(TASK_WRAPPER_PREFIX)
+        wrapper_data = serialized_tasks[1][len(TASK_WRAPPER_PREFIX):]
+        parsed_data = json.loads(wrapper_data)
+        assert parsed_data == {"object_task": "data", "id": 123}
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_push_tasks_empty_list(self, mock_redis, sample_queue):
+        """Test pushing empty task list."""
+        sample_queue.push_tasks([])
+        
+        mock_redis.lpush.assert_called_once_with(
+            "tenant_self_test-key_task_queue:tenant-123"
+        )
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_default_count(self, mock_redis, sample_queue):
+        """Test pulling tasks with default count (1)."""
+        mock_redis.rpop.side_effect = ["task1", None]
+        
+        result = sample_queue.pull_tasks()
+        
+        assert result == ["task1"]
+        assert mock_redis.rpop.call_count == 1
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_custom_count(self, mock_redis, sample_queue):
+        """Test pulling tasks with custom count."""
+        # First test: pull 3 tasks
+        mock_redis.rpop.side_effect = ["task1", "task2", "task3", None]
+        
+        result = sample_queue.pull_tasks(3)
+        
+        assert result == ["task1", "task2", "task3"]
+        assert mock_redis.rpop.call_count == 3
+
+        # Reset mock for second test
+        mock_redis.reset_mock()
+        mock_redis.rpop.side_effect = ["task1", "task2", None]
+        
+        result = sample_queue.pull_tasks(3)
+        
+        assert result == ["task1", "task2"]
+        assert mock_redis.rpop.call_count == 3
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_zero_count(self, mock_redis, sample_queue):
+        """Test pulling tasks with zero count returns empty list."""
+        result = sample_queue.pull_tasks(0)
+        
+        assert result == []
+        mock_redis.rpop.assert_not_called()
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_negative_count(self, mock_redis, sample_queue):
+        """Test pulling tasks with negative count returns empty list."""
+        result = sample_queue.pull_tasks(-1)
+        
+        assert result == []
+        mock_redis.rpop.assert_not_called()
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_with_wrapped_objects(self, mock_redis, sample_queue):
+        """Test pulling tasks that include wrapped objects."""
+        # Create a wrapped task
+        wrapper = TaskWrapper({"task_id": 123, "data": "test"})
+        wrapped_task = f"{TASK_WRAPPER_PREFIX}{wrapper.serialize()}"
+        
+        mock_redis.rpop.side_effect = [
+            "string_task",
+            wrapped_task.encode('utf-8'),  # Simulate bytes from Redis
+            None
+        ]
+        
+        result = sample_queue.pull_tasks(2)
+        
+        assert len(result) == 2
+        assert result[0] == "string_task"
+        assert result[1] == {"task_id": 123, "data": "test"}
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_with_invalid_wrapped_data(self, mock_redis, sample_queue):
+        """Test pulling tasks with invalid wrapped data falls back to string."""
+        invalid_wrapped = f"{TASK_WRAPPER_PREFIX}invalid json"
+        mock_redis.rpop.side_effect = [invalid_wrapped, None]
+        
+        result = sample_queue.pull_tasks(1)
+        
+        assert result == [invalid_wrapped]
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_pull_tasks_bytes_decoding(self, mock_redis, sample_queue):
+        """Test pulling tasks handles bytes from Redis correctly."""
+        mock_redis.rpop.side_effect = [
+            b"task1",  # bytes
+            "task2",   # string
+            None
+        ]
+        
+        result = sample_queue.pull_tasks(2)
+        
+        assert result == ["task1", "task2"]
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_get_next_task_success(self, mock_redis, sample_queue):
+        """Test getting next single task."""
+        mock_redis.rpop.side_effect = ["task1", None]
+        
+        result = sample_queue.get_next_task()
+        
+        assert result == "task1"
+        assert mock_redis.rpop.call_count == 1
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_get_next_task_empty_queue(self, mock_redis, sample_queue):
+        """Test getting next task when queue is empty."""
+        mock_redis.rpop.return_value = None
+        
+        result = sample_queue.get_next_task()
+        
+        assert result is None
+        mock_redis.rpop.assert_called_once()
+
+    def test_tenant_isolation(self):
+        """Test that different tenants have isolated queues."""
+        queue1 = TenantSelfTaskQueue("tenant-1", "key")
+        queue2 = TenantSelfTaskQueue("tenant-2", "key")
+        
+        assert queue1.queue != queue2.queue
+        assert queue1.task_key != queue2.task_key
+        assert queue1.queue == "tenant_self_key_task_queue:tenant-1"
+        assert queue2.queue == "tenant_self_key_task_queue:tenant-2"
+
+    def test_key_isolation(self):
+        """Test that different keys have isolated queues."""
+        queue1 = TenantSelfTaskQueue("tenant", "key1")
+        queue2 = TenantSelfTaskQueue("tenant", "key2")
+        
+        assert queue1.queue != queue2.queue
+        assert queue1.task_key != queue2.task_key
+        assert queue1.queue == "tenant_self_key1_task_queue:tenant"
+        assert queue2.queue == "tenant_self_key2_task_queue:tenant"
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_complex_object_serialization_roundtrip(self, mock_redis, sample_queue):
+        """Test complex object serialization and deserialization roundtrip."""
+        complex_task = {
+            "id": uuid4().hex,
+            "data": {
+                "nested": {
+                    "deep": [1, 2, 3],
+                    "unicode": "测试中文",
+                    "special": "!@#$%^&*()"
+                }
+            },
+            "metadata": {
+                "created_at": "2024-01-01T00:00:00Z",
+                "tags": ["tag1", "tag2", "tag3"]
+            }
+        }
+        
+        # Push the complex task
+        sample_queue.push_tasks([complex_task])
+        
+        # Verify it was wrapped
+        call_args = mock_redis.lpush.call_args
+        wrapped_task = call_args[0][1]
+        assert wrapped_task.startswith(TASK_WRAPPER_PREFIX)
+        
+        # Simulate pulling it back
+        mock_redis.rpop.return_value = wrapped_task
+        result = sample_queue.pull_tasks(1)
+        
+        assert len(result) == 1
+        assert result[0] == complex_task
+
+    @patch('core.rag.pipeline.queue.redis_client')
+    def test_large_task_list_handling(self, mock_redis, sample_queue):
+        """Test handling of large task lists."""
+        large_task_list = [f"task_{i}" for i in range(1000)]
+        
+        sample_queue.push_tasks(large_task_list)
+        
+        # Verify all tasks were pushed
+        call_args = mock_redis.lpush.call_args
+        assert len(call_args[0]) == 1001  # queue name + 1000 tasks
+        assert list(call_args[0][1:]) == large_task_list

api/tests/unit_tests/services/test_account_service.py

@@ -619,8 +619,13 @@ class TestTenantService:
                 mock_tenant_instance.name = "Test User's Workspace"
                 mock_tenant_class.return_value = mock_tenant_instance
 
-                # Execute test
-                TenantService.create_owner_tenant_if_not_exist(mock_account)
+                # Mock the db import in CreditPoolService to avoid database connection
+                with patch("services.credit_pool_service.db") as mock_credit_pool_db:
+                    mock_credit_pool_db.session.add = MagicMock()
+                    mock_credit_pool_db.session.commit = MagicMock()
+
+                    # Execute test
+                    TenantService.create_owner_tenant_if_not_exist(mock_account)
 
         # Verify tenant was created with correct parameters
         mock_db_dependencies["db"].session.add.assert_called()

api/tests/unit_tests/services/test_document_indexing_task_proxy.py

@@ -0,0 +1,332 @@
+
+from unittest.mock import Mock, patch
+
+from core.entities.document_task import DocumentTask
+from core.rag.pipeline.queue import TenantSelfTaskQueue
+from services.document_indexing_task_proxy import DocumentIndexingTaskProxy
+
+
+class DocumentIndexingTaskProxyTestDataFactory:
+    """Factory class for creating test data and mock objects for DocumentIndexingTaskProxy tests."""
+
+    @staticmethod
+    def create_mock_features(
+        billing_enabled: bool = False,
+        plan: str = "sandbox"
+    ) -> Mock:
+        """Create mock features with billing configuration."""
+        features = Mock()
+        features.billing = Mock()
+        features.billing.enabled = billing_enabled
+        features.billing.subscription = Mock()
+        features.billing.subscription.plan = plan
+        return features
+
+    @staticmethod
+    def create_mock_tenant_queue(has_task_key: bool = False) -> Mock:
+        """Create mock TenantSelfTaskQueue."""
+        queue = Mock(spec=TenantSelfTaskQueue)
+        queue.get_task_key.return_value = "task_key" if has_task_key else None
+        queue.push_tasks = Mock()
+        queue.set_task_waiting_time = Mock()
+        return queue
+
+    @staticmethod
+    def create_document_task_proxy(
+        tenant_id: str = "tenant-123",
+        dataset_id: str = "dataset-456",
+        document_ids: list[str] | None = None
+    ) -> DocumentIndexingTaskProxy:
+        """Create DocumentIndexingTaskProxy instance for testing."""
+        if document_ids is None:
+            document_ids = ["doc-1", "doc-2", "doc-3"]
+        return DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)
+
+
+class TestDocumentIndexingTaskProxy:
+    """Test cases for DocumentIndexingTaskProxy class."""
+
+    def test_initialization(self):
+        """Test DocumentIndexingTaskProxy initialization."""
+        # Arrange
+        tenant_id = "tenant-123"
+        dataset_id = "dataset-456"
+        document_ids = ["doc-1", "doc-2", "doc-3"]
+
+        # Act
+        proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)
+
+        # Assert
+        assert proxy.tenant_id == tenant_id
+        assert proxy.dataset_id == dataset_id
+        assert proxy.document_ids == document_ids
+        assert isinstance(proxy.tenant_self_task_queue, TenantSelfTaskQueue)
+        assert proxy.tenant_self_task_queue.tenant_id == tenant_id
+        assert proxy.tenant_self_task_queue.unique_key == "document_indexing"
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_features_property(self, mock_feature_service):
+        """Test cached_property features."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features()
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+
+        # Act
+        features1 = proxy.features
+        features2 = proxy.features  # Second call should use cached property
+
+        # Assert
+        assert features1 == mock_features
+        assert features2 == mock_features
+        assert features1 is features2  # Should be the same instance due to caching
+        mock_feature_service.get_features.assert_called_once_with("tenant-123")
+
+    @patch('services.document_indexing_task_proxy.normal_document_indexing_task')
+    def test_send_to_direct_queue(self, mock_task):
+        """Test _send_to_direct_queue method."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_direct_queue(mock_task)
+
+        # Assert
+        mock_task.delay.assert_called_once_with(
+            tenant_id="tenant-123",
+            dataset_id="dataset-456",
+            document_ids=["doc-1", "doc-2", "doc-3"]
+        )
+
+    @patch('services.document_indexing_task_proxy.normal_document_indexing_task')
+    def test_send_to_tenant_queue_with_existing_task_key(self, mock_task):
+        """Test _send_to_tenant_queue when task key exists."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy.tenant_self_task_queue = DocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(
+            has_task_key=True
+        )
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_tenant_queue(mock_task)
+
+        # Assert
+        proxy.tenant_self_task_queue.push_tasks.assert_called_once()
+        pushed_tasks = proxy.tenant_self_task_queue.push_tasks.call_args[0][0]
+        assert len(pushed_tasks) == 1
+        assert isinstance(DocumentTask(**pushed_tasks[0]), DocumentTask)
+        assert pushed_tasks[0]["tenant_id"] == "tenant-123"
+        assert pushed_tasks[0]["dataset_id"] == "dataset-456"
+        assert pushed_tasks[0]["document_ids"] == ["doc-1", "doc-2", "doc-3"]
+        mock_task.delay.assert_not_called()
+
+    @patch('services.document_indexing_task_proxy.normal_document_indexing_task')
+    def test_send_to_tenant_queue_without_task_key(self, mock_task):
+        """Test _send_to_tenant_queue when no task key exists."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy.tenant_self_task_queue = DocumentIndexingTaskProxyTestDataFactory.create_mock_tenant_queue(
+            has_task_key=False
+        )
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_tenant_queue(mock_task)
+
+        # Assert
+        proxy.tenant_self_task_queue.set_task_waiting_time.assert_called_once()
+        mock_task.delay.assert_called_once_with(
+            tenant_id="tenant-123",
+            dataset_id="dataset-456",
+            document_ids=["doc-1", "doc-2", "doc-3"]
+        )
+        proxy.tenant_self_task_queue.push_tasks.assert_not_called()
+
+    @patch('services.document_indexing_task_proxy.normal_document_indexing_task')
+    def test_send_to_default_tenant_queue(self, mock_task):
+        """Test _send_to_default_tenant_queue method."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_tenant_queue = Mock()
+
+        # Act
+        proxy._send_to_default_tenant_queue()
+
+        # Assert
+        proxy._send_to_tenant_queue.assert_called_once_with(mock_task)
+
+    @patch('services.document_indexing_task_proxy.priority_document_indexing_task')
+    def test_send_to_priority_tenant_queue(self, mock_task):
+        """Test _send_to_priority_tenant_queue method."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_tenant_queue = Mock()
+
+        # Act
+        proxy._send_to_priority_tenant_queue()
+
+        # Assert
+        proxy._send_to_tenant_queue.assert_called_once_with(mock_task)
+
+    @patch('services.document_indexing_task_proxy.priority_document_indexing_task')
+    def test_send_to_priority_direct_queue(self, mock_task):
+        """Test _send_to_priority_direct_queue method."""
+        # Arrange
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_direct_queue = Mock()
+
+        # Act
+        proxy._send_to_priority_direct_queue()
+
+        # Assert
+        proxy._send_to_direct_queue.assert_called_once_with(mock_task)
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_dispatch_with_billing_enabled_sandbox_plan(self, mock_feature_service):
+        """Test _dispatch method when billing is enabled with sandbox plan."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="sandbox"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_default_tenant_queue = Mock()
+
+        # Act
+        proxy._dispatch()
+
+        # Assert
+        proxy._send_to_default_tenant_queue.assert_called_once()
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_dispatch_with_billing_enabled_non_sandbox_plan(self, mock_feature_service):
+        """Test _dispatch method when billing is enabled with non-sandbox plan."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="team"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+
+        # Act
+        proxy._dispatch()
+
+        # If billing enabled with non sandbox plan, should send to priority tenant queue
+        proxy._send_to_priority_tenant_queue.assert_called_once()
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_dispatch_with_billing_disabled(self, mock_feature_service):
+        """Test _dispatch method when billing is disabled."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=False
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_priority_direct_queue = Mock()
+
+        # Act
+        proxy._dispatch()
+
+        # If billing disabled, for example: self-hosted or enterprise, should send to priority direct queue
+        proxy._send_to_priority_direct_queue.assert_called_once()
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_delay_method(self, mock_feature_service):
+        """Test delay method integration."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="sandbox"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_default_tenant_queue = Mock()
+
+        # Act
+        proxy.delay()
+
+        # Assert
+        # If billing enabled with sandbox plan, should send to default tenant queue
+        proxy._send_to_default_tenant_queue.assert_called_once()
+
+    def test_document_task_dataclass(self):
+        """Test DocumentTask dataclass."""
+        # Arrange
+        tenant_id = "tenant-123"
+        dataset_id = "dataset-456"
+        document_ids = ["doc-1", "doc-2"]
+
+        # Act
+        task = DocumentTask(tenant_id=tenant_id, dataset_id=dataset_id, document_ids=document_ids)
+
+        # Assert
+        assert task.tenant_id == tenant_id
+        assert task.dataset_id == dataset_id
+        assert task.document_ids == document_ids
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_dispatch_edge_case_empty_plan(self, mock_feature_service):
+        """Test _dispatch method with empty plan string."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan=""
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+
+        # Act
+        proxy._dispatch()
+
+        # Assert
+        proxy._send_to_priority_tenant_queue.assert_called_once()
+
+    @patch('services.document_indexing_task_proxy.FeatureService')
+    def test_dispatch_edge_case_none_plan(self, mock_feature_service):
+        """Test _dispatch method with None plan."""
+        # Arrange
+        mock_features = DocumentIndexingTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan=None
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = DocumentIndexingTaskProxyTestDataFactory.create_document_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+
+        # Act
+        proxy._dispatch()
+
+        # Assert
+        proxy._send_to_priority_tenant_queue.assert_called_once()
+
+    def test_initialization_with_empty_document_ids(self):
+        """Test initialization with empty document_ids list."""
+        # Arrange
+        tenant_id = "tenant-123"
+        dataset_id = "dataset-456"
+        document_ids = []
+
+        # Act
+        proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)
+
+        # Assert
+        assert proxy.tenant_id == tenant_id
+        assert proxy.dataset_id == dataset_id
+        assert proxy.document_ids == document_ids
+
+    def test_initialization_with_single_document_id(self):
+        """Test initialization with single document_id."""
+        # Arrange
+        tenant_id = "tenant-123"
+        dataset_id = "dataset-456"
+        document_ids = ["doc-1"]
+
+        # Act
+        proxy = DocumentIndexingTaskProxy(tenant_id, dataset_id, document_ids)
+
+        # Assert
+        assert proxy.tenant_id == tenant_id
+        assert proxy.dataset_id == dataset_id
+        assert proxy.document_ids == document_ids

api/tests/unit_tests/services/test_rag_pipeline_task_proxy.py

@@ -0,0 +1,499 @@
+import json
+from unittest.mock import Mock, patch
+
+import pytest
+
+from core.app.entities.rag_pipeline_invoke_entities import RagPipelineInvokeEntity
+from core.rag.pipeline.queue import TenantSelfTaskQueue
+from services.rag_pipeline.rag_pipeline_task_proxy import RagPipelineTaskProxy
+
+
+class RagPipelineTaskProxyTestDataFactory:
+    """Factory class for creating test data and mock objects for RagPipelineTaskProxy tests."""
+
+    @staticmethod
+    def create_mock_features(
+        billing_enabled: bool = False,
+        plan: str = "sandbox"
+    ) -> Mock:
+        """Create mock features with billing configuration."""
+        features = Mock()
+        features.billing = Mock()
+        features.billing.enabled = billing_enabled
+        features.billing.subscription = Mock()
+        features.billing.subscription.plan = plan
+        return features
+
+    @staticmethod
+    def create_mock_tenant_queue(has_task_key: bool = False) -> Mock:
+        """Create mock TenantSelfTaskQueue."""
+        queue = Mock(spec=TenantSelfTaskQueue)
+        queue.get_task_key.return_value = "task_key" if has_task_key else None
+        queue.push_tasks = Mock()
+        queue.set_task_waiting_time = Mock()
+        return queue
+
+    @staticmethod
+    def create_rag_pipeline_invoke_entity(
+        pipeline_id: str = "pipeline-123",
+        user_id: str = "user-456",
+        tenant_id: str = "tenant-789",
+        workflow_id: str = "workflow-101",
+        streaming: bool = True,
+        workflow_execution_id: str | None = None,
+        workflow_thread_pool_id: str | None = None
+    ) -> RagPipelineInvokeEntity:
+        """Create RagPipelineInvokeEntity instance for testing."""
+        return RagPipelineInvokeEntity(
+            pipeline_id=pipeline_id,
+            application_generate_entity={"key": "value"},
+            user_id=user_id,
+            tenant_id=tenant_id,
+            workflow_id=workflow_id,
+            streaming=streaming,
+            workflow_execution_id=workflow_execution_id,
+            workflow_thread_pool_id=workflow_thread_pool_id
+        )
+
+    @staticmethod
+    def create_rag_pipeline_task_proxy(
+        dataset_tenant_id: str = "tenant-123",
+        user_id: str = "user-456",
+        rag_pipeline_invoke_entities: list[RagPipelineInvokeEntity] | None = None
+    ) -> RagPipelineTaskProxy:
+        """Create RagPipelineTaskProxy instance for testing."""
+        if rag_pipeline_invoke_entities is None:
+            rag_pipeline_invoke_entities = [
+                RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity()
+            ]
+        return RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)
+
+    @staticmethod
+    def create_mock_upload_file(file_id: str = "file-123") -> Mock:
+        """Create mock upload file."""
+        upload_file = Mock()
+        upload_file.id = file_id
+        return upload_file
+
+
+class TestRagPipelineTaskProxy:
+    """Test cases for RagPipelineTaskProxy class."""
+
+    def test_initialization(self):
+        """Test RagPipelineTaskProxy initialization."""
+        # Arrange
+        dataset_tenant_id = "tenant-123"
+        user_id = "user-456"
+        rag_pipeline_invoke_entities = [
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity()
+        ]
+
+        # Act
+        proxy = RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)
+
+        # Assert
+        assert proxy.dataset_tenant_id == dataset_tenant_id
+        assert proxy.user_id == user_id
+        assert proxy.rag_pipeline_invoke_entities == rag_pipeline_invoke_entities
+        assert isinstance(proxy.tenant_self_pipeline_task_queue, TenantSelfTaskQueue)
+        assert proxy.tenant_self_pipeline_task_queue.tenant_id == dataset_tenant_id
+        assert proxy.tenant_self_pipeline_task_queue.unique_key == "pipeline"
+
+    def test_initialization_with_empty_entities(self):
+        """Test initialization with empty rag_pipeline_invoke_entities."""
+        # Arrange
+        dataset_tenant_id = "tenant-123"
+        user_id = "user-456"
+        rag_pipeline_invoke_entities = []
+
+        # Act
+        proxy = RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)
+
+        # Assert
+        assert proxy.dataset_tenant_id == dataset_tenant_id
+        assert proxy.user_id == user_id
+        assert proxy.rag_pipeline_invoke_entities == []
+
+    def test_initialization_with_multiple_entities(self):
+        """Test initialization with multiple rag_pipeline_invoke_entities."""
+        # Arrange
+        dataset_tenant_id = "tenant-123"
+        user_id = "user-456"
+        rag_pipeline_invoke_entities = [
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity(pipeline_id="pipeline-1"),
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity(pipeline_id="pipeline-2"),
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity(pipeline_id="pipeline-3")
+        ]
+
+        # Act
+        proxy = RagPipelineTaskProxy(dataset_tenant_id, user_id, rag_pipeline_invoke_entities)
+
+        # Assert
+        assert len(proxy.rag_pipeline_invoke_entities) == 3
+        assert proxy.rag_pipeline_invoke_entities[0].pipeline_id == "pipeline-1"
+        assert proxy.rag_pipeline_invoke_entities[1].pipeline_id == "pipeline-2"
+        assert proxy.rag_pipeline_invoke_entities[2].pipeline_id == "pipeline-3"
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    def test_features_property(self, mock_feature_service):
+        """Test cached_property features."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features()
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+
+        # Act
+        features1 = proxy.features
+        features2 = proxy.features  # Second call should use cached property
+
+        # Assert
+        assert features1 == mock_features
+        assert features2 == mock_features
+        assert features1 is features2  # Should be the same instance due to caching
+        mock_feature_service.get_features.assert_called_once_with("tenant-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_upload_invoke_entities(self, mock_db, mock_file_service_class):
+        """Test _upload_invoke_entities method."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        result = proxy._upload_invoke_entities()
+
+        # Assert
+        assert result == "file-123"
+        mock_file_service_class.assert_called_once_with(mock_db.engine)
+        
+        # Verify upload_text was called with correct parameters
+        mock_file_service.upload_text.assert_called_once()
+        call_args = mock_file_service.upload_text.call_args
+        json_text, name, user_id, tenant_id = call_args[0]
+        
+        assert name == "rag_pipeline_invoke_entities.json"
+        assert user_id == "user-456"
+        assert tenant_id == "tenant-123"
+        
+        # Verify JSON content
+        parsed_json = json.loads(json_text)
+        assert len(parsed_json) == 1
+        assert parsed_json[0]["pipeline_id"] == "pipeline-123"
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_upload_invoke_entities_with_multiple_entities(self, mock_db, mock_file_service_class):
+        """Test _upload_invoke_entities method with multiple entities."""
+        # Arrange
+        entities = [
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity(pipeline_id="pipeline-1"),
+            RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_invoke_entity(pipeline_id="pipeline-2")
+        ]
+        proxy = RagPipelineTaskProxy("tenant-123", "user-456", entities)
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-456")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        result = proxy._upload_invoke_entities()
+
+        # Assert
+        assert result == "file-456"
+        
+        # Verify JSON content contains both entities
+        call_args = mock_file_service.upload_text.call_args
+        json_text = call_args[0][0]
+        parsed_json = json.loads(json_text)
+        assert len(parsed_json) == 2
+        assert parsed_json[0]["pipeline_id"] == "pipeline-1"
+        assert parsed_json[1]["pipeline_id"] == "pipeline-2"
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task')
+    def test_send_to_direct_queue(self, mock_task):
+        """Test _send_to_direct_queue method."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy.tenant_self_pipeline_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue()
+        upload_file_id = "file-123"
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_direct_queue(upload_file_id, mock_task)
+
+        # If sent to direct queue, tenant_self_pipeline_task_queue should not be called
+        proxy.tenant_self_pipeline_task_queue.push_tasks.assert_not_called()
+
+        # Celery should be called directly
+        mock_task.delay.assert_called_once_with(
+            rag_pipeline_invoke_entities_file_id=upload_file_id,
+            tenant_id="tenant-123"
+        )
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task')
+    def test_send_to_tenant_queue_with_existing_task_key(self, mock_task):
+        """Test _send_to_tenant_queue when task key exists."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy.tenant_self_pipeline_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue(
+            has_task_key=True
+        )
+        upload_file_id = "file-123"
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_tenant_queue(upload_file_id, mock_task)
+
+        # If task key exists, should push tasks to the queue
+        proxy.tenant_self_pipeline_task_queue.push_tasks.assert_called_once_with([upload_file_id])
+        # Celery should not be called directly
+        mock_task.delay.assert_not_called()
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task')
+    def test_send_to_tenant_queue_without_task_key(self, mock_task):
+        """Test _send_to_tenant_queue when no task key exists."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy.tenant_self_pipeline_task_queue = RagPipelineTaskProxyTestDataFactory.create_mock_tenant_queue(
+            has_task_key=False
+        )
+        upload_file_id = "file-123"
+        mock_task.delay = Mock()
+
+        # Act
+        proxy._send_to_tenant_queue(upload_file_id, mock_task)
+
+        # If no task key, should set task waiting time key first
+        proxy.tenant_self_pipeline_task_queue.set_task_waiting_time.assert_called_once()
+        mock_task.delay.assert_called_once_with(
+            rag_pipeline_invoke_entities_file_id=upload_file_id,
+            tenant_id="tenant-123"
+        )
+
+        # The first task should be sent to celery directly, so push tasks should not be called
+        proxy.tenant_self_pipeline_task_queue.push_tasks.assert_not_called()
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.rag_pipeline_run_task')
+    def test_send_to_default_tenant_queue(self, mock_task):
+        """Test _send_to_default_tenant_queue method."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_tenant_queue = Mock()
+        upload_file_id = "file-123"
+
+        # Act
+        proxy._send_to_default_tenant_queue(upload_file_id)
+
+        # Assert
+        proxy._send_to_tenant_queue.assert_called_once_with(upload_file_id, mock_task)
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.priority_rag_pipeline_run_task')
+    def test_send_to_priority_tenant_queue(self, mock_task):
+        """Test _send_to_priority_tenant_queue method."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_tenant_queue = Mock()
+        upload_file_id = "file-123"
+
+        # Act
+        proxy._send_to_priority_tenant_queue(upload_file_id)
+
+        # Assert
+        proxy._send_to_tenant_queue.assert_called_once_with(upload_file_id, mock_task)
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.priority_rag_pipeline_run_task')
+    def test_send_to_priority_direct_queue(self, mock_task):
+        """Test _send_to_priority_direct_queue method."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_direct_queue = Mock()
+        upload_file_id = "file-123"
+
+        # Act
+        proxy._send_to_priority_direct_queue(upload_file_id)
+
+        # Assert
+        proxy._send_to_direct_queue.assert_called_once_with(upload_file_id, mock_task)
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_with_billing_enabled_sandbox_plan(self, mock_db, mock_file_service_class, mock_feature_service):
+        """Test _dispatch method when billing is enabled with sandbox plan."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="sandbox"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_default_tenant_queue = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy._dispatch()
+
+        # If billing is enabled with sandbox plan, should send to default tenant queue
+        proxy._send_to_default_tenant_queue.assert_called_once_with("file-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_with_billing_enabled_non_sandbox_plan(
+        self, mock_db, mock_file_service_class, mock_feature_service
+    ):
+        """Test _dispatch method when billing is enabled with non-sandbox plan."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="team"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy._dispatch()
+
+        # If billing is enabled with non-sandbox plan, should send to priority tenant queue
+        proxy._send_to_priority_tenant_queue.assert_called_once_with("file-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_with_billing_disabled(self, mock_db, mock_file_service_class, mock_feature_service):
+        """Test _dispatch method when billing is disabled."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=False
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_priority_direct_queue = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy._dispatch()
+
+        # If billing is disabled, for example: self-hosted or enterprise, should send to priority direct queue
+        proxy._send_to_priority_direct_queue.assert_called_once_with("file-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_with_empty_upload_file_id(self, mock_db, mock_file_service_class):
+        """Test _dispatch method when upload_file_id is empty."""
+        # Arrange
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = Mock()
+        mock_upload_file.id = ""  # Empty file ID
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act & Assert
+        with pytest.raises(ValueError, match="upload_file_id is empty"):
+            proxy._dispatch()
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_edge_case_empty_plan(self, mock_db, mock_file_service_class, mock_feature_service):
+        """Test _dispatch method with empty plan string."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan=""
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy._dispatch()
+
+        # Assert
+        proxy._send_to_priority_tenant_queue.assert_called_once_with("file-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_dispatch_edge_case_none_plan(self, mock_db, mock_file_service_class, mock_feature_service):
+        """Test _dispatch method with None plan."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan=None
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._send_to_priority_tenant_queue = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy._dispatch()
+
+        # Assert
+        proxy._send_to_priority_tenant_queue.assert_called_once_with("file-123")
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FeatureService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.FileService')
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.db')
+    def test_delay_method(self, mock_db, mock_file_service_class, mock_feature_service):
+        """Test delay method integration."""
+        # Arrange
+        mock_features = RagPipelineTaskProxyTestDataFactory.create_mock_features(
+            billing_enabled=True, plan="sandbox"
+        )
+        mock_feature_service.get_features.return_value = mock_features
+        proxy = RagPipelineTaskProxyTestDataFactory.create_rag_pipeline_task_proxy()
+        proxy._dispatch = Mock()
+        
+        mock_file_service = Mock()
+        mock_file_service_class.return_value = mock_file_service
+        mock_upload_file = RagPipelineTaskProxyTestDataFactory.create_mock_upload_file("file-123")
+        mock_file_service.upload_text.return_value = mock_upload_file
+
+        # Act
+        proxy.delay()
+
+        # Assert
+        proxy._dispatch.assert_called_once()
+
+    @patch('services.rag_pipeline.rag_pipeline_task_proxy.logger')
+    def test_delay_method_with_empty_entities(self, mock_logger):
+        """Test delay method with empty rag_pipeline_invoke_entities."""
+        # Arrange
+        proxy = RagPipelineTaskProxy("tenant-123", "user-456", [])
+
+        # Act
+        proxy.delay()
+
+        # Assert
+        mock_logger.warning.assert_called_once_with(
+            "Received empty rag pipeline invoke entities, no tasks delivered: %s %s",
+            "tenant-123",
+            "user-456"
+        )

api/uv.lock

@@ -562,6 +562,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/57/f4/a69c20ee4f660081a7dedb1ac57f29be9378e04edfcb90c526b923d4bebc/beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a", size = 142979 },
 ]
 
+[[package]]
+name = "bidict"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" },
+]
+
 [[package]]
 name = "billiard"
 version = "4.2.2"
@@ -1321,6 +1330,7 @@ dependencies = [
     { name = "flask-restx" },
     { name = "flask-sqlalchemy" },
     { name = "gevent" },
+    { name = "gevent-websocket" },
     { name = "gmpy2" },
     { name = "google-api-core" },
     { name = "google-api-python-client" },
@@ -1369,6 +1379,7 @@ dependencies = [
     { name = "pypdfium2" },
     { name = "python-docx" },
     { name = "python-dotenv" },
+    { name = "python-socketio" },
     { name = "pyyaml" },
     { name = "readabilipy" },
     { name = "redis", extra = ["hiredis"] },
@@ -1513,6 +1524,7 @@ requires-dist = [
     { name = "flask-restx", specifier = "~=1.3.0" },
     { name = "flask-sqlalchemy", specifier = "~=3.1.1" },
     { name = "gevent", specifier = "~=25.9.1" },
+    { name = "gevent-websocket", specifier = "~=0.10.1" },
     { name = "gmpy2", specifier = "~=2.2.1" },
     { name = "google-api-core", specifier = "==2.18.0" },
     { name = "google-api-python-client", specifier = "==2.90.0" },
@@ -1561,6 +1573,7 @@ requires-dist = [
     { name = "pypdfium2", specifier = "==4.30.0" },
     { name = "python-docx", specifier = "~=1.1.0" },
     { name = "python-dotenv", specifier = "==1.0.1" },
+    { name = "python-socketio", specifier = "~=5.13.0" },
     { name = "pyyaml", specifier = "~=6.0.1" },
     { name = "readabilipy", specifier = "~=0.3.0" },
     { name = "redis", extras = ["hiredis"], specifier = "~=6.1.0" },
@@ -2116,6 +2129,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d5/98/caf06d5d22a7c129c1fb2fc1477306902a2c8ddfd399cd26bbbd4caf2141/gevent-25.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:4acd6bcd5feabf22c7c5174bd3b9535ee9f088d2bbce789f740ad8d6554b18f3", size = 1682837 },
 ]
 
+[[package]]
+name = "gevent-websocket"
+version = "0.10.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "gevent" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/98/d2/6fa19239ff1ab072af40ebf339acd91fb97f34617c2ee625b8e34bf42393/gevent-websocket-0.10.1.tar.gz", hash = "sha256:7eaef32968290c9121f7c35b973e2cc302ffb076d018c9068d2f5ca8b2d85fb0", size = 18366, upload-time = "2017-03-12T22:46:05.68Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7b/84/2dc373eb6493e00c884cc11e6c059ec97abae2678d42f06bf780570b0193/gevent_websocket-0.10.1-py3-none-any.whl", hash = "sha256:17b67d91282f8f4c973eba0551183fc84f56f1c90c8f6b6b30256f31f66f5242", size = 22987, upload-time = "2017-03-12T22:46:03.611Z" },
+]
+
 [[package]]
 name = "gitdb"
 version = "4.0.12"
@@ -5070,6 +5095,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863 },
 ]
 
+[[package]]
+name = "python-engineio"
+version = "4.12.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "simple-websocket" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/d8/63e5535ab21dc4998ba1cfe13690ccf122883a38f025dca24d6e56c05eba/python_engineio-4.12.3.tar.gz", hash = "sha256:35633e55ec30915e7fc8f7e34ca8d73ee0c080cec8a8cd04faf2d7396f0a7a7a", size = 91910, upload-time = "2025-09-28T06:31:36.765Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d8/f0/c5aa0a69fd9326f013110653543f36ece4913c17921f3e1dbd78e1b423ee/python_engineio-4.12.3-py3-none-any.whl", hash = "sha256:7c099abb2a27ea7ab429c04da86ab2d82698cdd6c52406cb73766fe454feb7e1", size = 59637, upload-time = "2025-09-28T06:31:35.354Z" },
+]
+
 [[package]]
 name = "python-http-client"
 version = "3.3.7"
@@ -5126,6 +5163,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d9/4f/00be2196329ebbff56ce564aa94efb0fbc828d00de250b1980de1a34ab49/python_pptx-1.0.2-py3-none-any.whl", hash = "sha256:160838e0b8565a8b1f67947675886e9fea18aa5e795db7ae531606d68e785cba", size = 472788 },
 ]
 
+[[package]]
+name = "python-socketio"
+version = "5.13.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "bidict" },
+    { name = "python-engineio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/1a/396d50ccf06ee539fa758ce5623b59a9cb27637fc4b2dc07ed08bf495e77/python_socketio-5.13.0.tar.gz", hash = "sha256:ac4e19a0302ae812e23b712ec8b6427ca0521f7c582d6abb096e36e24a263029", size = 121125, upload-time = "2025-04-12T15:46:59.933Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/32/b4fb8585d1be0f68bde7e110dffbcf354915f77ad8c778563f0ad9655c02/python_socketio-5.13.0-py3-none-any.whl", hash = "sha256:51f68d6499f2df8524668c24bcec13ba1414117cfb3a90115c559b601ab10caf", size = 77800, upload-time = "2025-04-12T15:46:58.412Z" },
+]
+
 [[package]]
 name = "pytz"
 version = "2025.2"
@@ -5631,6 +5681,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 },
 ]
 
+[[package]]
+name = "simple-websocket"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "wsproto" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" },
+]
+
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -7030,6 +7092,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591 },
 ]
 
+[[package]]
+name = "wsproto"
+version = "1.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c9/4a/44d3c295350d776427904d73c189e10aeae66d7f555bb2feee16d1e4ba5a/wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", size = 53425, upload-time = "2022-08-23T19:58:21.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" },
+]
+
 [[package]]
 name = "xinference-client"
 version = "1.2.2"

dev/start-worker

@@ -7,4 +7,4 @@ cd "$SCRIPT_DIR/.."
 
 uv --directory api run \
 	celery -A app.celery worker \
-	-P gevent -c 1 --loglevel INFO -Q dataset,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,priority_pipeline,pipeline
+	-P gevent -c 1 --loglevel INFO -Q dataset,priority_dataset,generation,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,priority_pipeline,pipeline

docker/.env.example

@@ -122,6 +122,10 @@ MIGRATION_ENABLED=true
 # The default value is 300 seconds.
 FILES_ACCESS_TIMEOUT=300
 
+# Collaboration mode toggle
+# To open collaboration features, you also need to set SERVER_WORKER_CLASS=geventwebsocket.gunicorn.workers.GeventWebSocketWorker
+ENABLE_COLLABORATION_MODE=false
+
 # Access token expiration time in minutes
 ACCESS_TOKEN_EXPIRE_MINUTES=60
 
@@ -150,6 +154,7 @@ SERVER_WORKER_AMOUNT=1
 
 # Defaults to gevent. If using windows, it can be switched to sync or solo.
 #
+# If enable collaboration mode, it must be set to geventwebsocket.gunicorn.workers.GeventWebSocketWorker
 # Warning: Changing this parameter requires disabling patching for
 # psycopg2 and gRPC (see `gunicorn.conf.py` and `celery_entrypoint.py`).
 # Modifying it may also decrease throughput.

docker/nginx/conf.d/default.conf.template

@@ -14,6 +14,14 @@ server {
       include proxy.conf;
     }
 
+    location /socket.io/ {
+      proxy_pass http://api:5001;
+      include proxy.conf;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+      proxy_cache_bypass $http_upgrade;
+    }
+
     location /v1 {
       proxy_pass http://api:5001;
       include proxy.conf;

docker/nginx/proxy.conf.template

@@ -5,7 +5,7 @@ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_set_header X-Forwarded-Proto $scheme;
 proxy_set_header X-Forwarded-Port $server_port;
 proxy_http_version 1.1;
-proxy_set_header Connection "";
+# proxy_set_header Connection "";
 proxy_buffering off;
 proxy_read_timeout ${NGINX_PROXY_READ_TIMEOUT};
 proxy_send_timeout ${NGINX_PROXY_SEND_TIMEOUT};

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/card-view.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React from 'react'
+import React, { useEffect } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import AppCard from '@/app/components/app/overview/app-card'
@@ -19,6 +19,8 @@ import { asyncRunSafe } from '@/utils'
 import { NEED_REFRESH_APP_LIST_KEY } from '@/config'
 import type { IAppCardProps } from '@/app/components/app/overview/app-card'
 import { useStore as useAppStore } from '@/app/components/app/store'
+import { webSocketClient } from '@/app/components/workflow/collaboration/core/websocket-manager'
+import { collaborationManager } from '@/app/components/workflow/collaboration/core/collaboration-manager'
 
 export type ICardViewProps = {
   appId: string
@@ -47,15 +49,44 @@ const CardView: FC<ICardViewProps> = ({ appId, isInPanel, className }) => {
 
     message ||= (type === 'success' ? 'modifiedSuccessfully' : 'modifiedUnsuccessfully')
 
-    if (type === 'success')
+    if (type === 'success') {
       updateAppDetail()
 
+      // Emit collaboration event to notify other clients of app state changes
+      const socket = webSocketClient.getSocket(appId)
+      if (socket) {
+        socket.emit('collaboration_event', {
+          type: 'app_state_update',
+          data: { timestamp: Date.now() },
+          timestamp: Date.now(),
+        })
+      }
+    }
+
     notify({
       type,
       message: t(`common.actionMsg.${message}`),
     })
   }
 
+  // Listen for collaborative app state updates from other clients
+  useEffect(() => {
+    if (!appId) return
+
+    const unsubscribe = collaborationManager.onAppStateUpdate(async (update: any) => {
+      try {
+        console.log('Received app state update from collaboration:', update)
+        // Update app detail when other clients modify app state
+        await updateAppDetail()
+      }
+      catch (error) {
+        console.error('app state update failed:', error)
+      }
+    })
+
+    return unsubscribe
+  }, [appId])
+
   const onChangeSiteStatus = async (value: boolean) => {
     const [err] = await asyncRunSafe<App>(
       updateAppSiteStatus({

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/chart-view.tsx

@@ -1,19 +1,24 @@
 'use client'
-import React, { useState } from 'react'
+import React, { useCallback, useState } from 'react'
 import dayjs from 'dayjs'
 import quarterOfYear from 'dayjs/plugin/quarterOfYear'
 import { useTranslation } from 'react-i18next'
 import type { PeriodParams } from '@/app/components/app/overview/app-chart'
 import { AvgResponseTime, AvgSessionInteractions, AvgUserInteractions, ConversationsChart, CostChart, EndUsersChart, MessagesChart, TokenPerSecond, UserSatisfactionRate, WorkflowCostChart, WorkflowDailyTerminalsChart, WorkflowMessagesChart } from '@/app/components/app/overview/app-chart'
-import type { Item } from '@/app/components/base/select'
-import { SimpleSelect } from '@/app/components/base/select'
-import { TIME_PERIOD_MAPPING } from '@/app/components/app/log/filter'
 import { useStore as useAppStore } from '@/app/components/app/store'
+import TimeRangePicker from './time-range-picker'
+import { sendGAEvent } from '@/utils/gtag'
 
 dayjs.extend(quarterOfYear)
 
 const today = dayjs()
 
+const TIME_PERIOD_MAPPING = [
+  { value: 0, name: 'today' },
+  { value: 7, name: 'last7days' },
+  { value: 30, name: 'last30days' },
+]
+
 const queryDateFormat = 'YYYY-MM-DD HH:mm'
 
 export type IChartViewProps = {
@@ -26,22 +31,14 @@ export default function ChartView({ appId, headerRight }: IChartViewProps) {
   const appDetail = useAppStore(state => state.appDetail)
   const isChatApp = appDetail?.mode !== 'completion' && appDetail?.mode !== 'workflow'
   const isWorkflow = appDetail?.mode === 'workflow'
-  const [period, setPeriod] = useState<PeriodParams>({ name: t('appLog.filter.period.last7days'), query: { start: today.subtract(7, 'day').startOf('day').format(queryDateFormat), end: today.endOf('day').format(queryDateFormat) } })
-
-  const onSelect = (item: Item) => {
-    if (item.value === -1) {
-      setPeriod({ name: item.name, query: undefined })
-    }
-    else if (item.value === 0) {
-      const startOfToday = today.startOf('day').format(queryDateFormat)
-      const endOfToday = today.endOf('day').format(queryDateFormat)
-      setPeriod({ name: item.name, query: { start: startOfToday, end: endOfToday } })
-    }
-    else {
-      setPeriod({ name: item.name, query: { start: today.subtract(item.value as number, 'day').startOf('day').format(queryDateFormat), end: today.endOf('day').format(queryDateFormat) } })
-    }
-  }
-
+  const [period, doSetPeriod] = useState<PeriodParams>({ name: t('appLog.filter.period.today'), query: { start: today.startOf('day').format(queryDateFormat), end: today.endOf('day').format(queryDateFormat) } })
+  const setPeriod = useCallback((payload: PeriodParams) => {
+    doSetPeriod(payload)
+    sendGAEvent(isWorkflow ? 'filter_workflow_overview_period' : 'filter_chat_conversation_overview_period', {
+      period: payload.query,
+      period_name: payload.name,
+    })
+  }, [doSetPeriod, isWorkflow])
   if (!appDetail)
     return null
 
@@ -50,20 +47,11 @@ export default function ChartView({ appId, headerRight }: IChartViewProps) {
       <div className='mb-4'>
         <div className='system-xl-semibold mb-2 text-text-primary'>{t('common.appMenus.overview')}</div>
         <div className='flex items-center justify-between'>
-          <div className='flex flex-row items-center'>
-            <SimpleSelect
-              items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))}
-              className='mt-0 !w-40'
-              notClearable={true}
-              onSelect={(item) => {
-                const id = item.value
-                const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1'
-                const name = item.name || t('appLog.filter.period.allTime')
-                onSelect({ value, name })
-              }}
-              defaultValue={'2'}
-            />
-          </div>
+          <TimeRangePicker
+            ranges={TIME_PERIOD_MAPPING}
+            onSelect={setPeriod}
+            queryDateFormat={queryDateFormat}
+          />
           {headerRight}
         </div>
       </div>

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/time-range-picker/date-picker.tsx

@@ -0,0 +1,80 @@
+'use client'
+import { RiCalendarLine } from '@remixicon/react'
+import type { Dayjs } from 'dayjs'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import cn from '@/utils/classnames'
+import { formatToLocalTime } from '@/utils/format'
+import { useI18N } from '@/context/i18n'
+import Picker from '@/app/components/base/date-and-time-picker/date-picker'
+import type { TriggerProps } from '@/app/components/base/date-and-time-picker/types'
+import { noop } from 'lodash-es'
+import dayjs from 'dayjs'
+
+type Props = {
+  start: Dayjs
+  end: Dayjs
+  onStartChange: (date?: Dayjs) => void
+  onEndChange: (date?: Dayjs) => void
+}
+
+const today = dayjs()
+const DatePicker: FC<Props> = ({
+  start,
+  end,
+  onStartChange,
+  onEndChange,
+}) => {
+  const { locale } = useI18N()
+
+  const renderDate = useCallback(({ value, handleClickTrigger, isOpen }: TriggerProps) => {
+    return (
+      <div className={cn('system-sm-regular flex h-7 cursor-pointer items-center rounded-lg px-1 text-components-input-text-filled hover:bg-state-base-hover', isOpen && 'bg-state-base-hover')} onClick={handleClickTrigger}>
+        {value ? formatToLocalTime(value, locale, 'MMM D') : ''}
+      </div>
+    )
+  }, [locale])
+
+  const availableStartDate = end.subtract(30, 'day')
+  const startDateDisabled = useCallback((date: Dayjs) => {
+    if (date.isAfter(today, 'date'))
+      return true
+    return !((date.isAfter(availableStartDate, 'date') || date.isSame(availableStartDate, 'date')) && (date.isBefore(end, 'date') || date.isSame(end, 'date')))
+  }, [availableStartDate, end])
+
+  const availableEndDate = start.add(30, 'day')
+  const endDateDisabled = useCallback((date: Dayjs) => {
+    if (date.isAfter(today, 'date'))
+      return true
+    return !((date.isAfter(start, 'date') || date.isSame(start, 'date')) && (date.isBefore(availableEndDate, 'date') || date.isSame(availableEndDate, 'date')))
+  }, [availableEndDate, start])
+
+  return (
+    <div className='flex h-8 items-center space-x-0.5 rounded-lg bg-components-input-bg-normal px-2'>
+      <div className='p-px'>
+        <RiCalendarLine className='size-3.5 text-text-tertiary' />
+      </div>
+      <Picker
+        value={start}
+        onChange={onStartChange}
+        renderTrigger={renderDate}
+        needTimePicker={false}
+        onClear={noop}
+        noConfirm
+        getIsDateDisabled={startDateDisabled}
+      />
+      <span className='system-sm-regular text-text-tertiary'>-</span>
+      <Picker
+        value={end}
+        onChange={onEndChange}
+        renderTrigger={renderDate}
+        needTimePicker={false}
+        onClear={noop}
+        noConfirm
+        getIsDateDisabled={endDateDisabled}
+      />
+    </div>
+
+  )
+}
+export default React.memo(DatePicker)