chore: run shellctl under tini

dify-agent-runtime/docker/Dockerfile

@@ -40,6 +40,7 @@ RUN apt-get update \
         openssh-client \
         procps \
         ripgrep \
+        tini \
         tmux \
         unzip \
         xz-utils \
@@ -79,4 +80,5 @@ WORKDIR /home/dify
 
 EXPOSE 5004
 
+ENTRYPOINT ["/usr/bin/tini", "-g", "--"]
 CMD ["shellctl", "serve", "--listen", "0.0.0.0:5004"]

dify-agent-runtime/docker/sync-e2b-template.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly DEFAULT_TEMPLATE_NAME='dify-agent-local-sandbox'
+readonly DEFAULT_IMAGE_REPOSITORY='langgenius/dify-agent-local-sandbox'
+readonly DEFAULT_PLATFORM='linux/amd64'
+readonly DEFAULT_E2B_CLI_VERSION='2.13.3'
+readonly DEFAULT_WAIT_SECONDS='600'
+readonly DEFAULT_POLL_SECONDS='10'
+readonly DEFAULT_CPU_COUNT='2'
+readonly DEFAULT_MEMORY_MB='1024'
+readonly START_COMMAND='/usr/bin/env SHELLCTL_ENABLE_PATH_ISOLATION=true /usr/local/bin/shellctl serve --listen 0.0.0.0:5004'
+readonly READY_COMMAND='curl -fsS http://localhost:5004/healthz'
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [options]
+
+Synchronize an E2B Template with the published Dify agent runtime image for a
+Git commit. The image is resolved to an immutable platform-specific digest and
+its OCI revision label must match the source commit before E2B is updated.
+
+Options:
+  --source-ref REF          Git ref whose commit should be synchronized (default: HEAD)
+  --image-repository IMAGE Published image repository (default: ${DEFAULT_IMAGE_REPOSITORY})
+  --image-tag TAG           Published image tag (default: full source commit SHA)
+  --template-name NAME      E2B Template name (default: ${DEFAULT_TEMPLATE_NAME})
+  --platform OS/ARCH        Image platform consumed by E2B (default: ${DEFAULT_PLATFORM})
+  --wait-seconds SECONDS    Maximum time to wait for the image tag (default: ${DEFAULT_WAIT_SECONDS})
+  --poll-seconds SECONDS    Delay between registry checks (default: ${DEFAULT_POLL_SECONDS})
+  --cpu-count COUNT         Template vCPU count (default: ${DEFAULT_CPU_COUNT})
+  --memory-mb MIB           Template memory in MiB (default: ${DEFAULT_MEMORY_MB})
+  --no-cache                Disable E2B build cache
+  --dry-run                 Validate and print the build without changing E2B
+  -h, --help                Show this help
+
+Environment:
+  E2B_API_KEY               Required unless --dry-run is used
+  E2B_CLI_VERSION           E2B CLI version (default: ${DEFAULT_E2B_CLI_VERSION})
+  E2B_TEMPLATE_NAME         Default value for --template-name
+  E2B_BASE_IMAGE_REPOSITORY Default value for --image-repository
+EOF
+}
+
+die() {
+  echo "Error: $*" >&2
+  exit 1
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
+}
+
+require_non_negative_integer() {
+  local name="$1"
+  local value="$2"
+
+  [[ "${value}" =~ ^[0-9]+$ ]] || die "${name} must be a non-negative integer: ${value}"
+}
+
+require_positive_integer() {
+  local name="$1"
+  local value="$2"
+
+  require_non_negative_integer "${name}" "${value}"
+  (( value > 0 )) || die "${name} must be greater than zero"
+}
+
+print_command() {
+  printf 'Command:'
+  printf ' %q' "$@"
+  printf '\n'
+}
+
+source_ref='HEAD'
+image_repository="${E2B_BASE_IMAGE_REPOSITORY:-${DEFAULT_IMAGE_REPOSITORY}}"
+image_tag=''
+template_name="${E2B_TEMPLATE_NAME:-${DEFAULT_TEMPLATE_NAME}}"
+platform="${DEFAULT_PLATFORM}"
+e2b_cli_version="${E2B_CLI_VERSION:-${DEFAULT_E2B_CLI_VERSION}}"
+wait_seconds="${DEFAULT_WAIT_SECONDS}"
+poll_seconds="${DEFAULT_POLL_SECONDS}"
+cpu_count="${DEFAULT_CPU_COUNT}"
+memory_mb="${DEFAULT_MEMORY_MB}"
+dry_run=false
+no_cache=false
+
+while (( $# > 0 )); do
+  case "$1" in
+    --source-ref)
+      (( $# >= 2 )) || die '--source-ref requires a value'
+      source_ref="$2"
+      shift 2
+      ;;
+    --image-repository)
+      (( $# >= 2 )) || die '--image-repository requires a value'
+      image_repository="$2"
+      shift 2
+      ;;
+    --image-tag)
+      (( $# >= 2 )) || die '--image-tag requires a value'
+      image_tag="$2"
+      shift 2
+      ;;
+    --template-name)
+      (( $# >= 2 )) || die '--template-name requires a value'
+      template_name="$2"
+      shift 2
+      ;;
+    --platform)
+      (( $# >= 2 )) || die '--platform requires a value'
+      platform="$2"
+      shift 2
+      ;;
+    --wait-seconds)
+      (( $# >= 2 )) || die '--wait-seconds requires a value'
+      wait_seconds="$2"
+      shift 2
+      ;;
+    --poll-seconds)
+      (( $# >= 2 )) || die '--poll-seconds requires a value'
+      poll_seconds="$2"
+      shift 2
+      ;;
+    --cpu-count)
+      (( $# >= 2 )) || die '--cpu-count requires a value'
+      cpu_count="$2"
+      shift 2
+      ;;
+    --memory-mb)
+      (( $# >= 2 )) || die '--memory-mb requires a value'
+      memory_mb="$2"
+      shift 2
+      ;;
+    --no-cache)
+      no_cache=true
+      shift
+      ;;
+    --dry-run)
+      dry_run=true
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+[[ -n "${image_repository}" ]] || die 'image repository cannot be empty'
+[[ -n "${template_name}" ]] || die 'template name cannot be empty'
+[[ "${platform}" == */* ]] || die "platform must use OS/ARCH form: ${platform}"
+require_non_negative_integer 'wait-seconds' "${wait_seconds}"
+require_positive_integer 'poll-seconds' "${poll_seconds}"
+require_positive_integer 'cpu-count' "${cpu_count}"
+require_positive_integer 'memory-mb' "${memory_mb}"
+
+require_command git
+require_command docker
+require_command jq
+require_command mktemp
+
+source_sha="$(git -C "${SCRIPT_DIR}" rev-parse --verify "${source_ref}^{commit}" 2>/dev/null)" \
+  || die "cannot resolve source ref: ${source_ref}"
+[[ -n "${image_tag}" ]] || image_tag="${source_sha}"
+
+image_ref="${image_repository}:${image_tag}"
+platform_os="${platform%%/*}"
+platform_arch="${platform#*/}"
+deadline=$((SECONDS + wait_seconds))
+manifest_json=''
+inspect_error=''
+
+while true; do
+  if manifest_json="$(docker buildx imagetools inspect --format '{{json .Manifest}}' "${image_ref}" 2>&1)"; then
+    break
+  fi
+  inspect_error="${manifest_json}"
+  manifest_json=''
+
+  (( SECONDS < deadline )) || die "image is not available: ${image_ref}${inspect_error:+ (${inspect_error})}"
+  echo "Waiting for published image ${image_ref}..." >&2
+  sleep "${poll_seconds}"
+done
+
+platform_digest="$(
+  jq -r \
+    --arg os "${platform_os}" \
+    --arg arch "${platform_arch}" \
+    '[.manifests[]? | select(.platform.os == $os and .platform.architecture == $arch) | .digest][0] // empty' \
+    <<<"${manifest_json}"
+)"
+[[ "${platform_digest}" == sha256:* ]] \
+  || die "image ${image_ref} does not contain platform ${platform}"
+
+resolved_image="${image_repository}@${platform_digest}"
+image_json="$(docker buildx imagetools inspect --format '{{json .Image}}' "${resolved_image}")" \
+  || die "cannot inspect resolved image: ${resolved_image}"
+
+actual_os="$(jq -r '.os // empty' <<<"${image_json}")"
+actual_arch="$(jq -r '.architecture // empty' <<<"${image_json}")"
+actual_revision="$(jq -r '.config.Labels["org.opencontainers.image.revision"] // empty' <<<"${image_json}")"
+
+[[ "${actual_os}/${actual_arch}" == "${platform}" ]] \
+  || die "resolved image platform ${actual_os}/${actual_arch} does not match ${platform}"
+[[ "${actual_revision}" == "${source_sha}" ]] \
+  || die "image revision ${actual_revision:-<missing>} does not match source commit ${source_sha}"
+
+tmp_dir="$(mktemp -d)"
+cleanup() {
+  rm -rf "${tmp_dir}"
+}
+trap cleanup EXIT
+
+dockerfile="${tmp_dir}/e2b.Dockerfile"
+printf 'FROM %s\nUSER dify\nWORKDIR /home/dify\n' "${resolved_image}" > "${dockerfile}"
+
+e2b_command=(
+  npx --yes "@e2b/cli@${e2b_cli_version}"
+  template create "${template_name}"
+  --path "${tmp_dir}"
+  --dockerfile e2b.Dockerfile
+  --cmd "${START_COMMAND}"
+  --ready-cmd "${READY_COMMAND}"
+  --cpu-count "${cpu_count}"
+  --memory-mb "${memory_mb}"
+)
+if [[ "${no_cache}" == true ]]; then
+  e2b_command+=(--no-cache)
+fi
+
+echo "Source commit: ${source_sha}"
+echo "Published image: ${image_ref}"
+echo "Resolved image: ${resolved_image} (${platform})"
+echo "E2B Template: ${template_name}"
+print_command "${e2b_command[@]}"
+
+if [[ "${dry_run}" == true ]]; then
+  echo 'Dry run; E2B Template was not changed.'
+  exit 0
+fi
+
+require_command npx
+[[ -n "${E2B_API_KEY:-}" ]] || die 'E2B_API_KEY is required'
+
+"${e2b_command[@]}"
+echo "E2B Template synchronized: ${template_name} <- ${resolved_image}"

docker/docker-compose.yaml

@@ -21,12 +21,50 @@ x-shared-api-worker-config: &shared-api-worker-config
       required: false
     - path: ./envs/vectorstores/weaviate.env
       required: false
+    - path: ./envs/vectorstores/qdrant.env
+      required: false
+    - path: ./envs/vectorstores/oceanbase.env
+      required: false
+    - path: ./envs/vectorstores/seekdb.env
+      required: false
+    - path: ./envs/vectorstores/couchbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvector.env
+      required: false
+    - path: ./envs/vectorstores/vastbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvecto-rs.env
+      required: false
+    - path: ./envs/vectorstores/chroma.env
+      required: false
+    - path: ./envs/vectorstores/iris.env
+      required: false
+    - path: ./envs/vectorstores/oracle.env
+      required: false
+    - path: ./envs/vectorstores/opengauss.env
+      required: false
+    - path: ./envs/vectorstores/myscale.env
+      required: false
+    - path: ./envs/vectorstores/matrixone.env
+      required: false
+    - path: ./envs/vectorstores/elasticsearch.env
+      required: false
+    - path: ./envs/vectorstores/opensearch.env
+      required: false
+    - path: ./envs/vectorstores/milvus.env
+      required: false
     - path: ./envs/infrastructure/nginx.env
       required: false
     - path: ./envs/infrastructure/certbot.env
       required: false
     - path: ./envs/infrastructure/ssrf-proxy.env
       required: false
+    - path: ./envs/infrastructure/etcd.env
+      required: false
+    - path: ./envs/infrastructure/minio.env
+      required: false
+    - path: ./envs/infrastructure/milvus-standalone.env
+      required: false
     - ./.env
   networks:
     - ssrf_proxy_network
@@ -49,12 +87,50 @@ x-shared-worker-config: &shared-worker-config
       required: false
     - path: ./envs/vectorstores/weaviate.env
       required: false
+    - path: ./envs/vectorstores/qdrant.env
+      required: false
+    - path: ./envs/vectorstores/oceanbase.env
+      required: false
+    - path: ./envs/vectorstores/seekdb.env
+      required: false
+    - path: ./envs/vectorstores/couchbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvector.env
+      required: false
+    - path: ./envs/vectorstores/vastbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvecto-rs.env
+      required: false
+    - path: ./envs/vectorstores/chroma.env
+      required: false
+    - path: ./envs/vectorstores/iris.env
+      required: false
+    - path: ./envs/vectorstores/oracle.env
+      required: false
+    - path: ./envs/vectorstores/opengauss.env
+      required: false
+    - path: ./envs/vectorstores/myscale.env
+      required: false
+    - path: ./envs/vectorstores/matrixone.env
+      required: false
+    - path: ./envs/vectorstores/elasticsearch.env
+      required: false
+    - path: ./envs/vectorstores/opensearch.env
+      required: false
+    - path: ./envs/vectorstores/milvus.env
+      required: false
     - path: ./envs/infrastructure/nginx.env
       required: false
     - path: ./envs/infrastructure/certbot.env
       required: false
     - path: ./envs/infrastructure/ssrf-proxy.env
       required: false
+    - path: ./envs/infrastructure/etcd.env
+      required: false
+    - path: ./envs/infrastructure/minio.env
+      required: false
+    - path: ./envs/infrastructure/milvus-standalone.env
+      required: false
     - ./.env
   networks:
     - ssrf_proxy_network
@@ -77,12 +153,50 @@ x-shared-worker-beat-config: &shared-worker-beat-config
       required: false
     - path: ./envs/vectorstores/weaviate.env
       required: false
+    - path: ./envs/vectorstores/qdrant.env
+      required: false
+    - path: ./envs/vectorstores/oceanbase.env
+      required: false
+    - path: ./envs/vectorstores/seekdb.env
+      required: false
+    - path: ./envs/vectorstores/couchbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvector.env
+      required: false
+    - path: ./envs/vectorstores/vastbase.env
+      required: false
+    - path: ./envs/vectorstores/pgvecto-rs.env
+      required: false
+    - path: ./envs/vectorstores/chroma.env
+      required: false
+    - path: ./envs/vectorstores/iris.env
+      required: false
+    - path: ./envs/vectorstores/oracle.env
+      required: false
+    - path: ./envs/vectorstores/opengauss.env
+      required: false
+    - path: ./envs/vectorstores/myscale.env
+      required: false
+    - path: ./envs/vectorstores/matrixone.env
+      required: false
+    - path: ./envs/vectorstores/elasticsearch.env
+      required: false
+    - path: ./envs/vectorstores/opensearch.env
+      required: false
+    - path: ./envs/vectorstores/milvus.env
+      required: false
     - path: ./envs/infrastructure/nginx.env
       required: false
     - path: ./envs/infrastructure/certbot.env
       required: false
     - path: ./envs/infrastructure/ssrf-proxy.env
       required: false
+    - path: ./envs/infrastructure/etcd.env
+      required: false
+    - path: ./envs/infrastructure/minio.env
+      required: false
+    - path: ./envs/infrastructure/milvus-standalone.env
+      required: false
     - ./.env
   networks:
     - ssrf_proxy_network
@@ -137,11 +251,18 @@ services:
       db_mysql:
         condition: service_healthy
         required: false
+      oceanbase:
+        condition: service_healthy
+        required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
       agent_backend:
         condition: service_started
     volumes:
+      # Mount the storage directory to the container, for storing user files.
       - ./volumes/app/storage:/app/api/storage
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5001/health"]
@@ -149,6 +270,9 @@ services:
       timeout: 5s
       retries: 3
       start_period: 30s
+    networks:
+      - ssrf_proxy_network
+      - default
 
   # WebSocket service for workflow collaboration.
   api_websocket:
@@ -171,8 +295,12 @@ services:
         required: false
       redis:
         condition: service_started
+    networks:
+      - ssrf_proxy_network
+      - default
 
   # worker service
+  # The Celery worker for processing all queues (dataset, workflow, mail, etc.)
   worker:
     <<: *shared-worker-config
     image: langgenius/dify-api:1.16.1
@@ -197,11 +325,18 @@ services:
       db_mysql:
         condition: service_healthy
         required: false
+      oceanbase:
+        condition: service_healthy
+        required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
       agent_backend:
         condition: service_started
     volumes:
+      # Mount the storage directory to the container, for storing user files.
       - ./volumes/app/storage:/app/api/storage
     healthcheck:
       test: ["CMD-SHELL", "celery -A celery_healthcheck.celery inspect ping"]
@@ -210,8 +345,12 @@ services:
       retries: 3
       start_period: 60s
       disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true}
+    networks:
+      - ssrf_proxy_network
+      - default
 
   # worker_beat service
+  # Celery beat for scheduling periodic tasks.
   worker_beat:
     <<: *shared-worker-beat-config
     image: langgenius/dify-api:1.16.1
@@ -226,6 +365,12 @@ services:
       db_mysql:
         condition: service_healthy
         required: false
+      oceanbase:
+        condition: service_healthy
+        required: false
+      seekdb:
+        condition: service_healthy
+        required: false
       redis:
         condition: service_started
     healthcheck:
@@ -235,6 +380,9 @@ services:
       retries: 3
       start_period: 60s
       disable: ${COMPOSE_WORKER_HEALTHCHECK_DISABLED:-true}
+    networks:
+      - ssrf_proxy_network
+      - default
 
   # Frontend web application.
   web:
@@ -349,7 +497,9 @@ services:
     environment:
       REDISCLI_AUTH: ${REDIS_PASSWORD:-difyai123456}
     volumes:
+      # Mount the redis data directory to the container.
       - ./volumes/redis/data:/data
+    # Set the redis password when startup redis server.
     command: redis-server --requirepass ${REDIS_PASSWORD:-difyai123456}
     healthcheck:
       test:
@@ -369,6 +519,9 @@ services:
         required: false
       - ./.env
     environment:
+      # The DifySandbox configurations
+      # Make sure you are changing this key for your deployment with a strong key.
+      # You can generate a strong key using `openssl rand -base64 42`.
       API_KEY: ${SANDBOX_API_KEY:-dify-sandbox}
       GIN_MODE: ${SANDBOX_GIN_MODE:-release}
       WORKER_TIMEOUT: ${SANDBOX_WORKER_TIMEOUT:-15}
@@ -386,6 +539,14 @@ services:
       - ssrf_proxy_network
 
   # Local sandbox for Dify Agent shell workspaces.
+  # Network isolation: local_sandbox has NO direct route to `api`. Its only
+  # networks are `agent_sandbox_network` (so agent_backend can reach it on 5004
+  # for shellctl, and it can reach agent_backend directly) and
+  # `local_sandbox_proxy_network` (so its egress is forced through
+  # agent_ssrf_proxy).
+  # All non-agent_backend/localhost traffic goes through the Squid forward proxy
+  # on port 3128, which only allows agent_backend /agent-stub/ and the Dify API
+  # /files/* endpoints (see ssrf_proxy/squid-agent.conf.template).
   local_sandbox:
     image: langgenius/dify-agent-local-sandbox:1.16.1
     restart: always
@@ -487,6 +648,12 @@ services:
       db_mysql:
         condition: service_healthy
         required: false
+      oceanbase:
+        condition: service_healthy
+        required: false
+      seekdb:
+        condition: service_healthy
+        required: false
 
   # Dify Agent backend service.
   agent_backend:
@@ -517,6 +684,9 @@ services:
       DIFY_AGENT_E2B_SHELLCTL_PORT: ${DIFY_AGENT_E2B_SHELLCTL_PORT:-5004}
       DIFY_AGENT_SANDBOX_FILE_UPLOAD_MAX_BYTES: ${PLUGIN_MAX_FILE_SIZE:-52428800}
       DIFY_AGENT_STUB_API_BASE_URL: ${DIFY_AGENT_STUB_API_BASE_URL:-http://agent_backend:5050/agent-stub}
+      # This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens.
+      # Replace this development default in production.
+      # Generate one with: python -c 'import secrets; print(secrets.token_urlsafe(32))'
       DIFY_AGENT_SERVER_SECRET_KEY: ${DIFY_AGENT_SERVER_SECRET_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY}
       DIFY_AGENT_API_TOKEN: ${DIFY_AGENT_API_TOKEN:-dify-agent-run-token-for-dev-only}
       DIFY_AGENT_SHUTDOWN_GRACE_SECONDS: ${DIFY_AGENT_SHUTDOWN_GRACE_SECONDS:-30}
@@ -528,6 +698,8 @@ services:
         condition: service_started
     networks:
       - default
+      # Shared internal network with local_sandbox so agent_backend can reach it
+      # on port 5004 (shellctl entrypoint) while local_sandbox stays off `default`.
       - agent_sandbox_network
 
   # Dedicated SSRF proxy for the dify-agent local_sandbox.
@@ -548,10 +720,15 @@ services:
       HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
       COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid}
     networks:
+      # Needs to reach api and agent_backend as forward-proxy destinations.
       - default
+      # Only agent_ssrf_proxy and local_sandbox share this internal network, so
+      # the local_sandbox can reach Squid without gaining a direct route to `api`.
       - local_sandbox_proxy_network
 
   # ssrf_proxy server
+  # for more information, please refer to
+  # https://docs.dify.ai/learn-more/faq/install-faq#18-why-is-ssrf-proxy-needed%3F
   ssrf_proxy:
     image: ubuntu/squid:latest
     restart: always
@@ -566,6 +743,7 @@ services:
         "cp /docker-entrypoint-mount.sh /docker-entrypoint.sh && sed -i 's/\r$$//' /docker-entrypoint.sh && chmod +x /docker-entrypoint.sh && /docker-entrypoint.sh",
       ]
     environment:
+      # pls clearly modify the squid env vars to fit your network environment.
       HTTP_PORT: ${SSRF_HTTP_PORT:-3128}
       COREDUMP_DIR: ${SSRF_COREDUMP_DIR:-/var/spool/squid}
       SSRF_PROXY_ALLOW_PRIVATE_IPS: ${SSRF_PROXY_ALLOW_PRIVATE_IPS:-}
@@ -575,6 +753,7 @@ services:
       - default
 
   # Certbot service
+  # use `docker-compose --profile certbot up` to start the certbot service.
   certbot:
     image: certbot/certbot
     profiles:
@@ -594,6 +773,7 @@ services:
     command: ["tail", "-f", "/dev/null"]
 
   # The nginx reverse proxy.
+  # used for reverse proxying the API service and Web service.
   nginx:
     image: nginx:latest
     restart: always
@@ -603,8 +783,8 @@ services:
       - ./nginx/https.conf.template:/etc/nginx/https.conf.template
       - ./nginx/conf.d:/etc/nginx/conf.d
       - ./nginx/docker-entrypoint.sh:/docker-entrypoint-mount.sh
-      - ./nginx/ssl:/etc/ssl
-      - ./volumes/certbot/conf/live:/etc/letsencrypt/live
+      - ./nginx/ssl:/etc/ssl # cert dir (legacy)
+      - ./volumes/certbot/conf/live:/etc/letsencrypt/live # cert dir (with certbot container)
       - ./volumes/certbot/conf:/etc/letsencrypt
       - ./volumes/certbot/www:/var/www/html
     entrypoint:
@@ -618,6 +798,8 @@ services:
       NGINX_HTTPS_ENABLED: ${NGINX_HTTPS_ENABLED:-false}
       NGINX_SSL_PORT: ${NGINX_SSL_PORT:-443}
       NGINX_PORT: ${NGINX_PORT:-80}
+      # You're required to add your own SSL certificates/keys to the `./nginx/ssl` directory
+      # and modify the env vars below in .env if HTTPS_ENABLED is true.
       NGINX_SSL_CERT_FILENAME: ${NGINX_SSL_CERT_FILENAME:-dify.crt}
       NGINX_SSL_CERT_KEY_FILENAME: ${NGINX_SSL_CERT_KEY_FILENAME:-dify.key}
       NGINX_SSL_PROTOCOLS: ${NGINX_SSL_PROTOCOLS:-TLSv1.2 TLSv1.3}
@@ -632,9 +814,9 @@ services:
     depends_on:
       - api
       - web
-    #ports:
-    #  - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}"
-    #  - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}"
+    ports:
+      - "${EXPOSE_NGINX_PORT:-80}:${NGINX_PORT:-80}"
+      - "${EXPOSE_NGINX_SSL_PORT:-443}:${NGINX_SSL_PORT:-443}"
 
   # The Weaviate vector store.
   weaviate:
@@ -643,8 +825,11 @@ services:
       - weaviate
     restart: always
     volumes:
+      # Mount the Weaviate data directory to the con tainer.
       - ./volumes/weaviate:/var/lib/weaviate
     environment:
+      # The Weaviate configurations
+      # You can refer to the [Weaviate](https://weaviate.io/developers/weaviate/config-refs/env-vars) documentation for more information.
       PERSISTENCE_DATA_PATH: ${WEAVIATE_PERSISTENCE_DATA_PATH:-/var/lib/weaviate}
       QUERY_DEFAULTS_LIMIT: ${WEAVIATE_QUERY_DEFAULTS_LIMIT:-25}
       AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: ${WEAVIATE_AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:-false}
@@ -660,13 +845,487 @@ services:
       ENABLE_TOKENIZER_KAGOME_JA: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_JA:-false}
       ENABLE_TOKENIZER_KAGOME_KR: ${WEAVIATE_ENABLE_TOKENIZER_KAGOME_KR:-false}
 
+  # OceanBase vector database
+  oceanbase:
+    image: oceanbase/oceanbase-ce:4.3.5-lts
+    container_name: oceanbase
+    profiles:
+      - oceanbase
+    restart: always
+    volumes:
+      - ./volumes/oceanbase/data:/root/ob
+      - ./volumes/oceanbase/conf:/root/.obd/cluster
+      - ./volumes/oceanbase/init.d:/root/boot/init.d
+    environment:
+      OB_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G}
+      OB_SYS_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456}
+      OB_TENANT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456}
+      OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai}
+      OB_SERVER_IP: 127.0.0.1
+      MODE: mini
+      LANG: C.UTF-8
+      LC_ALL: C.UTF-8
+    ports:
+      - "${OCEANBASE_VECTOR_PORT:-2881}:2881"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          'obclient -h127.0.0.1 -P2881 -uroot@test -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
+        ]
+      interval: 10s
+      retries: 30
+      start_period: 30s
+      timeout: 10s
+
+  # seekdb vector database
+  seekdb:
+    image: oceanbase/seekdb:latest
+    container_name: seekdb
+    profiles:
+      - seekdb
+    restart: always
+    volumes:
+      - ./volumes/seekdb:/var/lib/oceanbase
+    environment:
+      ROOT_PASSWORD: ${OCEANBASE_VECTOR_PASSWORD:-difyai123456}
+      MEMORY_LIMIT: ${SEEKDB_MEMORY_LIMIT:-2G}
+      REPORTER: dify-ai-seekdb
+    ports:
+      - "${OCEANBASE_VECTOR_PORT:-2881}:2881"
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          'mysql -h127.0.0.1 -P2881 -uroot -p${OCEANBASE_VECTOR_PASSWORD:-difyai123456} -e "SELECT 1;"',
+        ]
+      interval: 5s
+      retries: 60
+      timeout: 5s
+
+  # Qdrant vector store.
+  # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)
+  qdrant:
+    image: langgenius/qdrant:v1.8.3
+    profiles:
+      - qdrant
+    restart: always
+    volumes:
+      - ./volumes/qdrant:/qdrant/storage
+    environment:
+      QDRANT_API_KEY: ${QDRANT_API_KEY:-difyai123456}
+
+  # The Couchbase vector store.
+  couchbase-server:
+    build: ./couchbase-server
+    profiles:
+      - couchbase
+    restart: always
+    environment:
+      - CLUSTER_NAME=dify_search
+      - COUCHBASE_ADMINISTRATOR_USERNAME=${COUCHBASE_USER:-Administrator}
+      - COUCHBASE_ADMINISTRATOR_PASSWORD=${COUCHBASE_PASSWORD:-password}
+      - COUCHBASE_BUCKET=${COUCHBASE_BUCKET_NAME:-Embeddings}
+      - COUCHBASE_BUCKET_RAMSIZE=512
+      - COUCHBASE_RAM_SIZE=2048
+      - COUCHBASE_EVENTING_RAM_SIZE=512
+      - COUCHBASE_INDEX_RAM_SIZE=512
+      - COUCHBASE_FTS_RAM_SIZE=1024
+    hostname: couchbase-server
+    container_name: couchbase-server
+    working_dir: /opt/couchbase
+    stdin_open: true
+    tty: true
+    entrypoint: [""]
+    command: sh -c "/opt/couchbase/init/init-cbserver.sh"
+    volumes:
+      - ./volumes/couchbase/data:/opt/couchbase/var/lib/couchbase/data
+    healthcheck:
+      # ensure bucket was created before proceeding
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -f -u Administrator:password http://localhost:8091/pools/default/buckets | grep -q '\\[{' || exit 1",
+        ]
+      interval: 10s
+      retries: 10
+      start_period: 30s
+      timeout: 10s
+
+  # The pgvector vector database.
+  pgvector:
+    image: pgvector/pgvector:pg16
+    profiles:
+      - pgvector
+    restart: always
+    environment:
+      PGUSER: ${PGVECTOR_PGUSER:-postgres}
+      # The password for the default postgres user.
+      POSTGRES_PASSWORD: ${PGVECTOR_POSTGRES_PASSWORD:-difyai123456}
+      # The name of the default postgres database.
+      POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify}
+      # postgres data directory
+      PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata}
+      # pg_bigm module for full text search
+      PG_BIGM: ${PGVECTOR_PG_BIGM:-false}
+      PG_BIGM_VERSION: ${PGVECTOR_PG_BIGM_VERSION:-1.2-20240606}
+    volumes:
+      - ./volumes/pgvector/data:/var/lib/postgresql/data
+      - ./pgvector/docker-entrypoint.sh:/docker-entrypoint.sh
+    entrypoint: ["/docker-entrypoint.sh"]
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  # get image from https://www.vastdata.com.cn/
+  vastbase:
+    image: vastdata/vastbase-vector
+    profiles:
+      - vastbase
+    restart: always
+    environment:
+      - VB_DBCOMPATIBILITY=PG
+      - VB_DB=dify
+      - VB_USERNAME=dify
+      - VB_PASSWORD=Difyai123456
+    ports:
+      - "5434:5432"
+    volumes:
+      - ./vastbase/lic:/home/vastbase/vastbase/lic
+      - ./vastbase/data:/home/vastbase/data
+      - ./vastbase/backup:/home/vastbase/backup
+      - ./vastbase/backup_log:/home/vastbase/backup_log
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  # pgvecto-rs vector store
+  pgvecto-rs:
+    image: tensorchord/pgvecto-rs:pg16-v0.3.0
+    profiles:
+      - pgvecto-rs
+    restart: always
+    environment:
+      PGUSER: ${PGVECTOR_PGUSER:-postgres}
+      # The password for the default postgres user.
+      POSTGRES_PASSWORD: ${PGVECTOR_POSTGRES_PASSWORD:-difyai123456}
+      # The name of the default postgres database.
+      POSTGRES_DB: ${PGVECTOR_POSTGRES_DB:-dify}
+      # postgres data directory
+      PGDATA: ${PGVECTOR_PGDATA:-/var/lib/postgresql/data/pgdata}
+    volumes:
+      - ./volumes/pgvecto_rs/data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  # Chroma vector database
+  chroma:
+    image: ghcr.io/chroma-core/chroma:0.5.20
+    profiles:
+      - chroma
+    restart: always
+    volumes:
+      - ./volumes/chroma:/chroma/chroma
+    environment:
+      CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456}
+      CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider}
+      IS_PERSISTENT: ${CHROMA_IS_PERSISTENT:-TRUE}
+
+  # InterSystems IRIS vector database
+  iris:
+    image: containers.intersystems.com/intersystems/iris-community:2025.3
+    profiles:
+      - iris
+    container_name: iris
+    restart: always
+    init: true
+    ports:
+      - "${IRIS_SUPER_SERVER_PORT:-1972}:1972"
+      - "${IRIS_WEB_SERVER_PORT:-52773}:52773"
+    volumes:
+      - ./volumes/iris:/durable
+      - ./iris/iris-init.script:/iris-init.script
+      - ./iris/docker-entrypoint.sh:/custom-entrypoint.sh
+    entrypoint: ["/custom-entrypoint.sh"]
+    tty: true
+    environment:
+      TZ: ${IRIS_TIMEZONE:-UTC}
+      ISC_DATA_DIRECTORY: /durable/iris
+
+  # Oracle vector database
+  oracle:
+    image: container-registry.oracle.com/database/free:latest
+    profiles:
+      - oracle
+    restart: always
+    volumes:
+      - source: oradata
+        type: volume
+        target: /opt/oracle/oradata
+      - ./startupscripts:/opt/oracle/scripts/startup
+    environment:
+      ORACLE_PWD: ${ORACLE_PWD:-Dify123456}
+      ORACLE_CHARACTERSET: ${ORACLE_CHARACTERSET:-AL32UTF8}
+
+  # Milvus vector database services
+  etcd:
+    container_name: milvus-etcd
+    image: quay.io/coreos/etcd:v3.5.5
+    profiles:
+      - milvus
+    environment:
+      ETCD_AUTO_COMPACTION_MODE: ${ETCD_AUTO_COMPACTION_MODE:-revision}
+      ETCD_AUTO_COMPACTION_RETENTION: ${ETCD_AUTO_COMPACTION_RETENTION:-1000}
+      ETCD_QUOTA_BACKEND_BYTES: ${ETCD_QUOTA_BACKEND_BYTES:-4294967296}
+      ETCD_SNAPSHOT_COUNT: ${ETCD_SNAPSHOT_COUNT:-50000}
+    volumes:
+      - ./volumes/milvus/etcd:/etcd
+    command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - milvus
+
+  minio:
+    container_name: milvus-minio
+    image: minio/minio:RELEASE.2023-03-20T20-16-18Z
+    profiles:
+      - milvus
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin}
+    volumes:
+      - ./volumes/milvus/minio:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    networks:
+      - milvus
+
+  milvus-standalone:
+    container_name: milvus-standalone
+    image: milvusdb/milvus:v2.6.3
+    profiles:
+      - milvus
+    command: ["milvus", "run", "standalone"]
+    environment:
+      ETCD_ENDPOINTS: ${ETCD_ENDPOINTS:-etcd:2379}
+      MINIO_ADDRESS: ${MINIO_ADDRESS:-minio:9000}
+      common.security.authorizationEnabled: ${MILVUS_AUTHORIZATION_ENABLED:-true}
+    volumes:
+      - ./volumes/milvus/milvus:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    depends_on:
+      - etcd
+      - minio
+    ports:
+      - 19530:19530
+      - 9091:9091
+    networks:
+      - milvus
+
+  # Opensearch vector database
+  opensearch:
+    container_name: opensearch
+    image: opensearchproject/opensearch:latest
+    profiles:
+      - opensearch
+    environment:
+      discovery.type: ${OPENSEARCH_DISCOVERY_TYPE:-single-node}
+      bootstrap.memory_lock: ${OPENSEARCH_BOOTSTRAP_MEMORY_LOCK:-true}
+      OPENSEARCH_JAVA_OPTS: -Xms${OPENSEARCH_JAVA_OPTS_MIN:-512m} -Xmx${OPENSEARCH_JAVA_OPTS_MAX:-1024m}
+      OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD:-Qazwsxedc!@#123}
+    ulimits:
+      memlock:
+        soft: ${OPENSEARCH_MEMLOCK_SOFT:--1}
+        hard: ${OPENSEARCH_MEMLOCK_HARD:--1}
+      nofile:
+        soft: ${OPENSEARCH_NOFILE_SOFT:-65536}
+        hard: ${OPENSEARCH_NOFILE_HARD:-65536}
+    volumes:
+      - ./volumes/opensearch/data:/usr/share/opensearch/data
+    networks:
+      - opensearch-net
+
+  opensearch-dashboards:
+    container_name: opensearch-dashboards
+    image: opensearchproject/opensearch-dashboards:latest
+    profiles:
+      - opensearch
+    environment:
+      OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
+    volumes:
+      - ./volumes/opensearch/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
+    networks:
+      - opensearch-net
+    depends_on:
+      - opensearch
+
+  # opengauss vector database.
+  opengauss:
+    image: opengauss/opengauss:7.0.0-RC1
+    profiles:
+      - opengauss
+    privileged: true
+    restart: always
+    environment:
+      GS_USERNAME: ${OPENGAUSS_USER:-postgres}
+      GS_PASSWORD: ${OPENGAUSS_PASSWORD:-Dify@123}
+      GS_PORT: ${OPENGAUSS_PORT:-6600}
+      GS_DB: ${OPENGAUSS_DATABASE:-dify}
+    volumes:
+      - ./volumes/opengauss/data:/var/lib/opengauss/data
+    healthcheck:
+      test: ["CMD-SHELL", "netstat -lntp | grep tcp6 > /dev/null 2>&1"]
+      interval: 10s
+      timeout: 10s
+      retries: 10
+    ports:
+      - ${OPENGAUSS_PORT:-6600}:${OPENGAUSS_PORT:-6600}
+
+  # MyScale vector database
+  myscale:
+    container_name: myscale
+    image: myscale/myscaledb:1.6.4
+    profiles:
+      - myscale
+    restart: always
+    tty: true
+    volumes:
+      - ./volumes/myscale/data:/var/lib/clickhouse
+      - ./volumes/myscale/log:/var/log/clickhouse-server
+      - ./volumes/myscale/config/users.d/custom_users_config.xml:/etc/clickhouse-server/users.d/custom_users_config.xml
+    ports:
+      - ${MYSCALE_PORT:-8123}:${MYSCALE_PORT:-8123}
+
+  # Matrixone vector store.
+  matrixone:
+    hostname: matrixone
+    image: matrixorigin/matrixone:2.1.1
+    profiles:
+      - matrixone
+    restart: always
+    volumes:
+      - ./volumes/matrixone/data:/mo-data
+    ports:
+      - ${MATRIXONE_PORT:-6001}:${MATRIXONE_PORT:-6001}
+
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html
+  # https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-prod-prerequisites
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.14.3
+    container_name: elasticsearch
+    profiles:
+      - elasticsearch
+      - elasticsearch-ja
+    restart: always
+    volumes:
+      - ./elasticsearch/docker-entrypoint.sh:/docker-entrypoint-mount.sh
+      - dify_es01_data:/usr/share/elasticsearch/data
+    environment:
+      ELASTIC_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic}
+      VECTOR_STORE: ${VECTOR_STORE:-}
+      cluster.name: dify-es-cluster
+      node.name: dify-es0
+      discovery.type: single-node
+      xpack.license.self_generated.type: basic
+      xpack.security.enabled: "true"
+      xpack.security.enrollment.enabled: "false"
+      xpack.security.http.ssl.enabled: "false"
+    ports:
+      - ${ELASTICSEARCH_PORT:-9200}:9200
+    deploy:
+      resources:
+        limits:
+          memory: 2g
+    entrypoint: ["sh", "-c", "sh /docker-entrypoint-mount.sh"]
+    healthcheck:
+      test:
+        ["CMD", "curl", "-s", "http://localhost:9200/_cluster/health?pretty"]
+      interval: 30s
+      timeout: 10s
+      retries: 50
+
+  # https://www.elastic.co/guide/en/kibana/current/docker.html
+  # https://www.elastic.co/guide/en/kibana/current/settings.html
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.14.3
+    container_name: kibana
+    profiles:
+      - elasticsearch
+    depends_on:
+      - elasticsearch
+    restart: always
+    environment:
+      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: d1a66dfd-c4d3-4a0a-8290-2abcb83ab3aa
+      NO_PROXY: localhost,127.0.0.1,elasticsearch,kibana
+      XPACK_SECURITY_ENABLED: "true"
+      XPACK_SECURITY_ENROLLMENT_ENABLED: "false"
+      XPACK_SECURITY_HTTP_SSL_ENABLED: "false"
+      XPACK_FLEET_ISAIRGAPPED: "true"
+      I18N_LOCALE: zh-CN
+      SERVER_PORT: "5601"
+      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
+    ports:
+      - ${KIBANA_PORT:-5601}:5601
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://localhost:5601 >/dev/null || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # unstructured .
+  # (if used, you need to set ETL_TYPE to Unstructured in the api & worker service.)
+  unstructured:
+    image: downloads.unstructured.io/unstructured-io/unstructured-api:latest
+    profiles:
+      - unstructured
+    restart: always
+    volumes:
+      - ./volumes/unstructured:/app/data
+
 networks:
+  # create a network between sandbox, api and ssrf_proxy, and can not access outside.
   ssrf_proxy_network:
     driver: bridge
     internal: true
+  # Internal network shared only by agent_ssrf_proxy and local_sandbox.
   local_sandbox_proxy_network:
     driver: bridge
     internal: true
+  # shellctl control channel (agent_backend -> local_sandbox:5004).
+  # sandbox can access agent backend through this network, this is
+  # a known limitation.
+  #
+  # The agent runtime respects HTTP(S)_PROXY, but arbitrary code execution
+  # is still possible through the shellctl channel.
   agent_sandbox_network:
     driver: bridge
-    internal: true
+    internal: true
+  milvus:
+    driver: bridge
+  opensearch-net:
+    driver: bridge
+    internal: true
+
+volumes:
+  oradata:
+  dify_es01_data: