dify/.github/workflows/cli-e2e.yml

name: CLI E2E Tests

on:
  workflow_dispatch:
    inputs:
      cli_ref:
        description: "Git ref (default: current branch)"
        type: string
        required: false

      # ── Suite toggles ──────────────────────────────────────────────────────
      suite_discovery:
        description: "Run discovery suite (get app / describe app)"
        type: boolean
        default: true
      suite_run:
        description: "Run run suite (basic / streaming / conversation / file / hitl / cache)"
        type: boolean
        default: true
      suite_auth:
        description: "Run auth suite (login / logout / status / whoami / devices / use)"
        type: boolean
        default: false
      suite_config:
        description: "Run config suite"
        type: boolean
        default: false

      # ── Case filter ────────────────────────────────────────────────────────
      test_scope:
        description: "smoke = [P0] only / full = all cases"
        type: choice
        required: false
        default: full
        options:
          - smoke
          - full

permissions:
  contents: read

jobs:
  e2e:
    name: E2E — difyctl
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
        with:
          ref: ${{ inputs.cli_ref || github.ref }}
          persist-credentials: false

      - name: Setup web environment
        uses: ./.github/actions/setup-web

      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest

      - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4
        with:
          package_json_field: packageManager
          run_install: false

      - name: Install CLI dependencies
        working-directory: cli
        run: pnpm install --frozen-lockfile

      - name: Generate command tree
        working-directory: cli
        run: pnpm tree:gen

      - name: Run E2E tests
        working-directory: cli
        env:
          DIFY_E2E_HOST:            ${{ secrets.DIFY_E2E_HOST }}
          DIFY_E2E_TOKEN:           ${{ secrets.DIFY_E2E_TOKEN }}
          DIFY_E2E_WORKSPACE_ID:    ${{ secrets.DIFY_E2E_WORKSPACE_ID }}
          DIFY_E2E_WORKSPACE_NAME:  ${{ secrets.DIFY_E2E_WORKSPACE_NAME }}
          DIFY_E2E_CHAT_APP_ID:     ${{ secrets.DIFY_E2E_CHAT_APP_ID }}
          DIFY_E2E_WORKFLOW_APP_ID: ${{ secrets.DIFY_E2E_WORKFLOW_APP_ID }}
          DIFY_E2E_EMAIL:           ${{ secrets.DIFY_E2E_EMAIL }}
          DIFY_E2E_PASSWORD:        ${{ secrets.DIFY_E2E_PASSWORD }}
          DIFY_E2E_HITL_APP_ID:     ${{ secrets.DIFY_E2E_HITL_APP_ID }}
          DIFY_E2E_FILE_APP_ID:     ${{ secrets.DIFY_E2E_FILE_APP_ID }}
          VITEST_RETRY:             "2"
        run: |
          # ── Collect test files with find (avoids unquoted-glob issues) ──────
          FILES=()
          if [ "${{ inputs.suite_discovery }}" = "true" ]; then
            while IFS= read -r f; do FILES+=("$f"); done \
              < <(find test/e2e/suites/discovery -name "*.e2e.ts" | sort)
          fi
          if [ "${{ inputs.suite_run }}" = "true" ]; then
            while IFS= read -r f; do FILES+=("$f"); done \
              < <(find test/e2e/suites/run -name "*.e2e.ts" | sort)
          fi
          if [ "${{ inputs.suite_auth }}" = "true" ]; then
            while IFS= read -r f; do FILES+=("$f"); done \
              < <(find test/e2e/suites/auth -name "*.e2e.ts" | sort)
          fi
          if [ "${{ inputs.suite_config }}" = "true" ]; then
            while IFS= read -r f; do FILES+=("$f"); done \
              < <(find test/e2e/suites/config -name "*.e2e.ts" | sort)
          fi

          if [ ${#FILES[@]} -eq 0 ]; then
            echo "No suites selected — nothing to run."
            exit 0
          fi

          echo "Running ${#FILES[@]} test files:"
          printf '  %s\n' "${FILES[@]}"
          echo "Scope: ${{ inputs.test_scope }}"

          # ── Apply P0-only filter if smoke scope ─────────────────────────────
          if [ "${{ inputs.test_scope }}" = "smoke" ]; then
            pnpm vp test --config vitest.e2e.config.ts \
              --testNamePattern="\[P0\]" \
              "${FILES[@]}"
          else
            pnpm vp test --config vitest.e2e.config.ts \
              "${FILES[@]}"
          fi

      - name: Upload test results on failure
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: e2e-results-${{ github.run_id }}
          path: cli/test-results/
          retention-days: 3