feat: update login type info

* main: (77 commits)
  feat: add voyage ai as a new model provider (#8747)
  docs: add english versions for the files customizable_model_scale_out and predefined_model_scale_out (#8871)
  fix: #8843 event: tts_message_end always return in api streaming resp… (#8846)
  Add Jamba and Llama3.2 model support (#8878)
  fix(workflow): update tagging logic in GitHub Actions (#8882)
  chore: bump ruff to 0.6.8 for fixing violation in SIM910 (#8869)
  refactor: update Callback to an abstract class (#8868)
  feat: deprecate gte-Qwen2-7B-instruct embedding model (#8866)
  feat: add internlm2.5-20b and qwen2.5-coder-7b model (#8862)
  fix: customize model credentials were invalid despite the provider credentials being active (#8864)
  fix: update qwen2.5-coder-7b model name (#8861)
  fix(workflow/nodes/knowledge-retrieval/use-config): Preserve rerankin… (#8842)
  chore: fix wrong VectorType match case (#8857)
  feat: add min-connection and max-connection for pgvector (#8841)
  feat(Tools): add feishu tools (#8800)
  fix: delete harm catalog settings for gemini (#8829)
  Add Llama3.2 models in Groq provider (#8831)
  feat: deprecate mistral model for siliconflow (#8828)
  fix: AnalyticdbVector retrieval scores (#8803)
  fix: close log status option raise error (#8826)
  ...

.github/workflows/api-tests.yml

@@ -39,7 +39,7 @@ jobs:
             api/pyproject.toml
             api/poetry.lock
 
-      - name: Check Poetry lockfile
+      - name: Poetry check
         run: |
           poetry check -C api --lock
           poetry show -C api
@@ -47,9 +47,6 @@ jobs:
       - name: Install dependencies
         run: poetry install -C api --with dev
 
-      - name: Check dependencies in pyproject.toml
-        run: poetry run -C api bash dev/pytest/pytest_artifacts.sh
-
       - name: Run Unit tests
         run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
 

.github/workflows/build-push.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - "main"
       - "deploy/dev"
-      - "fix/external-knowledge-retrieval-issues"
   release:
     types: [published]
 
@@ -126,7 +125,7 @@ jobs:
         with:
           images: ${{ env[matrix.image_name_env] }}
           tags: |
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-beta') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }}
             type=ref,event=branch
             type=sha,enable=true,priority=100,prefix=,suffix=,format=long
             type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}

README.md

@@ -17,7 +17,7 @@
             alt="chat on Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
+            alt="follow on Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -196,14 +196,10 @@ If you'd like to configure a highly-available setup, there are community-contrib
 
 #### Using Terraform for Deployment
 
-Deploy Dify to Cloud Platform with a single click using [terraform](https://www.terraform.io/)
-
 ##### Azure Global
+Deploy Dify to Azure with a single click using [terraform](https://www.terraform.io/).
 - [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
-
 ## Contributing
 
 For those who'd like to contribute code, see our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md). 
@@ -223,7 +219,7 @@ At the same time, please consider supporting Dify by sharing it on social media
 * [Github Discussion](https://github.com/langgenius/dify/discussions). Best for: sharing feedback and asking questions.
 * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
-* [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
+* [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
 
 ## Star history
 

README_AR.md

@@ -17,7 +17,7 @@
             alt="chat on Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
+            alt="follow on Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -179,13 +179,10 @@ docker compose up -d
 
 #### استخدام Terraform للتوزيع
 
-انشر Dify إلى منصة السحابة بنقرة واحدة باستخدام [terraform](https://www.terraform.io/)
-
 ##### Azure Global
+استخدم [terraform](https://www.terraform.io/) لنشر Dify على Azure بنقرة واحدة.
 - [Azure Terraform بواسطة @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform بواسطة @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 ## المساهمة
 

README_CN.md

@@ -17,7 +17,7 @@
             alt="chat on Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
+            alt="follow on Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -202,14 +202,10 @@ docker compose up -d
 
 #### 使用 Terraform 部署
 
-使用 [terraform](https://www.terraform.io/) 一键将 Dify 部署到云平台
-
 ##### Azure Global
+使用 [terraform](https://www.terraform.io/) 一键部署 Dify 到 Azure。
 - [Azure Terraform by @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform by @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=langgenius/dify&type=Date)](https://star-history.com/#langgenius/dify&Date)
@@ -236,7 +232,7 @@ docker compose up -d
 - [GitHub Issues](https://github.com/langgenius/dify/issues)。👉：使用 Dify.AI 时遇到的错误和问题，请参阅[贡献指南](CONTRIBUTING.md)。
 - [电子邮件支持](mailto:[email protected]?subject=[GitHub]Questions%20About%20Dify)。👉：关于使用 Dify.AI 的问题。
 - [Discord](https://discord.gg/FngNHpbcY7)。👉：分享您的应用程序并与社区交流。
-- [X(Twitter)](https://twitter.com/dify_ai)。👉：分享您的应用程序并与社区交流。
+- [Twitter](https://twitter.com/dify_ai)。👉：分享您的应用程序并与社区交流。
 - [商业许可](mailto:[email protected]?subject=[GitHub]Business%20License%20Inquiry)。👉：有关商业用途许可 Dify.AI 的商业咨询。
  - [微信]() 👉：扫描下方二维码，添加微信好友，备注 Dify，我们将邀请您加入 Dify 社区。  
 <img src="./images/wechat.png" alt="wechat" width="100"/>

README_ES.md

@@ -17,7 +17,7 @@
             alt="chat en Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="seguir en X(Twitter)"></a>
+            alt="seguir en Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Descargas de Docker" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -204,13 +204,10 @@ Si desea configurar una configuración de alta disponibilidad, la comunidad prop
 
 #### Uso de Terraform para el despliegue
 
-Despliega Dify en una plataforma en la nube con un solo clic utilizando [terraform](https://www.terraform.io/)
-
 ##### Azure Global
+Utiliza [terraform](https://www.terraform.io/) para desplegar Dify en Azure con un solo clic.
 - [Azure Terraform por @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform por @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 ## Contribuir
 
@@ -231,7 +228,7 @@ Al mismo tiempo, considera apoyar a Dify compartiéndolo en redes sociales y en
 * [Discusión en GitHub](https://github.com/langgenius/dify/discussions). Lo mejor para: compartir comentarios y hacer preguntas.
 * [Reporte de problemas en GitHub](https://github.com/langgenius/dify/issues). Lo mejor para: errores que encuentres usando Dify.AI y propuestas de características. Consulta nuestra [Guía de contribución](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 * [Discord](https://discord.gg/FngNHpbcY7). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad.
-* [X(Twitter)](https://twitter.com/dify_ai). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad.
+* [Twitter](https://twitter.com/dify_ai). Lo mejor para: compartir tus aplicaciones y pasar el rato con la comunidad.
 
 ## Historial de Estrellas
 

README_FR.md

@@ -17,7 +17,7 @@
             alt="chat sur Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="suivre sur X(Twitter)"></a>
+            alt="suivre sur Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Tirages Docker" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -202,13 +202,10 @@ Si vous souhaitez configurer une configuration haute disponibilité, la communau
 
 #### Utilisation de Terraform pour le déploiement
 
-Déployez Dify sur une plateforme cloud en un clic en utilisant [terraform](https://www.terraform.io/)
-
 ##### Azure Global
+Utilisez [terraform](https://www.terraform.io/) pour déployer Dify sur Azure en un clic.
 - [Azure Terraform par @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform par @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 ## Contribuer
 
@@ -229,7 +226,7 @@ Dans le même temps, veuillez envisager de soutenir Dify en le partageant sur le
 * [Discussion GitHub](https://github.com/langgenius/dify/discussions). Meilleur pour: partager des commentaires et poser des questions.
 * [Problèmes GitHub](https://github.com/langgenius/dify/issues). Meilleur pour: les bogues que vous rencontrez en utilisant Dify.AI et les propositions de fonctionnalités. Consultez notre [Guide de contribution](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 * [Discord](https://discord.gg/FngNHpbcY7). Meilleur pour: partager vos applications et passer du temps avec la communauté.
-* [X(Twitter)](https://twitter.com/dify_ai). Meilleur pour: partager vos applications et passer du temps avec la communauté.
+* [Twitter](https://twitter.com/dify_ai). Meilleur pour: partager vos applications et passer du temps avec la communauté.
 
 ## Historique des étoiles
 

README_JA.md

@@ -17,7 +17,7 @@
             alt="Discordでチャット"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="X(Twitter)でフォロー"></a>
+            alt="Twitterでフォロー"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -68,7 +68,7 @@ DifyはオープンソースのLLMアプリケーション開発プラットフ
   プロンプトの作成、モデルパフォーマンスの比較が行え、チャットベースのアプリに音声合成などの機能も追加できます。
 
 **4. RAGパイプライン**: 
-  ドキュメントの取り込みから検索までをカバーする広範なRAG機能ができます。ほかにもPDF、PPT、その他の一般的なドキュメントフォーマットからのテキスト抽出のサポートも提供します。
+  ドキュメントの取り込みから検索までをカバーする広範なRAG機能ができます。ほかにもPDF、PPT、その他の一般的なドキュメントフォーマットからのテキスト抽出のサーポイントも提供します。
 
 **5. エージェント機能**: 
   LLM Function CallingやReActに基づくエージェントの定義が可能で、AIエージェント用のプリビルトまたはカスタムツールを追加できます。Difyには、Google検索、DALL·E、Stable Diffusion、WolframAlphaなどのAIエージェント用の50以上の組み込みツールが提供します。
@@ -201,13 +201,10 @@ docker compose up -d
 
 #### Terraformを使用したデプロイ
 
-[terraform](https://www.terraform.io/) を使用して、ワンクリックでDifyをクラウドプラットフォームにデプロイします
-
 ##### Azure Global
-- [@nikawangによるAzure Terraform](https://github.com/nikawang/dify-azure-terraform)
+[terraform](https://www.terraform.io/) を使用して、AzureにDifyをワンクリックでデプロイします。
+- [nikawangのAzure Terraform](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [@sotazumによるGoogle Cloud Terraform](https://github.com/DeNA/dify-google-cloud-terraform)
 
 ## 貢献
 
@@ -228,7 +225,7 @@ docker compose up -d
 * [Github Discussion](https://github.com/langgenius/dify/discussions). 主に: フィードバックの共有や質問。
 * [GitHub Issues](https://github.com/langgenius/dify/issues). 主に: Dify.AIを使用する際に発生するエラーや問題については、[貢献ガイド](CONTRIBUTING_JA.md)を参照してください
 * [Discord](https://discord.gg/FngNHpbcY7). 主に: アプリケーションの共有やコミュニティとの交流。
-* [X(Twitter)](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。
+* [Twitter](https://twitter.com/dify_ai). 主に: アプリケーションの共有やコミュニティとの交流。
 
 
 

README_KL.md

@@ -17,7 +17,7 @@
             alt="chat on Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
+            alt="follow on Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -202,13 +202,10 @@ If you'd like to configure a highly-available setup, there are community-contrib
 
 #### Terraform atorlugu pilersitsineq
 
-wa'logh nIqHom neH ghun deployment toy'wI' [terraform](https://www.terraform.io/) lo'laH.
-
 ##### Azure Global
-- [Azure Terraform mung @nikawang](https://github.com/nikawang/dify-azure-terraform)
+Atoruk [terraform](https://www.terraform.io/) Dify-mik Azure-mut ataatsikkut ikkussuilluarlugu.
+- [Azure Terraform atorlugu @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform qachlot @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
 
 ## Contributing
 
@@ -231,7 +228,7 @@ At the same time, please consider supporting Dify by sharing it on social media
 ). Best for: sharing feedback and asking questions.
 * [GitHub Issues](https://github.com/langgenius/dify/issues). Best for: bugs you encounter using Dify.AI, and feature proposals. See our [Contribution Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md).
 * [Discord](https://discord.gg/FngNHpbcY7). Best for: sharing your applications and hanging out with the community.
-* [X(Twitter)](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
+* [Twitter](https://twitter.com/dify_ai). Best for: sharing your applications and hanging out with the community.
 
 ## Star History
 

README_KR.md

@@ -17,7 +17,7 @@
             alt="chat on Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="follow on X(Twitter)"></a>
+            alt="follow on Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -39,6 +39,7 @@
   <a href="./README_AR.md"><img alt="README بالعربية" src="https://img.shields.io/badge/العربية-d9d9d9"></a>
   <a href="./README_TR.md"><img alt="Türkçe README" src="https://img.shields.io/badge/Türkçe-d9d9d9"></a>
   <a href="./README_VI.md"><img alt="README Tiếng Việt" src="https://img.shields.io/badge/Ti%E1%BA%BFng%20Vi%E1%BB%87t-d9d9d9"></a>
+
 </p>
 
 
@@ -194,14 +195,10 @@ Dify를 Kubernetes에 배포하고 프리미엄 스케일링 설정을 구성했
 
 #### Terraform을 사용한 배포
 
-[terraform](https://www.terraform.io/)을 사용하여 단 한 번의 클릭으로 Dify를 클라우드 플랫폼에 배포하십시오
-
 ##### Azure Global
+[terraform](https://www.terraform.io/)을 사용하여 Azure에 Dify를 원클릭으로 배포하세요.
 - [nikawang의 Azure Terraform](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [sotazum의 Google Cloud Terraform](https://github.com/DeNA/dify-google-cloud-terraform)
-
 ## 기여
 
 코드에 기여하고 싶은 분들은 [기여 가이드](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md)를 참조하세요.

README_TR.md

@@ -17,7 +17,7 @@
             alt="Discord'da sohbet et"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="X(Twitter)'da takip et"></a>
+            alt="Twitter'da takip et"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Çekmeleri" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -200,13 +200,9 @@ Yüksek kullanılabilirliğe sahip bir kurulum yapılandırmak isterseniz, Dify'
 
 #### Dağıtım için Terraform Kullanımı
 
-Dify'ı bulut platformuna tek tıklamayla dağıtın [terraform](https://www.terraform.io/) kullanarak
-
 ##### Azure Global
-- [Azure Terraform tarafından @nikawang](https://github.com/nikawang/dify-azure-terraform)
-
-##### Google Cloud
-- [Google Cloud Terraform tarafından @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
+[Terraform](https://www.terraform.io/) kullanarak Dify'ı Azure'a tek tıklamayla dağıtın.
+- [@nikawang tarafından Azure Terraform](https://github.com/nikawang/dify-azure-terraform)
 
 ## Katkıda Bulunma
 
@@ -226,7 +222,7 @@ Aynı zamanda, lütfen Dify'ı sosyal medyada, etkinliklerde ve konferanslarda p
 * [Github Tartışmaları](https://github.com/langgenius/dify/discussions). En uygun: geri bildirim paylaşmak ve soru sormak için.
 * [GitHub Sorunları](https://github.com/langgenius/dify/issues). En uygun: Dify.AI kullanırken karşılaştığınız hatalar ve özellik önerileri için. [Katkı Kılavuzumuza](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) bakın.
 * [Discord](https://discord.gg/FngNHpbcY7). En uygun: uygulamalarınızı paylaşmak ve toplulukla vakit geçirmek için.
-* [X(Twitter)](https://twitter.com/dify_ai). En uygun: uygulamalarınızı paylaşmak ve toplulukla vakit geçirmek için.
+* [Twitter](https://twitter.com/dify_ai). En uygun: uygulamalarınızı paylaşmak ve toplulukla vakit geçirmek için.
 
 ## Star history
 

README_VI.md

@@ -17,7 +17,7 @@
             alt="chat trên Discord"></a>
     <a href="https://twitter.com/intent/follow?screen_name=dify_ai" target="_blank">
         <img src="https://img.shields.io/twitter/follow/dify_ai?logo=X&color=%20%23f5f5f5"
-            alt="theo dõi trên X(Twitter)"></a>
+            alt="theo dõi trên Twitter"></a>
     <a href="https://hub.docker.com/u/langgenius" target="_blank">
         <img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/langgenius/dify-web?labelColor=%20%23FDB062&color=%20%23f79009"></a>
     <a href="https://github.com/langgenius/dify/graphs/commit-activity" target="_blank">
@@ -196,14 +196,10 @@ Nếu bạn muốn cấu hình một cài đặt có độ sẵn sàng cao, có
 
 #### Sử dụng Terraform để Triển khai
 
-Triển khai Dify lên nền tảng đám mây với một cú nhấp chuột bằng cách sử dụng [terraform](https://www.terraform.io/)
-
 ##### Azure Global
+Triển khai Dify lên Azure chỉ với một cú nhấp chuột bằng cách sử dụng [terraform](https://www.terraform.io/).
 - [Azure Terraform bởi @nikawang](https://github.com/nikawang/dify-azure-terraform)
 
-##### Google Cloud
-- [Google Cloud Terraform bởi @sotazum](https://github.com/DeNA/dify-google-cloud-terraform)
-
 ## Đóng góp
 
 Đối với những người muốn đóng góp mã, xem [Hướng dẫn Đóng góp](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) của chúng tôi. 
@@ -223,7 +219,7 @@ Triển khai Dify lên nền tảng đám mây với một cú nhấp chuột b
 * [Thảo luận GitHub](https://github.com/langgenius/dify/discussions). Tốt nhất cho: chia sẻ phản hồi và đặt câu hỏi.
 * [Vấn đề GitHub](https://github.com/langgenius/dify/issues). Tốt nhất cho: lỗi bạn gặp phải khi sử dụng Dify.AI và đề xuất tính năng. Xem [Hướng dẫn Đóng góp](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) của chúng tôi.
 * [Discord](https://discord.gg/FngNHpbcY7). Tốt nhất cho: chia sẻ ứng dụng của bạn và giao lưu với cộng đồng.
-* [X(Twitter)](https://twitter.com/dify_ai). Tốt nhất cho: chia sẻ ứng dụng của bạn và giao lưu với cộng đồng.
+* [Twitter](https://twitter.com/dify_ai). Tốt nhất cho: chia sẻ ứng dụng của bạn và giao lưu với cộng đồng.
 
 ## Lịch sử Yêu thích
 

api/.env.example

@@ -39,7 +39,7 @@ DB_DATABASE=dify
 
 # Storage configuration
 # use for store upload files, private keys...
-# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos, baidu-obs
+# storage type: local, s3, azure-blob, google-storage, tencent-cos, huawei-obs, volcengine-tos
 STORAGE_TYPE=local
 STORAGE_LOCAL_PATH=storage
 S3_USE_AWS_MANAGED_IAM=false
@@ -79,12 +79,6 @@ HUAWEI_OBS_SECRET_KEY=your-secret-key
 HUAWEI_OBS_ACCESS_KEY=your-access-key
 HUAWEI_OBS_SERVER=your-server-url
 
-# Baidu OBS Storage Configuration
-BAIDU_OBS_BUCKET_NAME=your-bucket-name
-BAIDU_OBS_SECRET_KEY=your-secret-key
-BAIDU_OBS_ACCESS_KEY=your-access-key
-BAIDU_OBS_ENDPOINT=your-server-url
-
 # OCI Storage configuration
 OCI_ENDPOINT=your-endpoint
 OCI_BUCKET_NAME=your-bucket-name
@@ -271,9 +265,6 @@ HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
 HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
 HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
 
-# Respect X-* headers to redirect clients
-RESPECT_XFORWARD_HEADERS_ENABLED=false
-
 # Log file path
 LOG_FILE=
 
@@ -300,4 +291,4 @@ POSITION_TOOL_EXCLUDES=
 
 POSITION_PROVIDER_PINS=
 POSITION_PROVIDER_INCLUDES=
-POSITION_PROVIDER_EXCLUDES=
+POSITION_PROVIDER_EXCLUDES=

api/app.py

@@ -26,7 +26,7 @@ from commands import register_commands
 from configs import dify_config
 
 # DO NOT REMOVE BELOW
-from events import event_handlers  # noqa: F401
+from events import event_handlers
 from extensions import (
     ext_celery,
     ext_code_based_extension,
@@ -36,7 +36,6 @@ from extensions import (
     ext_login,
     ext_mail,
     ext_migrate,
-    ext_proxy_fix,
     ext_redis,
     ext_sentry,
     ext_storage,
@@ -46,7 +45,7 @@ from extensions.ext_login import login_manager
 from libs.passport import PassportService
 
 # TODO: Find a way to avoid importing models here
-from models import account, dataset, model, source, task, tool, tools, web  # noqa: F401
+from models import account, dataset, model, source, task, tool, tools, web
 from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE
@@ -157,7 +156,6 @@ def initialize_extensions(app):
     ext_mail.init_app(app)
     ext_hosting_provider.init_app(app)
     ext_sentry.init_app(app)
-    ext_proxy_fix.init_app(app)
 
 
 # Flask-Login configuration
@@ -183,10 +181,10 @@ def load_user_from_request(request_from_flask_login):
     decoded = PassportService().verify(auth_token)
     user_id = decoded.get("user_id")
 
-    logged_in_account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
-    if logged_in_account:
-        contexts.tenant_id.set(logged_in_account.current_tenant_id)
-    return logged_in_account
+    account = AccountService.load_logged_in_account(account_id=user_id, token=auth_token)
+    if account:
+        contexts.tenant_id.set(account.current_tenant_id)
+    return account
 
 
 @login_manager.unauthorized_handler

api/configs/feature/__init__.py

@@ -1,6 +1,15 @@
 from typing import Annotated, Optional
 
-from pydantic import AliasChoices, Field, HttpUrl, NegativeInt, NonNegativeInt, PositiveInt, computed_field
+from pydantic import (
+    AliasChoices,
+    Field,
+    HttpUrl,
+    NegativeInt,
+    NonNegativeInt,
+    PositiveFloat,
+    PositiveInt,
+    computed_field,
+)
 from pydantic_settings import BaseSettings
 
 from configs.feature.hosted_service import HostedServiceConfig
@@ -247,12 +256,6 @@ class HttpConfig(BaseSettings):
         default=None,
     )
 
-    RESPECT_XFORWARD_HEADERS_ENABLED: bool = Field(
-        description="Enable or disable the X-Forwarded-For Proxy Fix middleware from Werkzeug"
-        " to respect X-* headers to redirect clients",
-        default=False,
-    )
-
 
 class InnerAPIConfig(BaseSettings):
     """
@@ -468,6 +471,11 @@ class MailConfig(BaseSettings):
         default=False,
     )
 
+    EMAIL_SEND_IP_LIMIT_PER_MINUTE: PositiveInt = Field(
+        description="Maximum number of emails allowed to be sent from the same IP address in a minute",
+        default=50,
+    )
+
 
 class RagEtlConfig(BaseSettings):
     """
@@ -604,6 +612,33 @@ class PositionConfig(BaseSettings):
         return {item.strip() for item in self.POSITION_TOOL_EXCLUDES.split(",") if item.strip() != ""}
 
 
+class LoginConfig(BaseSettings):
+    ENABLE_EMAIL_CODE_LOGIN: bool = Field(
+        description="whether to enable email code login",
+        default=False,
+    )
+    ENABLE_EMAIL_PASSWORD_LOGIN: bool = Field(
+        description="whether to enable email password login",
+        default=True,
+    )
+    ENABLE_SOCIAL_OAUTH_LOGIN: bool = Field(
+        description="whether to enable github/google oauth login",
+        default=False,
+    )
+    EMAIL_CODE_LOGIN_TOKEN_EXPIRY_HOURS: PositiveFloat = Field(
+        description="expiry time in hours for email code login token",
+        default=1 / 12,
+    )
+    ALLOW_REGISTER: bool = Field(
+        description="whether to enable register",
+        default=True,
+    )
+    ALLOW_CREATE_WORKSPACE: bool = Field(
+        description="whether to enable create workspace",
+        default=False,
+    )
+
+
 class FeatureConfig(
     # place the configs in alphabet order
     AppExecutionConfig,
@@ -629,6 +664,7 @@ class FeatureConfig(
     WorkflowConfig,
     WorkspaceConfig,
     PositionConfig,
+    LoginConfig,
     # hosted services config
     HostedServiceConfig,
     CeleryBeatConfig,

api/configs/middleware/__init__.py

@@ -8,7 +8,6 @@ from configs.middleware.cache.redis_config import RedisConfig
 from configs.middleware.storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
 from configs.middleware.storage.amazon_s3_storage_config import S3StorageConfig
 from configs.middleware.storage.azure_blob_storage_config import AzureBlobStorageConfig
-from configs.middleware.storage.baidu_obs_storage_config import BaiduOBSStorageConfig
 from configs.middleware.storage.google_cloud_storage_config import GoogleCloudStorageConfig
 from configs.middleware.storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from configs.middleware.storage.oci_storage_config import OCIStorageConfig
@@ -191,22 +190,6 @@ class CeleryConfig(DatabaseConfig):
         return self.CELERY_BROKER_URL.startswith("rediss://") if self.CELERY_BROKER_URL else False
 
 
-class InternalTestConfig(BaseSettings):
-    """
-    Configuration settings for Internal Test
-    """
-
-    AWS_SECRET_ACCESS_KEY: Optional[str] = Field(
-        description="Internal test AWS secret access key",
-        default=None,
-    )
-
-    AWS_ACCESS_KEY_ID: Optional[str] = Field(
-        description="Internal test AWS access key ID",
-        default=None,
-    )
-
-
 class MiddlewareConfig(
     # place the configs in alphabet order
     CeleryConfig,
@@ -217,13 +200,12 @@ class MiddlewareConfig(
     StorageConfig,
     AliyunOSSStorageConfig,
     AzureBlobStorageConfig,
-    BaiduOBSStorageConfig,
     GoogleCloudStorageConfig,
-    HuaweiCloudOBSStorageConfig,
-    OCIStorageConfig,
-    S3StorageConfig,
     TencentCloudCOSStorageConfig,
+    HuaweiCloudOBSStorageConfig,
     VolcengineTOSStorageConfig,
+    S3StorageConfig,
+    OCIStorageConfig,
     # configs of vdb and vdb providers
     VectorStoreConfig,
     AnalyticdbConfig,
@@ -240,6 +222,5 @@ class MiddlewareConfig(
     TiDBVectorConfig,
     WeaviateConfig,
     ElasticsearchConfig,
-    InternalTestConfig,
 ):
     pass

api/configs/middleware/storage/baidu_obs_storage_config.py

@@ -1,29 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel, Field
-
-
-class BaiduOBSStorageConfig(BaseModel):
-    """
-    Configuration settings for Baidu Object Storage Service (OBS)
-    """
-
-    BAIDU_OBS_BUCKET_NAME: Optional[str] = Field(
-        description="Name of the Baidu OBS bucket to store and retrieve objects (e.g., 'my-obs-bucket')",
-        default=None,
-    )
-
-    BAIDU_OBS_ACCESS_KEY: Optional[str] = Field(
-        description="Access Key ID for authenticating with Baidu OBS",
-        default=None,
-    )
-
-    BAIDU_OBS_SECRET_KEY: Optional[str] = Field(
-        description="Secret Access Key for authenticating with Baidu OBS",
-        default=None,
-    )
-
-    BAIDU_OBS_ENDPOINT: Optional[str] = Field(
-        description="URL of the Baidu OSS endpoint for your chosen region (e.g., 'https://.bj.bcebos.com')",
-        default=None,
-    )

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.9.1-fix1",
+        default="0.8.3",
     )
 
     COMMIT_SHA: str = Field(

api/controllers/console/__init__.py

@@ -37,16 +37,7 @@ from .auth import activate, data_source_bearer_auth, data_source_oauth, forgot_p
 from .billing import billing
 
 # Import datasets controllers
-from .datasets import (
-    data_source,
-    datasets,
-    datasets_document,
-    datasets_segments,
-    external,
-    file,
-    hit_testing,
-    website,
-)
+from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing, website
 
 # Import explore controllers
 from .explore import (

api/controllers/console/app/conversation.py

@@ -188,7 +188,6 @@ class ChatConversationApi(Resource):
                         subquery.c.from_end_user_session_id.ilike(keyword_filter),
                     ),
                 )
-                .group_by(Conversation.id)
             )
 
         account = current_user

api/controllers/console/auth/activate.py

@@ -1,17 +1,15 @@
-import base64
 import datetime
-import secrets
 
+from flask import request
 from flask_restful import Resource, reqparse
 
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from libs.helper import StrLen, email, timezone
-from libs.password import hash_password, valid_password
-from models.account import AccountStatus
-from services.account_service import RegisterService
+from libs.helper import StrLen, email, get_remote_ip, timezone
+from models.account import AccountStatus, Tenant
+from services.account_service import AccountService, RegisterService
 
 
 class ActivateCheckApi(Resource):
@@ -27,8 +25,18 @@ class ActivateCheckApi(Resource):
         token = args["token"]
 
         invitation = RegisterService.get_invitation_if_token_valid(workspaceId, reg_email, token)
-
-        return {"is_valid": invitation is not None, "workspace_name": invitation["tenant"].name if invitation else None}
+        if invitation:
+            data = invitation.get("data", {})
+            tenant: Tenant = invitation.get("tenant", None)
+            workspace_name = tenant.name if tenant else None
+            workspace_id = tenant.id if tenant else None
+            invitee_email = data.get("email") if data else None
+            return {
+                "is_valid": invitation is not None,
+                "data": {"workspace_name": workspace_name, "workspace_id": workspace_id, "email": invitee_email},
+            }
+        else:
+            return {"is_valid": False}
 
 
 class ActivateApi(Resource):
@@ -38,7 +46,6 @@ class ActivateApi(Resource):
         parser.add_argument("email", type=email, required=False, nullable=True, location="json")
         parser.add_argument("token", type=str, required=True, nullable=False, location="json")
         parser.add_argument("name", type=StrLen(30), required=True, nullable=False, location="json")
-        parser.add_argument("password", type=valid_password, required=True, nullable=False, location="json")
         parser.add_argument(
             "interface_language", type=supported_language, required=True, nullable=False, location="json"
         )
@@ -54,15 +61,6 @@ class ActivateApi(Resource):
         account = invitation["account"]
         account.name = args["name"]
 
-        # generate password salt
-        salt = secrets.token_bytes(16)
-        base64_salt = base64.b64encode(salt).decode()
-
-        # encrypt password with salt
-        password_hashed = hash_password(args["password"], salt)
-        base64_password_hashed = base64.b64encode(password_hashed).decode()
-        account.password = base64_password_hashed
-        account.password_salt = base64_salt
         account.interface_language = args["interface_language"]
         account.timezone = args["timezone"]
         account.interface_theme = "light"
@@ -70,7 +68,9 @@ class ActivateApi(Resource):
         account.initialized_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
         db.session.commit()
 
-        return {"result": "success"}
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
+
+        return {"result": "success", "data": token}
 
 
 api.add_resource(ActivateCheckApi, "/activate/check")

api/controllers/console/auth/error.py

@@ -27,5 +27,29 @@ class InvalidTokenError(BaseHTTPException):
 
 class PasswordResetRateLimitExceededError(BaseHTTPException):
     error_code = "password_reset_rate_limit_exceeded"
-    description = "Password reset rate limit exceeded. Try again later."
+    description = "Too many password reset emails have been sent. Please try again in 5 minutes."
+    code = 429
+
+
+class EmailCodeError(BaseHTTPException):
+    error_code = "email_code_error"
+    description = "Email code is invalid or expired."
+    code = 400
+
+
+class EmailOrPasswordMismatchError(BaseHTTPException):
+    error_code = "email_or_password_mismatch"
+    description = "The email or password is mismatched."
+    code = 400
+
+
+class EmailPasswordLoginLimitError(BaseHTTPException):
+    error_code = "email_code_login_limit"
+    description = "The account was locked for 24 hours because the password was entered too many times."
+    code = 429
+
+
+class EmailCodeLoginRateLimitExceededError(BaseHTTPException):
+    error_code = "email_code_login_rate_limit_exceeded"
+    description = "Too many login emails have been sent. Please try again in 5 minutes."
     code = 429

api/controllers/console/auth/forgot_password.py

@@ -1,65 +1,80 @@
 import base64
-import logging
 import secrets
 
+from flask import request
 from flask_restful import Resource, reqparse
 
+from configs import dify_config
+from constants.languages import languages
 from controllers.console import api
 from controllers.console.auth.error import (
+    EmailCodeError,
     InvalidEmailError,
     InvalidTokenError,
     PasswordMismatchError,
-    PasswordResetRateLimitExceededError,
 )
+from controllers.console.error import EmailSendIpLimitError, NotAllowedCreateWorkspace, NotAllowedRegister
 from controllers.console.setup import setup_required
+from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
-from libs.helper import email as email_validate
+from libs.helper import email, get_remote_ip
 from libs.password import hash_password, valid_password
 from models.account import Account
-from services.account_service import AccountService
-from services.errors.account import RateLimitExceededError
+from services.account_service import AccountService, TenantService
 
 
 class ForgotPasswordSendEmailApi(Resource):
     @setup_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument("email", type=str, required=True, location="json")
+        parser.add_argument("email", type=email, required=True, location="json")
+        parser.add_argument("language", type=str, required=False, location="json")
         args = parser.parse_args()
 
-        email = args["email"]
+        ip_address = get_remote_ip(request)
+        if AccountService.is_email_send_ip_limit(ip_address):
+            raise EmailSendIpLimitError()
 
-        if not email_validate(email):
-            raise InvalidEmailError()
-
-        account = Account.query.filter_by(email=email).first()
-
-        if account:
-            try:
-                AccountService.send_reset_password_email(account=account)
-            except RateLimitExceededError:
-                logging.warning(f"Rate limit exceeded for email: {account.email}")
-                raise PasswordResetRateLimitExceededError()
+        if args["language"] is not None and args["language"] == "zh-Hans":
+            language = "zh-Hans"
         else:
-            # Return success to avoid revealing email registration status
-            logging.warning(f"Attempt to reset password for unregistered email: {email}")
+            language = "en-US"
 
-        return {"result": "success"}
+        account = Account.query.filter_by(email=args["email"]).first()
+        token = None
+        if account is None:
+            if dify_config.ALLOW_REGISTER:
+                token = AccountService.send_reset_password_email(email=args["email"], language=language)
+            else:
+                raise NotAllowedRegister()
+        else:
+            token = AccountService.send_reset_password_email(account=account, email=args["email"], language=language)
+
+        return {"result": "success", "data": token}
 
 
 class ForgotPasswordCheckApi(Resource):
     @setup_required
     def post(self):
         parser = reqparse.RequestParser()
+        parser.add_argument("email", type=str, required=True, location="json")
+        parser.add_argument("code", type=str, required=True, location="json")
         parser.add_argument("token", type=str, required=True, nullable=False, location="json")
         args = parser.parse_args()
-        token = args["token"]
 
-        reset_data = AccountService.get_reset_password_data(token)
+        user_email = args["email"]
 
-        if reset_data is None:
-            return {"is_valid": False, "email": None}
-        return {"is_valid": True, "email": reset_data.get("email")}
+        token_data = AccountService.get_reset_password_data(args["token"])
+        if token_data is None:
+            raise InvalidTokenError()
+
+        if user_email != token_data.get("email"):
+            raise InvalidEmailError()
+
+        if args["code"] != token_data.get("code"):
+            raise EmailCodeError()
+
+        return {"is_valid": True, "email": token_data.get("email")}
 
 
 class ForgotPasswordResetApi(Resource):
@@ -92,11 +107,30 @@ class ForgotPasswordResetApi(Resource):
         base64_password_hashed = base64.b64encode(password_hashed).decode()
 
         account = Account.query.filter_by(email=reset_data.get("email")).first()
-        account.password = base64_password_hashed
-        account.password_salt = base64_salt
-        db.session.commit()
+        if account:
+            account.password = base64_password_hashed
+            account.password_salt = base64_salt
+            db.session.commit()
+            tenant = TenantService.get_join_tenants(account)
+            if not tenant:
+                if not dify_config.ALLOW_CREATE_WORKSPACE:
+                    raise NotAllowedCreateWorkspace()
+                else:
+                    tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
+                    TenantService.create_tenant_member(tenant, account, role="owner")
+                    account.current_tenant = tenant
+                    tenant_was_created.send(tenant)
+        else:
+            account = AccountService.create_account_and_tenant(
+                email=reset_data.get("email"),
+                name=reset_data.get("email"),
+                password=password_confirm,
+                interface_language=languages[0],
+            )
 
-        return {"result": "success"}
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
+
+        return {"result": "success", "data": token}
 
 
 api.add_resource(ForgotPasswordSendEmailApi, "/forgot-password")

api/controllers/console/auth/login.py

@@ -1,16 +1,28 @@
 from typing import cast
 
 import flask_login
-from flask import request
+from flask import redirect, request
 from flask_restful import Resource, reqparse
 
 import services
+from configs import dify_config
+from constants.languages import languages
 from controllers.console import api
+from controllers.console.auth.error import (
+    EmailCodeError,
+    EmailOrPasswordMismatchError,
+    EmailPasswordLoginLimitError,
+    InvalidEmailError,
+    InvalidTokenError,
+)
+from controllers.console.error import EmailSendIpLimitError, NotAllowedCreateWorkspace, NotAllowedRegister
 from controllers.console.setup import setup_required
+from events.tenant_event import tenant_was_created
 from libs.helper import email, get_remote_ip
 from libs.password import valid_password
 from models.account import Account
-from services.account_service import AccountService, TenantService
+from services.account_service import AccountService, RegisterService, TenantService
+from services.errors.workspace import WorkSpaceNotAllowedCreateError
 
 
 class LoginApi(Resource):
@@ -23,15 +35,37 @@ class LoginApi(Resource):
         parser.add_argument("email", type=email, required=True, location="json")
         parser.add_argument("password", type=valid_password, required=True, location="json")
         parser.add_argument("remember_me", type=bool, required=False, default=False, location="json")
+        parser.add_argument("invite_token", type=str, required=False, default=None, location="json")
         args = parser.parse_args()
 
-        # todo: Verify the recaptcha
+        is_login_error_rate_limit = AccountService.is_login_error_rate_limit(args["email"])
+        if is_login_error_rate_limit:
+            raise EmailPasswordLoginLimitError()
+
+        invitation = args["invite_token"]
+        if invitation:
+            invitation = RegisterService.get_invitation_if_token_valid(None, args["email"], invitation)
 
         try:
-            account = AccountService.authenticate(args["email"], args["password"])
-        except services.errors.account.AccountLoginError as e:
-            return {"code": "unauthorized", "message": str(e)}, 401
+            if invitation:
+                data = invitation.get("data", {})
+                invitee_email = data.get("email") if data else None
+                if invitee_email != args["email"]:
+                    raise InvalidEmailError()
+                account = AccountService.authenticate(args["email"], args["password"], args["invite_token"])
+            else:
+                account = AccountService.authenticate(args["email"], args["password"])
+        except services.errors.account.AccountLoginError:
+            raise NotAllowedRegister()
+        except services.errors.account.AccountPasswordError:
+            AccountService.add_login_error_rate_limit(args["email"])
+            raise EmailOrPasswordMismatchError()
+        except services.errors.account.AccountNotFoundError:
+            if not dify_config.ALLOW_REGISTER:
+                raise NotAllowedRegister()
 
+            token = AccountService.send_reset_password_email(email=args["email"])
+            return {"result": "fail", "data": token, "message": "account_not_found"}
         # SELF_HOSTED only have one workspace
         tenants = TenantService.get_join_tenants(account)
         if len(tenants) == 0:
@@ -41,7 +75,7 @@ class LoginApi(Resource):
             }
 
         token = AccountService.login(account, ip_address=get_remote_ip(request))
-
+        AccountService.reset_login_error_rate_limit(args["email"])
         return {"result": "success", "data": token}
 
 
@@ -55,56 +89,111 @@ class LogoutApi(Resource):
         return {"result": "success"}
 
 
-class ResetPasswordApi(Resource):
+class ResetPasswordSendEmailApi(Resource):
     @setup_required
-    def get(self):
-        # parser = reqparse.RequestParser()
-        # parser.add_argument('email', type=email, required=True, location='json')
-        # args = parser.parse_args()
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=email, required=True, location="json")
+        parser.add_argument("language", type=str, required=False, location="json")
+        args = parser.parse_args()
 
-        # import mailchimp_transactional as MailchimpTransactional
-        # from mailchimp_transactional.api_client import ApiClientError
+        if args["language"] is not None and args["language"] == "zh-Hans":
+            language = "zh-Hans"
+        else:
+            language = "en-US"
 
-        # account = {'email': args['email']}
-        # account = AccountService.get_by_email(args['email'])
-        # if account is None:
-        #     raise ValueError('Email not found')
-        # new_password = AccountService.generate_password()
-        # AccountService.update_password(account, new_password)
+        account = AccountService.get_user_through_email(args["email"])
+        if account is None:
+            if dify_config.ALLOW_REGISTER:
+                token = AccountService.send_reset_password_email(email=args["email"], language=language)
+            else:
+                raise NotAllowedRegister()
+        else:
+            token = AccountService.send_reset_password_email(account=account, language=language)
 
-        # todo: Send email
-        # MAILCHIMP_API_KEY = dify_config.MAILCHIMP_TRANSACTIONAL_API_KEY
-        # mailchimp = MailchimpTransactional(MAILCHIMP_API_KEY)
+        return {"result": "success", "data": token}
 
-        # message = {
-        #     'from_email': '[email protected]',
-        #     'to': [{'email': account['email']}],
-        #     'subject': 'Reset your Dify password',
-        #     'html': """
-        #         <p>Dear User,</p>
-        #         <p>The Dify team has generated a new password for you, details as follows:</p>
-        #         <p><strong>{new_password}</strong></p>
-        #         <p>Please change your password to log in as soon as possible.</p>
-        #         <p>Regards,</p>
-        #         <p>The Dify Team</p>
-        #     """
-        # }
 
-        # response = mailchimp.messages.send({
-        #     'message': message,
-        #     # required for transactional email
-        #     ' settings': {
-        #         'sandbox_mode': dify_config.MAILCHIMP_SANDBOX_MODE,
-        #     },
-        # })
+class EmailCodeLoginSendEmailApi(Resource):
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=email, required=True, location="json")
+        parser.add_argument("language", type=str, required=False, location="json")
+        args = parser.parse_args()
 
-        # Check if MSG was sent
-        # if response.status_code != 200:
-        #     # handle error
-        #     pass
+        ip_address = get_remote_ip(request)
+        if AccountService.is_email_send_ip_limit(ip_address):
+            raise EmailSendIpLimitError()
 
-        return {"result": "success"}
+        if args["language"] is not None and args["language"] == "zh-Hans":
+            language = "zh-Hans"
+        else:
+            language = "en-US"
+
+        account = AccountService.get_user_through_email(args["email"])
+        if account is None:
+            if dify_config.ALLOW_REGISTER:
+                token = AccountService.send_email_code_login_email(email=args["email"], language=language)
+            else:
+                raise NotAllowedRegister()
+        else:
+            token = AccountService.send_email_code_login_email(account=account, language=language)
+
+        return {"result": "success", "data": token}
+
+
+class EmailCodeLoginApi(Resource):
+    @setup_required
+    def post(self):
+        parser = reqparse.RequestParser()
+        parser.add_argument("email", type=str, required=True, location="json")
+        parser.add_argument("code", type=str, required=True, location="json")
+        parser.add_argument("token", type=str, required=True, location="json")
+        args = parser.parse_args()
+
+        user_email = args["email"]
+
+        token_data = AccountService.get_email_code_login_data(args["token"])
+        if token_data is None:
+            raise InvalidTokenError()
+
+        if token_data["email"] != args["email"]:
+            raise InvalidEmailError()
+
+        if token_data["code"] != args["code"]:
+            raise EmailCodeError()
+
+        AccountService.revoke_email_code_login_token(args["token"])
+        account = AccountService.get_user_through_email(user_email)
+        if account:
+            tenant = TenantService.get_join_tenants(account)
+            if not tenant:
+                if not dify_config.ALLOW_CREATE_WORKSPACE:
+                    raise NotAllowedCreateWorkspace()
+                else:
+                    tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
+                    TenantService.create_tenant_member(tenant, account, role="owner")
+                    account.current_tenant = tenant
+                    tenant_was_created.send(tenant)
+
+        if account is None:
+            try:
+                account = AccountService.create_account_and_tenant(
+                    email=user_email, name=user_email, interface_language=languages[0]
+                )
+            except WorkSpaceNotAllowedCreateError:
+                return redirect(
+                    f"{dify_config.CONSOLE_WEB_URL}/signin"
+                    "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
+                )
+        token = AccountService.login(account, ip_address=get_remote_ip(request))
+        AccountService.reset_login_error_rate_limit(args["email"])
+        return {"result": "success", "data": token}
 
 
 api.add_resource(LoginApi, "/login")
 api.add_resource(LogoutApi, "/logout")
+api.add_resource(EmailCodeLoginSendEmailApi, "/email-code-login")
+api.add_resource(EmailCodeLoginApi, "/email-code-login/validity")
+api.add_resource(ResetPasswordSendEmailApi, "/reset-password")

api/controllers/console/auth/oauth.py

@@ -5,14 +5,18 @@ from typing import Optional
 import requests
 from flask import current_app, redirect, request
 from flask_restful import Resource
+from werkzeug.exceptions import Unauthorized
 
 from configs import dify_config
 from constants.languages import languages
+from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
 from libs.helper import get_remote_ip
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService, TenantService
+from services.errors.account import AccountNotFoundError
+from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError
 
 from .. import api
 
@@ -42,6 +46,7 @@ def get_oauth_providers():
 
 class OAuthLogin(Resource):
     def get(self, provider: str):
+        invite_token = request.args.get("invite_token") or None
         OAUTH_PROVIDERS = get_oauth_providers()
         with current_app.app_context():
             oauth_provider = OAUTH_PROVIDERS.get(provider)
@@ -49,7 +54,7 @@ class OAuthLogin(Resource):
         if not oauth_provider:
             return {"error": "Invalid provider"}, 400
 
-        auth_url = oauth_provider.get_authorization_url()
+        auth_url = oauth_provider.get_authorization_url(invite_token=invite_token)
         return redirect(auth_url)
 
 
@@ -62,6 +67,11 @@ class OAuthCallback(Resource):
             return {"error": "Invalid provider"}, 400
 
         code = request.args.get("code")
+        state = request.args.get("state")
+        invite_token = None
+        if state:
+            invite_token = state
+
         try:
             token = oauth_provider.get_access_token(code)
             user_info = oauth_provider.get_user_info(token)
@@ -69,7 +79,27 @@ class OAuthCallback(Resource):
             logging.exception(f"An error occurred during the OAuth process with {provider}: {e.response.text}")
             return {"error": "OAuth process failed"}, 400
 
-        account = _generate_account(provider, user_info)
+        if invite_token and RegisterService.is_valid_invite_token(invite_token):
+            invitation = RegisterService._get_invitation_by_token(token=invite_token)
+            if invitation:
+                invitation_email = invitation.get("email", None)
+                if invitation_email != user_info.email:
+                    return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Invalid invitation token.")
+
+            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin/invite-settings?invite_token={invite_token}")
+
+        try:
+            account = _generate_account(provider, user_info)
+        except AccountNotFoundError:
+            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Account not found.")
+        except WorkSpaceNotFoundError:
+            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Workspace not found.")
+        except WorkSpaceNotAllowedCreateError:
+            return redirect(
+                f"{dify_config.CONSOLE_WEB_URL}/signin"
+                "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
+            )
+
         # Check account status
         if account.status in {AccountStatus.BANNED.value, AccountStatus.CLOSED.value}:
             return {"error": "Account is banned or closed."}, 403
@@ -79,7 +109,15 @@ class OAuthCallback(Resource):
             account.initialized_at = datetime.now(timezone.utc).replace(tzinfo=None)
             db.session.commit()
 
-        TenantService.create_owner_tenant_if_not_exist(account)
+        try:
+            TenantService.create_owner_tenant_if_not_exist(account)
+        except Unauthorized:
+            return redirect(f"{dify_config.CONSOLE_WEB_URL}/signin?message=Workspace not found.")
+        except WorkSpaceNotAllowedCreateError:
+            return redirect(
+                f"{dify_config.CONSOLE_WEB_URL}/signin"
+                "?message=Workspace not found, please contact system admin to invite you to join in a workspace."
+            )
 
         token = AccountService.login(account, ip_address=get_remote_ip(request))
 
@@ -99,8 +137,20 @@ def _generate_account(provider: str, user_info: OAuthUserInfo):
     # Get account by openid or email.
     account = _get_account_by_openid_or_email(provider, user_info)
 
+    if account:
+        tenant = TenantService.get_join_tenants(account)
+        if not tenant:
+            if not dify_config.ALLOW_CREATE_WORKSPACE:
+                raise WorkSpaceNotAllowedCreateError()
+            else:
+                tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
+                TenantService.create_tenant_member(tenant, account, role="owner")
+                account.current_tenant = tenant
+                tenant_was_created.send(tenant)
+
     if not account:
-        # Create account
+        if not dify_config.ALLOW_REGISTER:
+            raise AccountNotFoundError()
         account_name = user_info.name or "Dify"
         account = RegisterService.register(
             email=user_info.email, name=account_name, password=None, open_id=user_info.id, provider=provider

api/controllers/console/datasets/datasets.py

@@ -49,7 +49,7 @@ class DatasetListApi(Resource):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
         ids = request.args.getlist("ids")
-        # provider = request.args.get("provider", default="vendor")
+        provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
@@ -57,7 +57,7 @@ class DatasetListApi(Resource):
             datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
         else:
             datasets, total = DatasetService.get_datasets(
-                page, limit, current_user.current_tenant_id, current_user, search, tag_ids
+                page, limit, provider, current_user.current_tenant_id, current_user, search, tag_ids
             )
 
         # check embedding setting
@@ -110,26 +110,6 @@ class DatasetListApi(Resource):
             nullable=True,
             help="Invalid indexing technique.",
         )
-        parser.add_argument(
-            "external_knowledge_api_id",
-            type=str,
-            nullable=True,
-            required=False,
-        )
-        parser.add_argument(
-            "provider",
-            type=str,
-            nullable=True,
-            choices=Dataset.PROVIDER_LIST,
-            required=False,
-            default="vendor",
-        )
-        parser.add_argument(
-            "external_knowledge_id",
-            type=str,
-            nullable=True,
-            required=False,
-        )
         args = parser.parse_args()
 
         # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
@@ -143,9 +123,6 @@ class DatasetListApi(Resource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=DatasetPermissionEnum.ONLY_ME,
-                provider=args["provider"],
-                external_knowledge_api_id=args["external_knowledge_api_id"],
-                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
@@ -234,33 +211,6 @@ class DatasetApi(Resource):
         )
         parser.add_argument("retrieval_model", type=dict, location="json", help="Invalid retrieval model.")
         parser.add_argument("partial_member_list", type=list, location="json", help="Invalid parent user list.")
-
-        parser.add_argument(
-            "external_retrieval_model",
-            type=dict,
-            required=False,
-            nullable=True,
-            location="json",
-            help="Invalid external retrieval model.",
-        )
-
-        parser.add_argument(
-            "external_knowledge_id",
-            type=str,
-            required=False,
-            nullable=True,
-            location="json",
-            help="Invalid external knowledge id.",
-        )
-
-        parser.add_argument(
-            "external_knowledge_api_id",
-            type=str,
-            required=False,
-            nullable=True,
-            location="json",
-            help="Invalid external knowledge api id.",
-        )
         args = parser.parse_args()
         data = request.get_json()
 

api/controllers/console/datasets/external.py

@@ -1,263 +0,0 @@
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, reqparse
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
-
-import services
-from controllers.console import api
-from controllers.console.datasets.error import DatasetNameDuplicateError
-from controllers.console.setup import setup_required
-from controllers.console.wraps import account_initialization_required
-from fields.dataset_fields import dataset_detail_fields
-from libs.login import login_required
-from services.dataset_service import DatasetService
-from services.external_knowledge_service import ExternalDatasetService
-from services.hit_testing_service import HitTestingService
-from services.knowledge_service import ExternalDatasetTestService
-
-
-def _validate_name(name):
-    if not name or len(name) < 1 or len(name) > 100:
-        raise ValueError("Name must be between 1 to 100 characters.")
-    return name
-
-
-def _validate_description_length(description):
-    if description and len(description) > 400:
-        raise ValueError("Description cannot exceed 400 characters.")
-    return description
-
-
-class ExternalApiTemplateListApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self):
-        page = request.args.get("page", default=1, type=int)
-        limit = request.args.get("limit", default=20, type=int)
-        search = request.args.get("keyword", default=None, type=str)
-
-        external_knowledge_apis, total = ExternalDatasetService.get_external_knowledge_apis(
-            page, limit, current_user.current_tenant_id, search
-        )
-        response = {
-            "data": [item.to_dict() for item in external_knowledge_apis],
-            "has_more": len(external_knowledge_apis) == limit,
-            "limit": limit,
-            "total": total,
-            "page": page,
-        }
-        return response, 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument(
-            "name",
-            nullable=False,
-            required=True,
-            help="Name is required. Name must be between 1 to 100 characters.",
-            type=_validate_name,
-        )
-        parser.add_argument(
-            "settings",
-            type=dict,
-            location="json",
-            nullable=False,
-            required=True,
-        )
-        args = parser.parse_args()
-
-        ExternalDatasetService.validate_api_list(args["settings"])
-
-        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
-        if not current_user.is_dataset_editor:
-            raise Forbidden()
-
-        try:
-            external_knowledge_api = ExternalDatasetService.create_external_knowledge_api(
-                tenant_id=current_user.current_tenant_id, user_id=current_user.id, args=args
-            )
-        except services.errors.dataset.DatasetNameDuplicateError:
-            raise DatasetNameDuplicateError()
-
-        return external_knowledge_api.to_dict(), 201
-
-
-class ExternalApiTemplateApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, external_knowledge_api_id):
-        external_knowledge_api_id = str(external_knowledge_api_id)
-        external_knowledge_api = ExternalDatasetService.get_external_knowledge_api(external_knowledge_api_id)
-        if external_knowledge_api is None:
-            raise NotFound("API template not found.")
-
-        return external_knowledge_api.to_dict(), 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def patch(self, external_knowledge_api_id):
-        external_knowledge_api_id = str(external_knowledge_api_id)
-
-        parser = reqparse.RequestParser()
-        parser.add_argument(
-            "name",
-            nullable=False,
-            required=True,
-            help="type is required. Name must be between 1 to 100 characters.",
-            type=_validate_name,
-        )
-        parser.add_argument(
-            "settings",
-            type=dict,
-            location="json",
-            nullable=False,
-            required=True,
-        )
-        args = parser.parse_args()
-        ExternalDatasetService.validate_api_list(args["settings"])
-
-        external_knowledge_api = ExternalDatasetService.update_external_knowledge_api(
-            tenant_id=current_user.current_tenant_id,
-            user_id=current_user.id,
-            external_knowledge_api_id=external_knowledge_api_id,
-            args=args,
-        )
-
-        return external_knowledge_api.to_dict(), 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def delete(self, external_knowledge_api_id):
-        external_knowledge_api_id = str(external_knowledge_api_id)
-
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor or current_user.is_dataset_operator:
-            raise Forbidden()
-
-        ExternalDatasetService.delete_external_knowledge_api(current_user.current_tenant_id, external_knowledge_api_id)
-        return {"result": "success"}, 200
-
-
-class ExternalApiUseCheckApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def get(self, external_knowledge_api_id):
-        external_knowledge_api_id = str(external_knowledge_api_id)
-
-        external_knowledge_api_is_using, count = ExternalDatasetService.external_knowledge_api_use_check(
-            external_knowledge_api_id
-        )
-        return {"is_using": external_knowledge_api_is_using, "count": count}, 200
-
-
-class ExternalDatasetCreateApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not current_user.is_editor:
-            raise Forbidden()
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("external_knowledge_api_id", type=str, required=True, nullable=False, location="json")
-        parser.add_argument("external_knowledge_id", type=str, required=True, nullable=False, location="json")
-        parser.add_argument(
-            "name",
-            nullable=False,
-            required=True,
-            help="name is required. Name must be between 1 to 100 characters.",
-            type=_validate_name,
-        )
-        parser.add_argument("description", type=str, required=False, nullable=True, location="json")
-        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
-
-        args = parser.parse_args()
-
-        # The role of the current user in the ta table must be admin, owner, or editor, or dataset_operator
-        if not current_user.is_dataset_editor:
-            raise Forbidden()
-
-        try:
-            dataset = ExternalDatasetService.create_external_dataset(
-                tenant_id=current_user.current_tenant_id,
-                user_id=current_user.id,
-                args=args,
-            )
-        except services.errors.dataset.DatasetNameDuplicateError:
-            raise DatasetNameDuplicateError()
-
-        return marshal(dataset, dataset_detail_fields), 201
-
-
-class ExternalKnowledgeHitTestingApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self, dataset_id):
-        dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-
-        try:
-            DatasetService.check_dataset_permission(dataset, current_user)
-        except services.errors.account.NoPermissionError as e:
-            raise Forbidden(str(e))
-
-        parser = reqparse.RequestParser()
-        parser.add_argument("query", type=str, location="json")
-        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
-        args = parser.parse_args()
-
-        HitTestingService.hit_testing_args_check(args)
-
-        try:
-            response = HitTestingService.external_retrieve(
-                dataset=dataset,
-                query=args["query"],
-                account=current_user,
-                external_retrieval_model=args["external_retrieval_model"],
-            )
-
-            return response
-        except Exception as e:
-            raise InternalServerError(str(e))
-
-
-class BedrockRetrievalApi(Resource):
-    # this api is only for internal testing
-    def post(self):
-        parser = reqparse.RequestParser()
-        parser.add_argument("retrieval_setting", nullable=False, required=True, type=dict, location="json")
-        parser.add_argument(
-            "query",
-            nullable=False,
-            required=True,
-            type=str,
-        )
-        parser.add_argument("knowledge_id", nullable=False, required=True, type=str)
-        args = parser.parse_args()
-
-        # Call the knowledge retrieval service
-        result = ExternalDatasetTestService.knowledge_retrieval(
-            args["retrieval_setting"], args["query"], args["knowledge_id"]
-        )
-        return result, 200
-
-
-api.add_resource(ExternalKnowledgeHitTestingApi, "/datasets/<uuid:dataset_id>/external-hit-testing")
-api.add_resource(ExternalDatasetCreateApi, "/datasets/external")
-api.add_resource(ExternalApiTemplateListApi, "/datasets/external-knowledge-api")
-api.add_resource(ExternalApiTemplateApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
-api.add_resource(ExternalApiUseCheckApi, "/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")
-# this api is only for internal test
-api.add_resource(BedrockRetrievalApi, "/test/retrieval")

api/controllers/console/datasets/hit_testing.py

@@ -47,7 +47,6 @@ class HitTestingApi(Resource):
         parser = reqparse.RequestParser()
         parser.add_argument("query", type=str, location="json")
         parser.add_argument("retrieval_model", type=dict, required=False, location="json")
-        parser.add_argument("external_retrieval_model", type=dict, required=False, location="json")
         args = parser.parse_args()
 
         HitTestingService.hit_testing_args_check(args)
@@ -58,7 +57,6 @@ class HitTestingApi(Resource):
                 query=args["query"],
                 account=current_user,
                 retrieval_model=args["retrieval_model"],
-                external_retrieval_model=args["external_retrieval_model"],
                 limit=10,
             )
 

api/controllers/console/datasets/website.py

@@ -14,9 +14,7 @@ class WebsiteCrawlApi(Resource):
     @account_initialization_required
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument(
-            "provider", type=str, choices=["firecrawl", "jinareader"], required=True, nullable=True, location="json"
-        )
+        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, nullable=True, location="json")
         parser.add_argument("url", type=str, required=True, nullable=True, location="json")
         parser.add_argument("options", type=dict, required=True, nullable=True, location="json")
         args = parser.parse_args()
@@ -35,7 +33,7 @@ class WebsiteCrawlStatusApi(Resource):
     @account_initialization_required
     def get(self, job_id: str):
         parser = reqparse.RequestParser()
-        parser.add_argument("provider", type=str, choices=["firecrawl", "jinareader"], required=True, location="args")
+        parser.add_argument("provider", type=str, choices=["firecrawl"], required=True, location="args")
         args = parser.parse_args()
         # get crawl status
         try:

api/controllers/console/error.py

@@ -38,3 +38,21 @@ class AlreadyActivateError(BaseHTTPException):
     error_code = "already_activate"
     description = "Auth Token is invalid or account already activated, please check again."
     code = 403
+
+
+class NotAllowedCreateWorkspace(BaseHTTPException):
+    error_code = "unauthorized"
+    description = "Workspace not found, please contact system admin to invite you to join in a workspace."
+    code = 400
+
+
+class NotAllowedRegister(BaseHTTPException):
+    error_code = "unauthorized"
+    description = "Account not found."
+    code = 400
+
+
+class EmailSendIpLimitError(BaseHTTPException):
+    error_code = "email_send_ip_limit"
+    description = "Too many emails have been sent from this IP address recently. Please try again later."
+    code = 429

api/controllers/console/version.py

@@ -38,52 +38,11 @@ class VersionApi(Resource):
             return result
 
         content = json.loads(response.content)
-        if _has_new_version(latest_version=content["version"], current_version=f"{args.get('current_version')}"):
-            result["version"] = content["version"]
-            result["release_date"] = content["releaseDate"]
-            result["release_notes"] = content["releaseNotes"]
-            result["can_auto_update"] = content["canAutoUpdate"]
+        result["version"] = content["version"]
+        result["release_date"] = content["releaseDate"]
+        result["release_notes"] = content["releaseNotes"]
+        result["can_auto_update"] = content["canAutoUpdate"]
         return result
 
 
-def _has_new_version(*, latest_version: str, current_version: str) -> bool:
-    def parse_version(version: str) -> tuple:
-        # Split version into parts and pre-release suffix if any
-        parts = version.split("-")
-        version_parts = parts[0].split(".")
-        pre_release = parts[1] if len(parts) > 1 else None
-
-        # Validate version format
-        if len(version_parts) != 3:
-            raise ValueError(f"Invalid version format: {version}")
-
-        try:
-            # Convert version parts to integers
-            major, minor, patch = map(int, version_parts)
-            return (major, minor, patch, pre_release)
-        except ValueError:
-            raise ValueError(f"Invalid version format: {version}")
-
-    latest = parse_version(latest_version)
-    current = parse_version(current_version)
-
-    # Compare major, minor, and patch versions
-    for latest_part, current_part in zip(latest[:3], current[:3]):
-        if latest_part > current_part:
-            return True
-        elif latest_part < current_part:
-            return False
-
-    # If versions are equal, check pre-release suffixes
-    if latest[3] is None and current[3] is not None:
-        return True
-    elif latest[3] is not None and current[3] is None:
-        return False
-    elif latest[3] is not None and current[3] is not None:
-        # Simple string comparison for pre-release versions
-        return latest[3] > current[3]
-
-    return False
-
-
 api.add_resource(VersionApi, "/version")

api/controllers/console/workspace/model_providers.py

@@ -126,12 +126,13 @@ class ModelProviderIconApi(Resource):
     Get model provider icon
     """
 
+    @setup_required
+    @login_required
+    @account_initialization_required
     def get(self, provider: str, icon_type: str, lang: str):
         model_provider_service = ModelProviderService()
         icon, mimetype = model_provider_service.get_model_provider_icon(
-            provider=provider,
-            icon_type=icon_type,
-            lang=lang,
+            provider=provider, icon_type=icon_type, lang=lang
         )
 
         return send_file(io.BytesIO(icon), mimetype=mimetype)

api/controllers/console/workspace/models.py

@@ -72,9 +72,8 @@ class DefaultModelApi(Resource):
                     provider=model_setting["provider"],
                     model=model_setting["model"],
                 )
-            except Exception as ex:
-                logging.exception(f"{model_setting['model_type']} save error: {ex}")
-                raise ex
+            except Exception:
+                logging.warning(f"{model_setting['model_type']} save error")
 
         return {"result": "success"}
 

api/controllers/files/error.py

@@ -1,7 +0,0 @@
-from libs.exception import BaseHTTPException
-
-
-class UnsupportedFileTypeError(BaseHTTPException):
-    error_code = "unsupported_file_type"
-    description = "File type not allowed."
-    code = 415

api/controllers/files/image_preview.py

@@ -4,7 +4,7 @@ from werkzeug.exceptions import NotFound
 
 import services
 from controllers.files import api
-from controllers.files.error import UnsupportedFileTypeError
+from libs.exception import BaseHTTPException
 from services.account_service import TenantService
 from services.file_service import FileService
 
@@ -50,3 +50,9 @@ class WorkspaceWebappLogoApi(Resource):
 
 api.add_resource(ImagePreviewApi, "/files/<uuid:file_id>/image-preview")
 api.add_resource(WorkspaceWebappLogoApi, "/files/workspaces/<uuid:workspace_id>/webapp-logo")
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = "unsupported_file_type"
+    description = "File type not allowed."
+    code = 415

api/controllers/files/tool_files.py

@@ -3,8 +3,8 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import Forbidden, NotFound
 
 from controllers.files import api
-from controllers.files.error import UnsupportedFileTypeError
 from core.tools.tool_file_manager import ToolFileManager
+from libs.exception import BaseHTTPException
 
 
 class ToolFilePreviewApi(Resource):
@@ -43,3 +43,9 @@ class ToolFilePreviewApi(Resource):
 
 
 api.add_resource(ToolFilePreviewApi, "/files/tools/<uuid:file_id>.<string:extension>")
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = "unsupported_file_type"
+    description = "File type not allowed."
+    code = 415

api/controllers/service_api/app/completion.py

@@ -4,7 +4,6 @@ from flask_restful import Resource, reqparse
 from werkzeug.exceptions import InternalServerError, NotFound
 
 import services
-from constants import UUID_NIL
 from controllers.service_api import api
 from controllers.service_api.app.error import (
     AppUnavailableError,
@@ -108,7 +107,6 @@ class ChatApi(Resource):
         parser.add_argument("conversation_id", type=uuid_value, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="dev", location="json")
         parser.add_argument("auto_generate_name", type=bool, required=False, default=True, location="json")
-        parser.add_argument("parent_message_id", type=uuid_value, required=False, default=UUID_NIL, location="json")
 
         args = parser.parse_args()
 

api/controllers/service_api/dataset/dataset.py

@@ -28,11 +28,11 @@ class DatasetListApi(DatasetApiResource):
 
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
-        # provider = request.args.get("provider", default="vendor")
+        provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
 
-        datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids)
+        datasets, total = DatasetService.get_datasets(page, limit, provider, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)
@@ -82,26 +82,6 @@ class DatasetListApi(DatasetApiResource):
             required=False,
             nullable=False,
         )
-        parser.add_argument(
-            "external_knowledge_api_id",
-            type=str,
-            nullable=True,
-            required=False,
-            default="_validate_name",
-        )
-        parser.add_argument(
-            "provider",
-            type=str,
-            nullable=True,
-            required=False,
-            default="vendor",
-        )
-        parser.add_argument(
-            "external_knowledge_id",
-            type=str,
-            nullable=True,
-            required=False,
-        )
         args = parser.parse_args()
 
         try:
@@ -111,9 +91,6 @@ class DatasetListApi(DatasetApiResource):
                 indexing_technique=args["indexing_technique"],
                 account=current_user,
                 permission=args["permission"],
-                provider=args["provider"],
-                external_knowledge_api_id=args["external_knowledge_api_id"],
-                external_knowledge_id=args["external_knowledge_id"],
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()

api/core/agent/cot_agent_runner.py

@@ -369,7 +369,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
         return message
 
     def _organize_historic_prompt_messages(
-        self, current_session_messages: Optional[list[PromptMessage]] = None
+        self, current_session_messages: list[PromptMessage] = None
     ) -> list[PromptMessage]:
         """
         organize historic prompt messages

api/core/agent/cot_chat_agent_runner.py

@@ -27,7 +27,7 @@ class CotChatAgentRunner(CotAgentRunner):
 
         return SystemPromptMessage(content=system_prompt)
 
-    def _organize_user_query(self, query, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+    def _organize_user_query(self, query, prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
         """
         Organize user query
         """

api/core/agent/cot_completion_agent_runner.py

@@ -1,5 +1,4 @@
 import json
-from typing import Optional
 
 from core.agent.cot_agent_runner import CotAgentRunner
 from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
@@ -22,7 +21,7 @@ class CotCompletionAgentRunner(CotAgentRunner):
 
         return system_prompt
 
-    def _organize_historic_prompt(self, current_session_messages: Optional[list[PromptMessage]] = None) -> str:
+    def _organize_historic_prompt(self, current_session_messages: list[PromptMessage] = None) -> str:
         """
         Organize historic prompt
         """

api/core/agent/fc_agent_runner.py

@@ -2,7 +2,7 @@ import json
 import logging
 from collections.abc import Generator
 from copy import deepcopy
-from typing import Any, Optional, Union
+from typing import Any, Union
 
 from core.agent.base_agent_runner import BaseAgentRunner
 from core.app.apps.base_app_queue_manager import PublishFrom
@@ -370,7 +370,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
         return tool_calls
 
     def _init_system_message(
-        self, prompt_template: str, prompt_messages: Optional[list[PromptMessage]] = None
+        self, prompt_template: str, prompt_messages: list[PromptMessage] = None
     ) -> list[PromptMessage]:
         """
         Initialize system message
@@ -385,7 +385,7 @@ class FunctionCallAgentRunner(BaseAgentRunner):
 
         return prompt_messages
 
-    def _organize_user_query(self, query, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
+    def _organize_user_query(self, query, prompt_messages: list[PromptMessage] = None) -> list[PromptMessage]:
         """
         Organize user query
         """

api/core/agent/output_parser/cot_output_parser.py

@@ -14,7 +14,7 @@ class CotAgentOutputParser:
     ) -> Generator[Union[str, AgentScratchpadUnit.Action], None, None]:
         def parse_action(json_str):
             try:
-                action = json.loads(json_str, strict=False)
+                action = json.loads(json_str)
                 action_name = None
                 action_input = None
 

api/core/app/apps/advanced_chat/app_generator.py

@@ -113,7 +113,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             # always enable retriever resource in debugger mode
             app_config.additional_features.show_retrieve_source = True
 
-        workflow_run_id = str(uuid.uuid4())
         # init application generate entity
         application_generate_entity = AdvancedChatAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -128,7 +127,6 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             invoke_from=invoke_from,
             extras=extras,
             trace_manager=trace_manager,
-            workflow_run_id=workflow_run_id,
         )
         contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 

api/core/app/apps/advanced_chat/app_runner.py

@@ -149,9 +149,6 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
                 SystemVariableKey.CONVERSATION_ID: self.conversation.id,
                 SystemVariableKey.USER_ID: user_id,
                 SystemVariableKey.DIALOGUE_COUNT: conversation_dialogue_count,
-                SystemVariableKey.APP_ID: app_config.app_id,
-                SystemVariableKey.WORKFLOW_ID: app_config.workflow_id,
-                SystemVariableKey.WORKFLOW_RUN_ID: self.application_generate_entity.workflow_run_id,
             }
 
             # init variable pool

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -45,7 +45,6 @@ from core.app.entities.task_entities import (
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.app.task_pipeline.message_cycle_manage import MessageCycleManage
 from core.app.task_pipeline.workflow_cycle_manage import WorkflowCycleManage
-from core.model_runtime.entities.llm_entities import LLMUsage
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.ops.ops_trace_manager import TraceQueueManager
 from core.workflow.enums import SystemVariableKey
@@ -108,10 +107,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             SystemVariableKey.FILES: application_generate_entity.files,
             SystemVariableKey.CONVERSATION_ID: conversation.id,
             SystemVariableKey.USER_ID: user_id,
-            SystemVariableKey.DIALOGUE_COUNT: conversation.dialogue_count,
-            SystemVariableKey.APP_ID: application_generate_entity.app_config.app_id,
-            SystemVariableKey.WORKFLOW_ID: workflow.id,
-            SystemVariableKey.WORKFLOW_RUN_ID: application_generate_entity.workflow_run_id,
         }
 
         self._task_state = WorkflowTaskState()
@@ -510,10 +505,6 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
             self._message.total_price = usage.total_price
             self._message.currency = usage.currency
 
-            self._task_state.metadata["usage"] = jsonable_encoder(usage)
-        else:
-            self._task_state.metadata["usage"] = jsonable_encoder(LLMUsage.empty_usage())
-
         db.session.commit()
 
         message_was_created.send(

api/core/app/apps/workflow/app_generator.py

@@ -99,7 +99,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
         user_id = user.id if isinstance(user, Account) else user.session_id
         trace_manager = TraceQueueManager(app_model.id, user_id)
 
-        workflow_run_id = str(uuid.uuid4())
         # init application generate entity
         application_generate_entity = WorkflowAppGenerateEntity(
             task_id=str(uuid.uuid4()),
@@ -111,7 +110,6 @@ class WorkflowAppGenerator(BaseAppGenerator):
             invoke_from=invoke_from,
             call_depth=call_depth,
             trace_manager=trace_manager,
-            workflow_run_id=workflow_run_id,
         )
         contexts.tenant_id.set(application_generate_entity.app_config.tenant_id)
 

api/core/app/apps/workflow/app_runner.py

@@ -90,9 +90,6 @@ class WorkflowAppRunner(WorkflowBasedAppRunner):
             system_inputs = {
                 SystemVariableKey.FILES: files,
                 SystemVariableKey.USER_ID: user_id,
-                SystemVariableKey.APP_ID: app_config.app_id,
-                SystemVariableKey.WORKFLOW_ID: app_config.workflow_id,
-                SystemVariableKey.WORKFLOW_RUN_ID: self.application_generate_entity.workflow_run_id,
             }
 
             variable_pool = VariablePool(

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -97,9 +97,6 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
         self._workflow_system_variables = {
             SystemVariableKey.FILES: application_generate_entity.files,
             SystemVariableKey.USER_ID: user_id,
-            SystemVariableKey.APP_ID: application_generate_entity.app_config.app_id,
-            SystemVariableKey.WORKFLOW_ID: workflow.id,
-            SystemVariableKey.WORKFLOW_RUN_ID: application_generate_entity.workflow_run_id,
         }
 
         self._task_state = WorkflowTaskState()

api/core/app/entities/app_invoke_entities.py

@@ -152,7 +152,6 @@ class AdvancedChatAppGenerateEntity(AppGenerateEntity):
 
     conversation_id: Optional[str] = None
     parent_message_id: Optional[str] = None
-    workflow_run_id: Optional[str] = None
     query: str
 
     class SingleIterationRunEntity(BaseModel):
@@ -173,7 +172,6 @@ class WorkflowAppGenerateEntity(AppGenerateEntity):
 
     # app config
     app_config: WorkflowUIBasedAppConfig
-    workflow_run_id: Optional[str] = None
 
     class SingleIterationRunEntity(BaseModel):
         """

api/core/app/segments/exc.py

@@ -1,2 +1,2 @@
-class VariableError(ValueError):
+class VariableError(Exception):
     pass

api/core/app/task_pipeline/message_cycle_manage.py

@@ -82,8 +82,8 @@ class MessageCycleManage:
                 try:
                     name = LLMGenerator.generate_conversation_name(app_model.tenant_id, query)
                     conversation.name = name
-                except Exception as e:
-                    logging.exception(f"generate conversation name failed: {e}")
+                except:
+                    pass
 
                 db.session.merge(conversation)
                 db.session.commit()

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -85,9 +85,6 @@ class WorkflowCycleManage:
 
         # init workflow run
         workflow_run = WorkflowRun()
-        workflow_run_id = self._workflow_system_variables[SystemVariableKey.WORKFLOW_RUN_ID]
-        if workflow_run_id:
-            workflow_run.id = workflow_run_id
         workflow_run.tenant_id = self._workflow.tenant_id
         workflow_run.app_id = self._workflow.app_id
         workflow_run.sequence_number = new_sequence_number

api/core/callback_handler/agent_tool_callback_handler.py

@@ -1,9 +1,9 @@
+import os
 from collections.abc import Mapping, Sequence
 from typing import Any, Optional, TextIO, Union
 
 from pydantic import BaseModel
 
-from configs import dify_config
 from core.ops.entities.trace_entity import TraceTaskName
 from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.tools.entities.tool_entities import ToolInvokeMessage
@@ -50,8 +50,7 @@ class DifyAgentCallbackHandler(BaseModel):
         tool_inputs: Mapping[str, Any],
     ) -> None:
         """Do nothing."""
-        if dify_config.DEBUG:
-            print_text("\n[on_tool_start] ToolCall:" + tool_name + "\n" + str(tool_inputs) + "\n", color=self.color)
+        print_text("\n[on_tool_start] ToolCall:" + tool_name + "\n" + str(tool_inputs) + "\n", color=self.color)
 
     def on_tool_end(
         self,
@@ -63,12 +62,11 @@ class DifyAgentCallbackHandler(BaseModel):
         trace_manager: Optional[TraceQueueManager] = None,
     ) -> None:
         """If not the final action, print out observation."""
-        if dify_config.DEBUG:
-            print_text("\n[on_tool_end]\n", color=self.color)
-            print_text("Tool: " + tool_name + "\n", color=self.color)
-            print_text("Inputs: " + str(tool_inputs) + "\n", color=self.color)
-            print_text("Outputs: " + str(tool_outputs)[:1000] + "\n", color=self.color)
-            print_text("\n")
+        print_text("\n[on_tool_end]\n", color=self.color)
+        print_text("Tool: " + tool_name + "\n", color=self.color)
+        print_text("Inputs: " + str(tool_inputs) + "\n", color=self.color)
+        print_text("Outputs: " + str(tool_outputs)[:1000] + "\n", color=self.color)
+        print_text("\n")
 
         if trace_manager:
             trace_manager.add_trace_task(
@@ -84,33 +82,30 @@ class DifyAgentCallbackHandler(BaseModel):
 
     def on_tool_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any) -> None:
         """Do nothing."""
-        if dify_config.DEBUG:
-            print_text("\n[on_tool_error] Error: " + str(error) + "\n", color="red")
+        print_text("\n[on_tool_error] Error: " + str(error) + "\n", color="red")
 
     def on_agent_start(self, thought: str) -> None:
         """Run on agent start."""
-        if dify_config.DEBUG:
-            if thought:
-                print_text(
-                    "\n[on_agent_start] \nCurrent Loop: " + str(self.current_loop) + "\nThought: " + thought + "\n",
-                    color=self.color,
-                )
-            else:
-                print_text("\n[on_agent_start] \nCurrent Loop: " + str(self.current_loop) + "\n", color=self.color)
+        if thought:
+            print_text(
+                "\n[on_agent_start] \nCurrent Loop: " + str(self.current_loop) + "\nThought: " + thought + "\n",
+                color=self.color,
+            )
+        else:
+            print_text("\n[on_agent_start] \nCurrent Loop: " + str(self.current_loop) + "\n", color=self.color)
 
     def on_agent_finish(self, color: Optional[str] = None, **kwargs: Any) -> None:
         """Run on agent end."""
-        if dify_config.DEBUG:
-            print_text("\n[on_agent_finish]\n Loop: " + str(self.current_loop) + "\n", color=self.color)
+        print_text("\n[on_agent_finish]\n Loop: " + str(self.current_loop) + "\n", color=self.color)
 
         self.current_loop += 1
 
     @property
     def ignore_agent(self) -> bool:
         """Whether to ignore agent callbacks."""
-        return not dify_config.DEBUG
+        return not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != "true"
 
     @property
     def ignore_chat_model(self) -> bool:
         """Whether to ignore chat model callbacks."""
-        return not dify_config.DEBUG
+        return not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != "true"

api/core/callback_handler/index_tool_callback_handler.py

@@ -44,6 +44,7 @@ class DatasetIndexToolCallbackHandler:
                 DocumentSegment.index_node_id == document.metadata["doc_id"]
             )
 
+            # if 'dataset_id' in document.metadata:
             if "dataset_id" in document.metadata:
                 query = query.filter(DocumentSegment.dataset_id == document.metadata["dataset_id"])
 
@@ -58,7 +59,7 @@ class DatasetIndexToolCallbackHandler:
             for item in resource:
                 dataset_retriever_resource = DatasetRetrieverResource(
                     message_id=self._message_id,
-                    position=item.get("position") or 0,
+                    position=item.get("position"),
                     dataset_id=item.get("dataset_id"),
                     dataset_name=item.get("dataset_name"),
                     document_id=item.get("document_id"),

api/core/file/message_file_parser.py

@@ -198,34 +198,16 @@ class MessageFileParser:
                     if "amazonaws.com" not in parsed_url.netloc:
                         return False
                     query_params = parse_qs(parsed_url.query)
-
-                    def check_presign_v2(query_params):
-                        required_params = ["Signature", "Expires"]
-                        for param in required_params:
-                            if param not in query_params:
-                                return False
-                        if not query_params["Expires"][0].isdigit():
+                    required_params = ["Signature", "Expires"]
+                    for param in required_params:
+                        if param not in query_params:
                             return False
-                        signature = query_params["Signature"][0]
-                        if not re.match(r"^[A-Za-z0-9+/]+={0,2}$", signature):
-                            return False
-
-                        return True
-
-                    def check_presign_v4(query_params):
-                        required_params = ["X-Amz-Signature", "X-Amz-Expires"]
-                        for param in required_params:
-                            if param not in query_params:
-                                return False
-                        if not query_params["X-Amz-Expires"][0].isdigit():
-                            return False
-                        signature = query_params["X-Amz-Signature"][0]
-                        if not re.match(r"^[A-Za-z0-9+/]+={0,2}$", signature):
-                            return False
-
-                        return True
-
-                    return check_presign_v4(query_params) or check_presign_v2(query_params)
+                    if not query_params["Expires"][0].isdigit():
+                        return False
+                    signature = query_params["Signature"][0]
+                    if not re.match(r"^[A-Za-z0-9+/]+={0,2}$", signature):
+                        return False
+                    return True
                 except Exception:
                     return False
 

api/core/indexing_runner.py

@@ -211,9 +211,9 @@ class IndexingRunner:
         tenant_id: str,
         extract_settings: list[ExtractSetting],
         tmp_processing_rule: dict,
-        doc_form: Optional[str] = None,
+        doc_form: str = None,
         doc_language: str = "English",
-        dataset_id: Optional[str] = None,
+        dataset_id: str = None,
         indexing_technique: str = "economy",
     ) -> dict:
         """

api/core/memory/token_buffer_memory.py

@@ -58,11 +58,7 @@ class TokenBufferMemory:
         # instead of all messages from the conversation, we only need to extract messages
         # that belong to the thread of last message
         thread_messages = extract_thread_messages(messages)
-
-        # for newly created message, its answer is temporarily empty, we don't need to add it to memory
-        if thread_messages and not thread_messages[-1].answer:
-            thread_messages.pop()
-
+        thread_messages.pop(0)
         messages = list(reversed(thread_messages))
 
         message_file_parser = MessageFileParser(tenant_id=app_record.tenant_id, app_id=app_record.id)

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -94,7 +94,7 @@ class LargeLanguageModel(AIModel):
         )
 
         try:
-            if "response_format" in model_parameters and model_parameters["response_format"] in {"JSON", "XML"}:
+            if "response_format" in model_parameters:
                 result = self._code_block_mode_wrapper(
                     model=model,
                     credentials=credentials,

api/core/model_runtime/model_providers/__base/tts_model.py

@@ -1,7 +1,7 @@
 import logging
 import re
 from abc import abstractmethod
-from typing import Any, Optional
+from typing import Optional
 
 from pydantic import ConfigDict
 
@@ -88,7 +88,7 @@ class TTSModel(AIModel):
             else:
                 return [{"name": d["name"], "value": d["mode"]} for d in voices]
 
-    def _get_model_default_voice(self, model: str, credentials: dict) -> Any:
+    def _get_model_default_voice(self, model: str, credentials: dict) -> any:
         """
         Get voice for given tts model
 

api/core/model_runtime/model_providers/anthropic/llm/llm.py

@@ -169,7 +169,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel):
         stop: Optional[list[str]] = None,
         stream: bool = True,
         user: Optional[str] = None,
-        callbacks: Optional[list[Callback]] = None,
+        callbacks: list[Callback] = None,
     ) -> Union[LLMResult, Generator]:
         """
         Code block mode wrapper for invoking large language model

api/core/model_runtime/model_providers/azure_openai/_constant.py

@@ -1081,81 +1081,8 @@ LLM_BASE_MODELS = [
             ),
         ),
     ),
-    AzureBaseModel(
-        base_model_name="o1-preview",
-        entity=AIModelEntity(
-            model="fake-deployment-name",
-            label=I18nObject(
-                en_US="fake-deployment-name-label",
-            ),
-            model_type=ModelType.LLM,
-            features=[
-                ModelFeature.AGENT_THOUGHT,
-            ],
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={
-                ModelPropertyKey.MODE: LLMMode.CHAT.value,
-                ModelPropertyKey.CONTEXT_SIZE: 128000,
-            },
-            parameter_rules=[
-                ParameterRule(
-                    name="response_format",
-                    label=I18nObject(zh_Hans="回复格式", en_US="response_format"),
-                    type="string",
-                    help=I18nObject(
-                        zh_Hans="指定模型必须输出的格式", en_US="specifying the format that the model must output"
-                    ),
-                    required=False,
-                    options=["text", "json_object"],
-                ),
-                _get_max_tokens(default=512, min_val=1, max_val=32768),
-            ],
-            pricing=PriceConfig(
-                input=15.00,
-                output=60.00,
-                unit=0.000001,
-                currency="USD",
-            ),
-        ),
-    ),
-    AzureBaseModel(
-        base_model_name="o1-mini",
-        entity=AIModelEntity(
-            model="fake-deployment-name",
-            label=I18nObject(
-                en_US="fake-deployment-name-label",
-            ),
-            model_type=ModelType.LLM,
-            features=[
-                ModelFeature.AGENT_THOUGHT,
-            ],
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={
-                ModelPropertyKey.MODE: LLMMode.CHAT.value,
-                ModelPropertyKey.CONTEXT_SIZE: 128000,
-            },
-            parameter_rules=[
-                ParameterRule(
-                    name="response_format",
-                    label=I18nObject(zh_Hans="回复格式", en_US="response_format"),
-                    type="string",
-                    help=I18nObject(
-                        zh_Hans="指定模型必须输出的格式", en_US="specifying the format that the model must output"
-                    ),
-                    required=False,
-                    options=["text", "json_object"],
-                ),
-                _get_max_tokens(default=512, min_val=1, max_val=65536),
-            ],
-            pricing=PriceConfig(
-                input=3.00,
-                output=12.00,
-                unit=0.000001,
-                currency="USD",
-            ),
-        ),
-    ),
 ]
+
 EMBEDDING_BASE_MODELS = [
     AzureBaseModel(
         base_model_name="text-embedding-ada-002",

api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml

@@ -53,9 +53,6 @@ model_credential_schema:
       type: select
       required: true
       options:
-        - label:
-            en_US: 2024-09-01-preview
-          value: 2024-09-01-preview
         - label:
             en_US: 2024-08-01-preview
           value: 2024-08-01-preview
@@ -123,18 +120,6 @@ model_credential_schema:
           show_on:
             - variable: __model_type
               value: llm
-        - label:
-            en_US: o1-mini
-          value: o1-mini
-          show_on:
-            - variable: __model_type
-              value: llm
-        - label:
-            en_US: o1-preview
-          value: o1-preview
-          show_on:
-            - variable: __model_type
-              value: llm
         - label:
             en_US: gpt-4o-mini
           value: gpt-4o-mini

api/core/model_runtime/model_providers/azure_openai/llm/llm.py

@@ -312,24 +312,10 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel):
         if user:
             extra_model_kwargs["user"] = user
 
-        # clear illegal prompt messages
-        prompt_messages = self._clear_illegal_prompt_messages(model, prompt_messages)
-
-        block_as_stream = False
-        if model.startswith("o1"):
-            if stream:
-                block_as_stream = True
-                stream = False
-
-                if "stream_options" in extra_model_kwargs:
-                    del extra_model_kwargs["stream_options"]
-
-            if "stop" in extra_model_kwargs:
-                del extra_model_kwargs["stop"]
-
         # chat model
+        messages = [self._convert_prompt_message_to_dict(m) for m in prompt_messages]
         response = client.chat.completions.create(
-            messages=[self._convert_prompt_message_to_dict(m) for m in prompt_messages],
+            messages=messages,
             model=model,
             stream=stream,
             **model_parameters,
@@ -339,91 +325,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel):
         if stream:
             return self._handle_chat_generate_stream_response(model, credentials, response, prompt_messages, tools)
 
-        block_result = self._handle_chat_generate_response(model, credentials, response, prompt_messages, tools)
-
-        if block_as_stream:
-            return self._handle_chat_block_as_stream_response(block_result, prompt_messages, stop)
-
-        return block_result
-
-    def _handle_chat_block_as_stream_response(
-        self,
-        block_result: LLMResult,
-        prompt_messages: list[PromptMessage],
-        stop: Optional[list[str]] = None,
-    ) -> Generator[LLMResultChunk, None, None]:
-        """
-        Handle llm chat response
-
-        :param model: model name
-        :param credentials: credentials
-        :param response: response
-        :param prompt_messages: prompt messages
-        :param tools: tools for tool calling
-        :param stop: stop words
-        :return: llm response chunk generator
-        """
-        text = block_result.message.content
-        text = cast(str, text)
-
-        if stop:
-            text = self.enforce_stop_tokens(text, stop)
-
-        yield LLMResultChunk(
-            model=block_result.model,
-            prompt_messages=prompt_messages,
-            system_fingerprint=block_result.system_fingerprint,
-            delta=LLMResultChunkDelta(
-                index=0,
-                message=AssistantPromptMessage(content=text),
-                finish_reason="stop",
-                usage=block_result.usage,
-            ),
-        )
-
-    def _clear_illegal_prompt_messages(self, model: str, prompt_messages: list[PromptMessage]) -> list[PromptMessage]:
-        """
-        Clear illegal prompt messages for OpenAI API
-
-        :param model: model name
-        :param prompt_messages: prompt messages
-        :return: cleaned prompt messages
-        """
-        checklist = ["gpt-4-turbo", "gpt-4-turbo-2024-04-09"]
-
-        if model in checklist:
-            # count how many user messages are there
-            user_message_count = len([m for m in prompt_messages if isinstance(m, UserPromptMessage)])
-            if user_message_count > 1:
-                for prompt_message in prompt_messages:
-                    if isinstance(prompt_message, UserPromptMessage):
-                        if isinstance(prompt_message.content, list):
-                            prompt_message.content = "\n".join(
-                                [
-                                    item.data
-                                    if item.type == PromptMessageContentType.TEXT
-                                    else "[IMAGE]"
-                                    if item.type == PromptMessageContentType.IMAGE
-                                    else ""
-                                    for item in prompt_message.content
-                                ]
-                            )
-
-        if model.startswith("o1"):
-            system_message_count = len([m for m in prompt_messages if isinstance(m, SystemPromptMessage)])
-            if system_message_count > 0:
-                new_prompt_messages = []
-                for prompt_message in prompt_messages:
-                    if isinstance(prompt_message, SystemPromptMessage):
-                        prompt_message = UserPromptMessage(
-                            content=prompt_message.content,
-                            name=prompt_message.name,
-                        )
-
-                    new_prompt_messages.append(prompt_message)
-                prompt_messages = new_prompt_messages
-
-        return prompt_messages
+        return self._handle_chat_generate_response(model, credentials, response, prompt_messages, tools)
 
     def _handle_chat_generate_response(
         self,
@@ -658,7 +560,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel):
             tokens_per_message = 4
             # if there's a name, the role is omitted
             tokens_per_name = -1
-        elif model.startswith("gpt-35-turbo") or model.startswith("gpt-4") or model.startswith("o1"):
+        elif model.startswith("gpt-35-turbo") or model.startswith("gpt-4"):
             tokens_per_message = 3
             tokens_per_name = 1
         else:

api/core/model_runtime/model_providers/azure_openai/tts/tts.py

@@ -1,6 +1,6 @@
 import concurrent.futures
 import copy
-from typing import Any, Optional
+from typing import Optional
 
 from openai import AzureOpenAI
 
@@ -19,7 +19,7 @@ class AzureOpenAIText2SpeechModel(_CommonAzureOpenAI, TTSModel):
 
     def _invoke(
         self, model: str, tenant_id: str, credentials: dict, content_text: str, voice: str, user: Optional[str] = None
-    ) -> Any:
+    ) -> any:
         """
         _invoke text2speech model
 
@@ -56,7 +56,7 @@ class AzureOpenAIText2SpeechModel(_CommonAzureOpenAI, TTSModel):
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
 
-    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> Any:
+    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> any:
         """
         _tts_invoke_streaming text2speech model
         :param model: model name

api/core/model_runtime/model_providers/bedrock/bedrock.yaml

@@ -50,62 +50,34 @@ provider_credential_schema:
           label:
             en_US: US East (N. Virginia)
             zh_Hans: 美国东部 (弗吉尼亚北部)
-        - value: us-east-2
-          label:
-            en_US: US East (Ohio)
-            zh_Hans: 美国东部 (弗吉尼亚北部)
         - value: us-west-2
           label:
             en_US: US West (Oregon)
             zh_Hans: 美国西部 (俄勒冈州)
-        - value: ap-south-1
-          label:
-            en_US: Asia Pacific (Mumbai)
-            zh_Hans: 亚太地区（孟买）
         - value: ap-southeast-1
           label:
             en_US: Asia Pacific (Singapore)
             zh_Hans: 亚太地区 (新加坡)
-        - value: ap-southeast-2
-          label:
-            en_US: Asia Pacific (Sydney)
-            zh_Hans: 亚太地区 (悉尼)
         - value: ap-northeast-1
           label:
             en_US: Asia Pacific (Tokyo)
             zh_Hans: 亚太地区 (东京)
-        - value: ap-northeast-2
-          label:
-            en_US: Asia Pacific (Seoul)
-            zh_Hans: 亚太地区（首尔）
-        - value: ca-central-1
-          label:
-            en_US: Canada (Central)
-            zh_Hans: 加拿大（中部）
         - value: eu-central-1
           label:
             en_US: Europe (Frankfurt)
             zh_Hans: 欧洲 (法兰克福)
-        - value: eu-west-1
-          label:
-            en_US: Europe (Ireland)
-            zh_Hans: 欧洲（爱尔兰）
         - value: eu-west-2
           label:
-            en_US: Europe (London)
+            en_US: Eu west London (London)
             zh_Hans: 欧洲西部 (伦敦)
-        - value: eu-west-3
-          label:
-            en_US: Europe (Paris)
-            zh_Hans: 欧洲（巴黎）
-        - value: sa-east-1
-          label:
-            en_US: South America (São Paulo)
-            zh_Hans: 南美洲（圣保罗）
         - value: us-gov-west-1
           label:
             en_US: AWS GovCloud (US-West)
             zh_Hans: AWS GovCloud (US-West)
+        - value: ap-southeast-2
+          label:
+            en_US: Asia Pacific (Sydney)
+            zh_Hans: 亚太地区 (悉尼)
     - variable: model_for_validation
       required: false
       label:

api/core/model_runtime/model_providers/bedrock/llm/llm.py

@@ -92,7 +92,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel):
         stop: Optional[list[str]] = None,
         stream: bool = True,
         user: Optional[str] = None,
-        callbacks: Optional[list[Callback]] = None,
+        callbacks: list[Callback] = None,
     ) -> Union[LLMResult, Generator]:
         """
         Code block mode wrapper for invoking large language model

api/core/model_runtime/model_providers/fireworks/llm/llm.py

@@ -511,7 +511,7 @@ class FireworksLargeLanguageModel(_CommonFireworks, LargeLanguageModel):
         model: str,
         messages: list[PromptMessage],
         tools: Optional[list[PromptMessageTool]] = None,
-        credentials: Optional[dict] = None,
+        credentials: dict = None,
     ) -> int:
         """
         Approximate num tokens with GPT2 tokenizer.

api/core/model_runtime/model_providers/fishaudio/tts/tts.py

@@ -1,4 +1,4 @@
-from typing import Any, Optional
+from typing import Optional
 
 import httpx
 
@@ -46,7 +46,7 @@ class FishAudioText2SpeechModel(TTSModel):
         content_text: str,
         voice: str,
         user: Optional[str] = None,
-    ) -> Any:
+    ) -> any:
         """
         Invoke text2speech model
 
@@ -87,7 +87,7 @@ class FishAudioText2SpeechModel(TTSModel):
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
 
-    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> Any:
+    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> any:
         """
         Invoke streaming text2speech model
         :param model: model name
@@ -112,7 +112,7 @@ class FishAudioText2SpeechModel(TTSModel):
         except Exception as ex:
             raise InvokeBadRequestError(str(ex))
 
-    def _tts_invoke_streaming_sentence(self, credentials: dict, content_text: str, voice: Optional[str] = None) -> Any:
+    def _tts_invoke_streaming_sentence(self, credentials: dict, content_text: str, voice: Optional[str] = None) -> any:
         """
         Invoke streaming text2speech model
 

api/core/model_runtime/model_providers/google/llm/_position.yaml

@@ -1,15 +0,0 @@
-- gemini-1.5-pro
-- gemini-1.5-pro-latest
-- gemini-1.5-pro-001
-- gemini-1.5-pro-002
-- gemini-1.5-pro-exp-0801
-- gemini-1.5-pro-exp-0827
-- gemini-1.5-flash
-- gemini-1.5-flash-latest
-- gemini-1.5-flash-001
-- gemini-1.5-flash-002
-- gemini-1.5-flash-exp-0827
-- gemini-1.5-flash-8b-exp-0827
-- gemini-1.5-flash-8b-exp-0924
-- gemini-pro
-- gemini-pro-vision

api/core/model_runtime/model_providers/groq/llm/_position.yaml

@@ -5,4 +5,3 @@
 - llama3-8b-8192
 - mixtral-8x7b-32768
 - llama2-70b-4096
-- llama-guard-3-8b

api/core/model_runtime/model_providers/groq/llm/llama-guard-3-8b.yaml

@@ -1,25 +0,0 @@
-model: llama-guard-3-8b
-label:
-  zh_Hans: Llama-Guard-3-8B
-  en_US: Llama-Guard-3-8B
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 8192
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: top_p
-    use_template: top_p
-  - name: max_tokens
-    use_template: max_tokens
-    default: 512
-    min: 1
-    max: 8192
-pricing:
-  input: '0.20'
-  output: '0.20'
-  unit: '0.000001'
-  currency: USD

api/core/model_runtime/model_providers/jina/rerank/rerank.py

@@ -61,19 +61,11 @@ class JinaRerankModel(RerankModel):
 
             rerank_documents = []
             for result in results["results"]:
-                index = result["index"]
-                if "document" in result:
-                    text = result["document"]["text"]
-                else:
-                    # llama.cpp rerank maynot return original documents
-                    text = docs[index]
-
                 rerank_document = RerankDocument(
-                    index=index,
-                    text=text,
+                    index=result["index"],
+                    text=result["document"]["text"],
                     score=result["relevance_score"],
                 )
-
                 if score_threshold is None or result["relevance_score"] >= score_threshold:
                     rerank_documents.append(rerank_document)
 

api/core/model_runtime/model_providers/localai/rerank/rerank.py

@@ -70,19 +70,11 @@ class LocalaiRerankModel(RerankModel):
 
             rerank_documents = []
             for result in results["results"]:
-                index = result["index"]
-                if "document" in result:
-                    text = result["document"]["text"]
-                else:
-                    # llama.cpp rerank maynot return original documents
-                    text = docs[index]
-
                 rerank_document = RerankDocument(
-                    index=index,
-                    text=text,
+                    index=result["index"],
+                    text=result["document"]["text"],
                     score=result["relevance_score"],
                 )
-
                 if score_threshold is None or result["relevance_score"] >= score_threshold:
                     rerank_documents.append(rerank_document)
 

api/core/model_runtime/model_providers/openai/llm/llm.py

@@ -111,7 +111,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel):
         stop: Optional[list[str]] = None,
         stream: bool = True,
         user: Optional[str] = None,
-        callbacks: Optional[list[Callback]] = None,
+        callbacks: list[Callback] = None,
     ) -> Union[LLMResult, Generator]:
         """
         Code block mode wrapper for invoking large language model

api/core/model_runtime/model_providers/openai/speech2text/speech2text.py

@@ -2,8 +2,6 @@ from typing import IO, Optional
 
 from openai import OpenAI
 
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel
 from core.model_runtime.model_providers.openai._common import _CommonOpenAI
@@ -60,18 +58,3 @@ class OpenAISpeech2TextModel(_CommonOpenAI, Speech2TextModel):
         response = client.audio.transcriptions.create(model=model, file=file)
 
         return response.text
-
-    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None:
-        """
-        used to define customizable model schema
-        """
-        entity = AIModelEntity(
-            model=model,
-            label=I18nObject(en_US=model),
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_type=ModelType.SPEECH2TEXT,
-            model_properties={},
-            parameter_rules=[],
-        )
-
-        return entity

api/core/model_runtime/model_providers/openai/tts/tts.py

@@ -1,5 +1,5 @@
 import concurrent.futures
-from typing import Any, Optional
+from typing import Optional
 
 from openai import OpenAI
 
@@ -16,7 +16,7 @@ class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel):
 
     def _invoke(
         self, model: str, tenant_id: str, credentials: dict, content_text: str, voice: str, user: Optional[str] = None
-    ) -> Any:
+    ) -> any:
         """
         _invoke text2speech model
 
@@ -55,7 +55,7 @@ class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel):
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
 
-    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> Any:
+    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> any:
         """
         _tts_invoke_streaming text2speech model
 

api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py

@@ -688,7 +688,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOaiApiCompat, LargeLanguageModel):
         model: str,
         messages: list[PromptMessage],
         tools: Optional[list[PromptMessageTool]] = None,
-        credentials: Optional[dict] = None,
+        credentials: dict = None,
     ) -> int:
         """
         Approximate num tokens with GPT2 tokenizer.

api/core/model_runtime/model_providers/openai_api_compatible/speech2text/speech2text.py

@@ -3,8 +3,6 @@ from urllib.parse import urljoin
 
 import requests
 
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType
 from core.model_runtime.errors.invoke import InvokeBadRequestError
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel
@@ -61,18 +59,3 @@ class OAICompatSpeech2TextModel(_CommonOaiApiCompat, Speech2TextModel):
                 self._invoke(model, credentials, audio_file)
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
-
-    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None:
-        """
-        used to define customizable model schema
-        """
-        entity = AIModelEntity(
-            model=model,
-            label=I18nObject(en_US=model),
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_type=ModelType.SPEECH2TEXT,
-            model_properties={},
-            parameter_rules=[],
-        )
-
-        return entity

api/core/model_runtime/model_providers/openrouter/llm/_position.yaml

@@ -14,10 +14,6 @@
 - google/gemini-pro
 - cohere/command-r-plus
 - cohere/command-r
-- meta-llama/llama-3.2-1b-instruct
-- meta-llama/llama-3.2-3b-instruct
-- meta-llama/llama-3.2-11b-vision-instruct
-- meta-llama/llama-3.2-90b-vision-instruct
 - meta-llama/llama-3.1-405b-instruct
 - meta-llama/llama-3.1-70b-instruct
 - meta-llama/llama-3.1-8b-instruct
@@ -26,7 +22,6 @@
 - mistralai/mixtral-8x22b-instruct
 - mistralai/mixtral-8x7b-instruct
 - mistralai/mistral-7b-instruct
-- qwen/qwen-2.5-72b-instruct
 - qwen/qwen-2-72b-instruct
 - deepseek/deepseek-chat
 - deepseek/deepseek-coder

api/core/model_runtime/model_providers/openrouter/llm/claude-3-5-sonnet.yaml

@@ -27,9 +27,9 @@ parameter_rules:
   - name: max_tokens
     use_template: max_tokens
     required: true
-    default: 8192
+    default: 4096
     min: 1
-    max: 8192
+    max: 4096
   - name: response_format
     use_template: response_format
 pricing:

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-11b-vision-instruct.yaml

@@ -1,45 +0,0 @@
-model: meta-llama/llama-3.2-11b-vision-instruct
-label:
-  zh_Hans: llama-3.2-11b-vision-instruct
-  en_US: llama-3.2-11b-vision-instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 131072
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: top_p
-    use_template: top_p
-  - name: top_k
-    label:
-      zh_Hans: 取样数量
-      en_US: Top k
-    type: int
-    help:
-      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
-      en_US: Only sample from the top K options for each subsequent token.
-  - name: max_tokens
-    use_template: max_tokens
-  - name: context_length_exceeded_behavior
-    default: None
-    label:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    help:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    type: string
-    options:
-      - None
-      - truncate
-      - error
-  - name: response_format
-    use_template: response_format
-pricing:
-  input: '0.055'
-  output: '0.055'
-  unit: '0.000001'
-  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-1b-instruct.yaml

@@ -1,45 +0,0 @@
-model: meta-llama/llama-3.2-1b-instruct
-label:
-  zh_Hans: llama-3.2-1b-instruct
-  en_US: llama-3.2-1b-instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 131072
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: top_p
-    use_template: top_p
-  - name: top_k
-    label:
-      zh_Hans: 取样数量
-      en_US: Top k
-    type: int
-    help:
-      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
-      en_US: Only sample from the top K options for each subsequent token.
-  - name: max_tokens
-    use_template: max_tokens
-  - name: context_length_exceeded_behavior
-    default: None
-    label:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    help:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    type: string
-    options:
-      - None
-      - truncate
-      - error
-  - name: response_format
-    use_template: response_format
-pricing:
-  input: '0.01'
-  output: '0.02'
-  unit: '0.000001'
-  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-3b-instruct.yaml

@@ -1,45 +0,0 @@
-model: meta-llama/llama-3.2-3b-instruct
-label:
-  zh_Hans: llama-3.2-3b-instruct
-  en_US: llama-3.2-3b-instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 131072
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: top_p
-    use_template: top_p
-  - name: top_k
-    label:
-      zh_Hans: 取样数量
-      en_US: Top k
-    type: int
-    help:
-      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
-      en_US: Only sample from the top K options for each subsequent token.
-  - name: max_tokens
-    use_template: max_tokens
-  - name: context_length_exceeded_behavior
-    default: None
-    label:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    help:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    type: string
-    options:
-      - None
-      - truncate
-      - error
-  - name: response_format
-    use_template: response_format
-pricing:
-  input: '0.03'
-  output: '0.05'
-  unit: '0.000001'
-  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/llama-3.2-90b-vision-instruct.yaml

@@ -1,45 +0,0 @@
-model: meta-llama/llama-3.2-90b-vision-instruct
-label:
-  zh_Hans: llama-3.2-90b-vision-instruct
-  en_US: llama-3.2-90b-vision-instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 131072
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: top_p
-    use_template: top_p
-  - name: top_k
-    label:
-      zh_Hans: 取样数量
-      en_US: Top k
-    type: int
-    help:
-      zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
-      en_US: Only sample from the top K options for each subsequent token.
-  - name: max_tokens
-    use_template: max_tokens
-  - name: context_length_exceeded_behavior
-    default: None
-    label:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    help:
-      zh_Hans: 上下文长度超出行为
-      en_US: Context Length Exceeded Behavior
-    type: string
-    options:
-      - None
-      - truncate
-      - error
-  - name: response_format
-    use_template: response_format
-pricing:
-  input: '0.35'
-  output: '0.4'
-  unit: '0.000001'
-  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/qwen2.5-72b-instruct.yaml

@@ -1,30 +0,0 @@
-model: qwen/qwen-2.5-72b-instruct
-label:
-  en_US: qwen-2.5-72b-instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 131072
-parameter_rules:
-  - name: temperature
-    use_template: temperature
-  - name: max_tokens
-    use_template: max_tokens
-    type: int
-    default: 512
-    min: 1
-    max: 8192
-    help:
-      zh_Hans: 指定生成结果长度的上限。如果生成结果截断，可以调大该参数。
-      en_US: Specifies the upper limit on the length of generated results. If the generated results are truncated, you can increase this parameter.
-  - name: top_p
-    use_template: top_p
-  - name: frequency_penalty
-    use_template: frequency_penalty
-pricing:
-  input: "0.35"
-  output: "0.4"
-  unit: "0.000001"
-  currency: USD

api/core/model_runtime/model_providers/sagemaker/tts/tts.py

@@ -77,7 +77,7 @@ class SageMakerText2SpeechModel(TTSModel):
         """
         pass
 
-    def _detect_lang_code(self, content: str, map_dict: Optional[dict] = None):
+    def _detect_lang_code(self, content: str, map_dict: dict = None):
         map_dict = {"zh": "<|zh|>", "en": "<|en|>", "ja": "<|jp|>", "zh-TW": "<|yue|>", "ko": "<|ko|>"}
 
         response = self.comprehend_client.detect_dominant_language(Text=content)
@@ -192,7 +192,7 @@ class SageMakerText2SpeechModel(TTSModel):
             InvokeBadRequestError: [InvokeBadRequestError, KeyError, ValueError],
         }
 
-    def _get_model_default_voice(self, model: str, credentials: dict) -> Any:
+    def _get_model_default_voice(self, model: str, credentials: dict) -> any:
         return ""
 
     def _get_model_word_limit(self, model: str, credentials: dict) -> int:
@@ -225,7 +225,7 @@ class SageMakerText2SpeechModel(TTSModel):
         json_obj = json.loads(json_str)
         return json_obj
 
-    def _tts_invoke_streaming(self, model_type: str, payload: dict, sagemaker_endpoint: str) -> Any:
+    def _tts_invoke_streaming(self, model_type: str, payload: dict, sagemaker_endpoint: str) -> any:
         """
         _tts_invoke_streaming text2speech model
 

api/core/model_runtime/model_providers/siliconflow/llm/llm.py

@@ -1,18 +1,8 @@
 from collections.abc import Generator
 from typing import Optional, Union
 
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.llm_entities import LLMMode, LLMResult
+from core.model_runtime.entities.llm_entities import LLMResult
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool
-from core.model_runtime.entities.model_entities import (
-    AIModelEntity,
-    FetchFrom,
-    ModelFeature,
-    ModelPropertyKey,
-    ModelType,
-    ParameterRule,
-    ParameterType,
-)
 from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel
 
 
@@ -39,53 +29,3 @@ class SiliconflowLargeLanguageModel(OAIAPICompatLargeLanguageModel):
     def _add_custom_parameters(cls, credentials: dict) -> None:
         credentials["mode"] = "chat"
         credentials["endpoint_url"] = "https://api.siliconflow.cn/v1"
-
-    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity | None:
-        return AIModelEntity(
-            model=model,
-            label=I18nObject(en_US=model, zh_Hans=model),
-            model_type=ModelType.LLM,
-            features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL, ModelFeature.STREAM_TOOL_CALL]
-            if credentials.get("function_calling_type") == "tool_call"
-            else [],
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={
-                ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size", 8000)),
-                ModelPropertyKey.MODE: LLMMode.CHAT.value,
-            },
-            parameter_rules=[
-                ParameterRule(
-                    name="temperature",
-                    use_template="temperature",
-                    label=I18nObject(en_US="Temperature", zh_Hans="温度"),
-                    type=ParameterType.FLOAT,
-                ),
-                ParameterRule(
-                    name="max_tokens",
-                    use_template="max_tokens",
-                    default=512,
-                    min=1,
-                    max=int(credentials.get("max_tokens", 1024)),
-                    label=I18nObject(en_US="Max Tokens", zh_Hans="最大标记"),
-                    type=ParameterType.INT,
-                ),
-                ParameterRule(
-                    name="top_p",
-                    use_template="top_p",
-                    label=I18nObject(en_US="Top P", zh_Hans="Top P"),
-                    type=ParameterType.FLOAT,
-                ),
-                ParameterRule(
-                    name="top_k",
-                    use_template="top_k",
-                    label=I18nObject(en_US="Top K", zh_Hans="Top K"),
-                    type=ParameterType.FLOAT,
-                ),
-                ParameterRule(
-                    name="frequency_penalty",
-                    use_template="frequency_penalty",
-                    label=I18nObject(en_US="Frequency Penalty", zh_Hans="重复惩罚"),
-                    type=ParameterType.FLOAT,
-                ),
-            ],
-        )

api/core/model_runtime/model_providers/siliconflow/siliconflow.yaml

@@ -20,7 +20,6 @@ supported_model_types:
   - speech2text
 configurate_methods:
   - predefined-model
-  - customizable-model
 provider_credential_schema:
   credential_form_schemas:
     - variable: api_key
@@ -31,57 +30,3 @@ provider_credential_schema:
       placeholder:
         zh_Hans: 在此输入您的 API Key
         en_US: Enter your API Key
-model_credential_schema:
-  model:
-    label:
-      en_US: Model Name
-      zh_Hans: 模型名称
-    placeholder:
-      en_US: Enter your model name
-      zh_Hans: 输入模型名称
-  credential_form_schemas:
-    - variable: api_key
-      label:
-        en_US: API Key
-      type: secret-input
-      required: true
-      placeholder:
-        zh_Hans: 在此输入您的 API Key
-        en_US: Enter your API Key
-    - variable: context_size
-      label:
-        zh_Hans: 模型上下文长度
-        en_US: Model context size
-      required: true
-      type: text-input
-      default: '4096'
-      placeholder:
-        zh_Hans: 在此输入您的模型上下文长度
-        en_US: Enter your Model context size
-    - variable: max_tokens
-      label:
-        zh_Hans: 最大 token 上限
-        en_US: Upper bound for max tokens
-      default: '4096'
-      type: text-input
-      show_on:
-        - variable: __model_type
-          value: llm
-    - variable: function_calling_type
-      label:
-        en_US: Function calling
-      type: select
-      required: false
-      default: no_call
-      options:
-        - value: no_call
-          label:
-            en_US: Not Support
-            zh_Hans: 不支持
-        - value: function_call
-          label:
-            en_US: Support
-            zh_Hans: 支持
-      show_on:
-        - variable: __model_type
-          value: llm

api/core/model_runtime/model_providers/tongyi/rerank/_position.yaml

@@ -1 +0,0 @@
-- gte-rerank

api/core/model_runtime/model_providers/tongyi/rerank/gte-rerank.yaml

@@ -1,4 +0,0 @@
-model: gte-rerank
-model_type: rerank
-model_properties:
-  context_size: 4000

api/core/model_runtime/model_providers/tongyi/rerank/rerank.py

@@ -1,136 +0,0 @@
-from typing import Optional
-
-import dashscope
-from dashscope.common.error import (
-    AuthenticationError,
-    InvalidParameter,
-    RequestFailure,
-    ServiceUnavailableError,
-    UnsupportedHTTPMethod,
-    UnsupportedModel,
-)
-
-from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult
-from core.model_runtime.errors.invoke import (
-    InvokeAuthorizationError,
-    InvokeBadRequestError,
-    InvokeConnectionError,
-    InvokeError,
-    InvokeRateLimitError,
-    InvokeServerUnavailableError,
-)
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.__base.rerank_model import RerankModel
-
-
-class GTERerankModel(RerankModel):
-    """
-    Model class for GTE rerank model.
-    """
-
-    def _invoke(
-        self,
-        model: str,
-        credentials: dict,
-        query: str,
-        docs: list[str],
-        score_threshold: Optional[float] = None,
-        top_n: Optional[int] = None,
-        user: Optional[str] = None,
-    ) -> RerankResult:
-        """
-        Invoke rerank model
-
-        :param model: model name
-        :param credentials: model credentials
-        :param query: search query
-        :param docs: docs for reranking
-        :param score_threshold: score threshold
-        :param top_n: top n
-        :param user: unique user id
-        :return: rerank result
-        """
-        if len(docs) == 0:
-            return RerankResult(model=model, docs=docs)
-
-        # initialize client
-        dashscope.api_key = credentials["dashscope_api_key"]
-
-        response = dashscope.TextReRank.call(
-            query=query,
-            documents=docs,
-            model=model,
-            top_n=top_n,
-            return_documents=True,
-        )
-
-        rerank_documents = []
-        for _, result in enumerate(response.output.results):
-            # format document
-            rerank_document = RerankDocument(
-                index=result.index,
-                score=result.relevance_score,
-                text=result["document"]["text"],
-            )
-
-            # score threshold check
-            if score_threshold is not None:
-                if result.relevance_score >= score_threshold:
-                    rerank_documents.append(rerank_document)
-            else:
-                rerank_documents.append(rerank_document)
-
-        return RerankResult(model=model, docs=rerank_documents)
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        """
-        Validate model credentials
-
-        :param model: model name
-        :param credentials: model credentials
-        :return:
-        """
-        try:
-            self.invoke(
-                model=model,
-                credentials=credentials,
-                query="What is the capital of the United States?",
-                docs=[
-                    "Carson City is the capital city of the American state of Nevada. At the 2010 United States "
-                    "Census, Carson City had a population of 55,274.",
-                    "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that "
-                    "are a political division controlled by the United States. Its capital is Saipan.",
-                ],
-                score_threshold=0.8,
-            )
-        except Exception as ex:
-            print(ex)
-            raise CredentialsValidateFailedError(str(ex))
-
-    @property
-    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
-        """
-        Map model invoke error to unified error
-        The key is the error type thrown to the caller
-        The value is the error type thrown by the model,
-        which needs to be converted into a unified error type for the caller.
-
-        :return: Invoke error mapping
-        """
-        return {
-            InvokeConnectionError: [
-                RequestFailure,
-            ],
-            InvokeServerUnavailableError: [
-                ServiceUnavailableError,
-            ],
-            InvokeRateLimitError: [],
-            InvokeAuthorizationError: [
-                AuthenticationError,
-            ],
-            InvokeBadRequestError: [
-                InvalidParameter,
-                UnsupportedModel,
-                UnsupportedHTTPMethod,
-            ],
-        }

api/core/model_runtime/model_providers/tongyi/tongyi.yaml

@@ -18,7 +18,6 @@ supported_model_types:
   - llm
   - tts
   - text-embedding
-  - rerank
 configurate_methods:
   - predefined-model
   - customizable-model

api/core/model_runtime/model_providers/tongyi/tts/tts.py

@@ -1,6 +1,6 @@
 import threading
 from queue import Queue
-from typing import Any, Optional
+from typing import Optional
 
 import dashscope
 from dashscope import SpeechSynthesizer
@@ -20,7 +20,7 @@ class TongyiText2SpeechModel(_CommonTongyi, TTSModel):
 
     def _invoke(
         self, model: str, tenant_id: str, credentials: dict, content_text: str, voice: str, user: Optional[str] = None
-    ) -> Any:
+    ) -> any:
         """
         _invoke text2speech model
 
@@ -58,7 +58,7 @@ class TongyiText2SpeechModel(_CommonTongyi, TTSModel):
         except Exception as ex:
             raise CredentialsValidateFailedError(str(ex))
 
-    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> Any:
+    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> any:
         """
         _tts_invoke_streaming text2speech model
 

api/core/model_runtime/model_providers/vertex_ai/llm/llm.py

@@ -7,7 +7,6 @@ from collections.abc import Generator
 from typing import Optional, Union, cast
 
 import google.auth.transport.requests
-import requests
 import vertexai.generative_models as glm
 from anthropic import AnthropicVertex, Stream
 from anthropic.types import (
@@ -654,15 +653,9 @@ class VertexAiLargeLanguageModel(LargeLanguageModel):
                     if c.type == PromptMessageContentType.TEXT:
                         parts.append(glm.Part.from_text(c.data))
                     else:
-                        message_content = cast(ImagePromptMessageContent, c)
-                        if not message_content.data.startswith("data:"):
-                            url_arr = message_content.data.split(".")
-                            mime_type = f"image/{url_arr[-1]}"
-                            parts.append(glm.Part.from_uri(mime_type=mime_type, uri=message_content.data))
-                        else:
-                            metadata, data = c.data.split(",", 1)
-                            mime_type = metadata.split(";", 1)[0].split(":")[1]
-                            parts.append(glm.Part.from_data(mime_type=mime_type, data=data))
+                        metadata, data = c.data.split(",", 1)
+                        mime_type = metadata.split(";", 1)[0].split(":")[1]
+                        parts.append(glm.Part.from_data(mime_type=mime_type, data=data))
                 glm_content = glm.Content(role="user", parts=parts)
             return glm_content
         elif isinstance(message, AssistantPromptMessage):

api/core/model_runtime/model_providers/voyage/rerank/_position.yaml

@@ -1,4 +0,0 @@
-- rerank-2
-- rerank-lite-2
-- rerank-1
-- rerank-lite-1

api/core/model_runtime/model_providers/voyage/rerank/rerank-2.yaml

@@ -1,4 +0,0 @@
-model: rerank-2
-model_type: rerank
-model_properties:
-  context_size: 16000

api/core/model_runtime/model_providers/voyage/rerank/rerank-lite-2.yaml

@@ -1,4 +0,0 @@
-model: rerank-lite-2
-model_type: rerank
-model_properties:
-  context_size: 8000

api/core/model_runtime/model_providers/voyage/text_embedding/_position.yaml

@@ -1,6 +0,0 @@
-- voyage-3
-- voyage-3-lite
-- voyage-finance-2
-- voyage-multilingual-2
-- voyage-law-2
-- voyage-code-2