feat(web): refine onboarding UI (#37433)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: yyh <yuanyouhuilyz@gmail.com>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: hjlarry <hjlarry@163.com>
Co-authored-by: fatelei <fatelei@gmail.com>
Co-authored-by: Asuka Minato <i@asukaminato.eu.org>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Xiyuan Chen <52963600+GareArc@users.noreply.github.com>
Co-authored-by: gigglewang <gigglewang@dify.ai>
Co-authored-by: Yunlu Wen <yunlu.wen@dify.ai>
Co-authored-by: chariri <w@chariri.moe>
Co-authored-by: Evan <2869018789@qq.com>
Co-authored-by: yyh <92089059+lyzno1@users.noreply.github.com>

api/controllers/console/workspace/plugin.py

@@ -1,10 +1,11 @@
 import io
 from collections.abc import Mapping
-from typing import Any, Literal
+from datetime import datetime
+from typing import Any, Literal, TypedDict
 
 from flask import request, send_file
 from flask_restx import Resource
-from pydantic import BaseModel, Field, RootModel
+from pydantic import BaseModel, ConfigDict, Field, RootModel
 from werkzeug.datastructures import FileStorage
 from werkzeug.exceptions import Forbidden
 
@@ -26,15 +27,33 @@ from controllers.console.wraps import (
     with_current_user,
     with_current_user_id,
 )
+from core.helper.position_helper import is_filtered
+from core.plugin.entities.plugin import PluginCategory, PluginInstallationSource
 from core.plugin.impl.exc import PluginDaemonClientSideError
 from core.plugin.plugin_service import PluginService
+from core.tools.builtin_tool.providers._positions import BuiltinToolProviderSort
+from core.tools.entities.common_entities import I18nObject
+from core.tools.entities.tool_entities import ToolProviderType
+from core.tools.tool_manager import ToolManager
 from fields.base import ResponseModel
 from graphon.model_runtime.utils.encoders import jsonable_encoder
+from libs.helper import dump_response
 from libs.login import login_required
 from models.account import Account, TenantPluginAutoUpgradeStrategy, TenantPluginPermission
+from models.provider_ids import ToolProviderID
+from services.entities.model_provider_entities import ProviderEntityResponse
 from services.plugin.plugin_auto_upgrade_service import PluginAutoUpgradeService
 from services.plugin.plugin_parameter_service import PluginParameterService
 from services.plugin.plugin_permission_service import PluginPermissionService
+from services.tools.tools_transform_service import ToolTransformService
+
+
+class AutoUpgradeSettingsResponse(TypedDict):
+    strategy_setting: TenantPluginAutoUpgradeStrategy.StrategySetting
+    upgrade_time_of_day: int
+    upgrade_mode: TenantPluginAutoUpgradeStrategy.UpgradeMode
+    exclude_plugins: list[str]
+    include_plugins: list[str]
 
 
 class ParserList(BaseModel):
@@ -42,6 +61,11 @@ class ParserList(BaseModel):
     page_size: int = Field(default=256, ge=1, le=256, description="Page size (1-256)")
 
 
+class PluginCategoryListQuery(BaseModel):
+    page: int = Field(default=1, ge=1, description="Page number")
+    page_size: int = Field(default=256, ge=1, le=256, description="Page size (1-256)")
+
+
 class ParserLatest(BaseModel):
     plugin_ids: list[str]
 
@@ -100,8 +124,8 @@ class ParserUninstall(BaseModel):
 
 
 class ParserPermissionChange(BaseModel):
-    install_permission: TenantPluginPermission.InstallPermission
-    debug_permission: TenantPluginPermission.DebugPermission
+    install_permission: TenantPluginPermission.InstallPermission = TenantPluginPermission.InstallPermission.EVERYONE
+    debug_permission: TenantPluginPermission.DebugPermission = TenantPluginPermission.DebugPermission.EVERYONE
 
 
 class ParserDynamicOptions(BaseModel):
@@ -137,13 +161,64 @@ class PluginAutoUpgradeSettingsPayload(BaseModel):
     include_plugins: list[str] = Field(default_factory=list)
 
 
-class ParserPreferencesChange(BaseModel):
-    permission: PluginPermissionSettingsPayload
+class PluginAutoUpgradeChangeResponse(ResponseModel):
+    success: bool
+    message: str | None = None
+
+
+class PluginAutoUpgradeSettingsResponseModel(ResponseModel):
+    strategy_setting: TenantPluginAutoUpgradeStrategy.StrategySetting
+    upgrade_time_of_day: int
+    upgrade_mode: TenantPluginAutoUpgradeStrategy.UpgradeMode
+    exclude_plugins: list[str]
+    include_plugins: list[str]
+
+
+class PluginAutoUpgradeFetchResponse(ResponseModel):
+    category: TenantPluginAutoUpgradeStrategy.PluginCategory
+    auto_upgrade: PluginAutoUpgradeSettingsResponseModel
+
+
+class PluginDeclarationResponse(ResponseModel):
+    version: str
+    author: str | None
+    name: str
+    description: I18nObject
+    icon: str
+    icon_dark: str | None = None
+    label: I18nObject
+    category: PluginCategory
+    created_at: datetime
+    resource: Mapping[str, Any]
+    plugins: Mapping[str, list[str] | None]
+    tags: list[str] = Field(default_factory=list)
+    repo: str | None = None
+    verified: bool = False
+    tool: Mapping[str, Any] | None = None
+    model: ProviderEntityResponse | None = None
+    endpoint: Mapping[str, Any] | None = None
+    agent_strategy: Mapping[str, Any] | None = None
+    datasource: Mapping[str, Any] | None = None
+    trigger: Mapping[str, Any] | None = None
+    meta: Mapping[str, Any]
+
+
+class ParserAutoUpgradeChange(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    category: TenantPluginAutoUpgradeStrategy.PluginCategory
     auto_upgrade: PluginAutoUpgradeSettingsPayload
 
 
+class ParserAutoUpgradeFetch(BaseModel):
+    category: TenantPluginAutoUpgradeStrategy.PluginCategory
+
+
 class ParserExcludePlugin(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
     plugin_id: str
+    category: TenantPluginAutoUpgradeStrategy.PluginCategory
 
 
 class ParserReadme(BaseModel):
@@ -157,6 +232,63 @@ class PluginDebuggingKeyResponse(ResponseModel):
     port: int
 
 
+class PluginCategoryInstalledPluginResponse(ResponseModel):
+    id: str
+    name: str
+    tenant_id: str
+    plugin_id: str
+    plugin_unique_identifier: str
+    endpoints_active: int
+    endpoints_setups: int
+    installation_id: str
+    declaration: PluginDeclarationResponse
+    runtime_type: str
+    version: str
+    created_at: datetime
+    updated_at: datetime
+    source: PluginInstallationSource
+    checksum: str
+    meta: Mapping[str, Any]
+
+
+class PluginCategoryBuiltinToolResponse(ResponseModel):
+    model_config = ConfigDict(extra="allow")
+
+    author: str
+    name: str
+    label: I18nObject
+    description: I18nObject
+    parameters: list[Mapping[str, Any]] | None = None
+    labels: list[str]
+    output_schema: Mapping[str, object]
+
+
+class PluginCategoryBuiltinToolProviderResponse(ResponseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str
+    author: str
+    name: str
+    plugin_id: str | None
+    plugin_unique_identifier: str | None
+    description: I18nObject
+    icon: str | Mapping[str, str]
+    icon_dark: str | Mapping[str, str] | None
+    label: I18nObject
+    type: ToolProviderType
+    team_credentials: Mapping[str, object]
+    is_team_authorization: bool
+    allow_delete: bool
+    tools: list[PluginCategoryBuiltinToolResponse]
+    labels: list[str]
+
+
+class PluginCategoryListResponse(ResponseModel):
+    plugins: list[PluginCategoryInstalledPluginResponse]
+    builtin_tools: list[PluginCategoryBuiltinToolProviderResponse]
+    has_more: bool
+
+
 class PluginDaemonOperationResponse(RootModel[Any]):
     root: Any
 
@@ -200,11 +332,6 @@ class PluginOperationSuccessResponse(ResponseModel):
     message: str | None = None
 
 
-class PluginPreferencesResponse(ResponseModel):
-    permission: PluginPermissionSettingsPayload
-    auto_upgrade: PluginAutoUpgradeSettingsPayload
-
-
 class PluginReadmeResponse(ResponseModel):
     readme: str
 
@@ -212,6 +339,7 @@ class PluginReadmeResponse(ResponseModel):
 register_schema_models(
     console_ns,
     ParserList,
+    PluginCategoryListQuery,
     PluginAutoUpgradeSettingsPayload,
     PluginPermissionSettingsPayload,
     ParserLatest,
@@ -228,13 +356,21 @@ register_schema_models(
     ParserPermissionChange,
     ParserDynamicOptions,
     ParserDynamicOptionsWithCredentials,
-    ParserPreferencesChange,
+    ParserAutoUpgradeChange,
+    ParserAutoUpgradeFetch,
     ParserExcludePlugin,
     ParserReadme,
 )
 register_response_schema_models(
     console_ns,
+    PluginAutoUpgradeChangeResponse,
+    PluginAutoUpgradeFetchResponse,
+    PluginAutoUpgradeSettingsResponseModel,
     BinaryFileResponse,
+    PluginCategoryBuiltinToolProviderResponse,
+    PluginCategoryBuiltinToolResponse,
+    PluginCategoryInstalledPluginResponse,
+    PluginCategoryListResponse,
     PluginDaemonOperationResponse,
     PluginDebuggingKeyResponse,
     PluginDynamicOptionsResponse,
@@ -243,7 +379,6 @@ register_response_schema_models(
     PluginManifestResponse,
     PluginOperationSuccessResponse,
     PluginPermissionResponse,
-    PluginPreferencesResponse,
     PluginReadmeResponse,
     PluginTaskResponse,
     PluginTasksResponse,
@@ -254,12 +389,36 @@ register_response_schema_models(
 register_enum_models(
     console_ns,
     TenantPluginPermission.DebugPermission,
+    TenantPluginAutoUpgradeStrategy.PluginCategory,
     TenantPluginAutoUpgradeStrategy.UpgradeMode,
     TenantPluginAutoUpgradeStrategy.StrategySetting,
     TenantPluginPermission.InstallPermission,
 )
 
 
+def _default_auto_upgrade_settings(
+    tenant_id: str,
+    category: TenantPluginAutoUpgradeStrategy.PluginCategory,
+) -> AutoUpgradeSettingsResponse:
+    return {
+        "strategy_setting": PluginAutoUpgradeService.default_strategy_setting_for_category(category),
+        "upgrade_time_of_day": PluginAutoUpgradeService.default_upgrade_time_of_day(tenant_id),
+        "upgrade_mode": TenantPluginAutoUpgradeStrategy.UpgradeMode.EXCLUDE,
+        "exclude_plugins": [],
+        "include_plugins": [],
+    }
+
+
+def _auto_upgrade_settings_to_dict(strategy: TenantPluginAutoUpgradeStrategy) -> AutoUpgradeSettingsResponse:
+    return {
+        "strategy_setting": strategy.strategy_setting,
+        "upgrade_time_of_day": strategy.upgrade_time_of_day,
+        "upgrade_mode": strategy.upgrade_mode,
+        "exclude_plugins": strategy.exclude_plugins,
+        "include_plugins": strategy.include_plugins,
+    }
+
+
 def _read_upload_content(file: FileStorage, max_size: int) -> bytes:
     """
     Read the uploaded file and validate its actual size before delegating to the plugin service.
@@ -274,6 +433,33 @@ def _read_upload_content(file: FileStorage, max_size: int) -> bytes:
     return content
 
 
+def _list_hardcoded_builtin_tool_providers(tenant_id: str) -> list[dict[str, Any]]:
+    db_builtin_providers = {
+        str(ToolProviderID(provider.provider)): provider
+        for provider in ToolManager.list_default_builtin_providers(tenant_id)
+    }
+    builtin_providers = []
+
+    for provider in ToolManager.list_hardcoded_providers():
+        if is_filtered(
+            include_set=dify_config.POSITION_TOOL_INCLUDES_SET,
+            exclude_set=dify_config.POSITION_TOOL_EXCLUDES_SET,
+            data=provider,
+            name_func=lambda provider_controller: provider_controller.entity.identity.name,
+        ):
+            continue
+
+        user_provider = ToolTransformService.builtin_provider_to_user_provider(
+            provider_controller=provider,
+            db_provider=db_builtin_providers.get(provider.entity.identity.name),
+            decrypt_credentials=False,
+        )
+        ToolTransformService.repack_provider(tenant_id=tenant_id, provider=user_provider)
+        builtin_providers.append(user_provider)
+
+    return [provider.to_dict() for provider in BuiltinToolProviderSort.sort(builtin_providers)]
+
+
 @console_ns.route("/workspaces/current/plugin/debugging-key")
 class PluginDebuggingKeyApi(Resource):
     @console_ns.response(200, "Success", console_ns.models[PluginDebuggingKeyResponse.__name__])
@@ -312,6 +498,41 @@ class PluginListApi(Resource):
         return jsonable_encoder({"plugins": plugins_with_total.list, "total": plugins_with_total.total})
 
 
+@console_ns.route("/workspaces/current/plugin/<string:category>/list")
+class PluginCategoryListApi(Resource):
+    @console_ns.doc(params=query_params_from_model(PluginCategoryListQuery))
+    @console_ns.response(200, "Success", console_ns.models[PluginCategoryListResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @with_current_tenant_id
+    def get(self, tenant_id: str, category: str):
+        args = PluginCategoryListQuery.model_validate(request.args.to_dict(flat=True))
+
+        try:
+            plugin_category = PluginCategory(category)
+        except ValueError:
+            return {"code": "invalid_param", "message": "invalid plugin category"}, 400
+
+        try:
+            plugins = PluginService.list_by_category(tenant_id, plugin_category, args.page, args.page_size)
+        except PluginDaemonClientSideError as e:
+            return {"code": "plugin_error", "message": e.description}, 400
+
+        builtin_tools = []
+        if plugin_category == PluginCategory.Tool:
+            builtin_tools = _list_hardcoded_builtin_tool_providers(tenant_id)
+
+        return dump_response(
+            PluginCategoryListResponse,
+            {
+                "plugins": jsonable_encoder(plugins.list),
+                "builtin_tools": builtin_tools,
+                "has_more": plugins.has_more,
+            },
+        )
+
+
 @console_ns.route("/workspaces/current/plugin/list/latest-versions")
 class PluginListLatestVersionsApi(Resource):
     @console_ns.expect(console_ns.models[ParserLatest.__name__])
@@ -713,11 +934,13 @@ class PluginChangePermissionApi(Resource):
 
         args = ParserPermissionChange.model_validate(console_ns.payload)
 
-        return {
-            "success": PluginPermissionService.change_permission(
-                tenant_id, args.install_permission, args.debug_permission
-            )
-        }
+        set_permission_result = PluginPermissionService.change_permission(
+            tenant_id, args.install_permission, args.debug_permission
+        )
+        if not set_permission_result:
+            return jsonable_encoder({"success": False, "message": "Failed to set permission"})
+
+        return jsonable_encoder({"success": True})
 
 
 @console_ns.route("/workspaces/current/plugin/permission/fetch")
@@ -806,10 +1029,10 @@ class PluginFetchDynamicSelectOptionsWithCredentialsApi(Resource):
         return jsonable_encoder({"options": options})
 
 
-@console_ns.route("/workspaces/current/plugin/preferences/change")
-class PluginChangePreferencesApi(Resource):
-    @console_ns.expect(console_ns.models[ParserPreferencesChange.__name__])
-    @console_ns.response(200, "Success", console_ns.models[PluginOperationSuccessResponse.__name__])
+@console_ns.route("/workspaces/current/plugin/auto-upgrade/change")
+class PluginChangeAutoUpgradeApi(Resource):
+    @console_ns.expect(console_ns.models[ParserAutoUpgradeChange.__name__])
+    @console_ns.response(200, "Success", console_ns.models[PluginAutoUpgradeChangeResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -819,38 +1042,17 @@ class PluginChangePreferencesApi(Resource):
         if not user.is_admin_or_owner:
             raise Forbidden()
 
-        args = ParserPreferencesChange.model_validate(console_ns.payload)
-
-        permission = args.permission
-
-        install_permission = permission.install_permission
-        debug_permission = permission.debug_permission
+        args = ParserAutoUpgradeChange.model_validate(console_ns.payload)
 
         auto_upgrade = args.auto_upgrade
-
-        strategy_setting = auto_upgrade.strategy_setting
-        upgrade_time_of_day = auto_upgrade.upgrade_time_of_day
-        upgrade_mode = auto_upgrade.upgrade_mode
-        exclude_plugins = auto_upgrade.exclude_plugins
-        include_plugins = auto_upgrade.include_plugins
-
-        # set permission
-        set_permission_result = PluginPermissionService.change_permission(
-            tenant_id,
-            install_permission,
-            debug_permission,
-        )
-        if not set_permission_result:
-            return jsonable_encoder({"success": False, "message": "Failed to set permission"})
-
-        # set auto upgrade strategy
         set_auto_upgrade_strategy_result = PluginAutoUpgradeService.change_strategy(
             tenant_id,
-            strategy_setting,
-            upgrade_time_of_day,
-            upgrade_mode,
-            exclude_plugins,
-            include_plugins,
+            auto_upgrade.strategy_setting,
+            auto_upgrade.upgrade_time_of_day,
+            auto_upgrade.upgrade_mode,
+            auto_upgrade.exclude_plugins,
+            auto_upgrade.include_plugins,
+            category=args.category,
         )
         if not set_auto_upgrade_strategy_result:
             return jsonable_encoder({"success": False, "message": "Failed to set auto upgrade strategy"})
@@ -858,49 +1060,35 @@ class PluginChangePreferencesApi(Resource):
         return jsonable_encoder({"success": True})
 
 
-@console_ns.route("/workspaces/current/plugin/preferences/fetch")
-class PluginFetchPreferencesApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[PluginPreferencesResponse.__name__])
+@console_ns.route("/workspaces/current/plugin/auto-upgrade/fetch")
+class PluginFetchAutoUpgradeApi(Resource):
+    @console_ns.doc(params=query_params_from_model(ParserAutoUpgradeFetch))
+    @console_ns.response(200, "Success", console_ns.models[PluginAutoUpgradeFetchResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str):
-        permission = PluginPermissionService.get_permission(tenant_id)
-        permission_dict = {
-            "install_permission": TenantPluginPermission.InstallPermission.EVERYONE,
-            "debug_permission": TenantPluginPermission.DebugPermission.EVERYONE,
-        }
+        args = ParserAutoUpgradeFetch.model_validate(request.args.to_dict(flat=True))
+        auto_upgrade = PluginAutoUpgradeService.get_strategy(tenant_id, args.category)
+        auto_upgrade_dict = (
+            _auto_upgrade_settings_to_dict(auto_upgrade)
+            if auto_upgrade
+            else _default_auto_upgrade_settings(tenant_id, args.category)
+        )
 
-        if permission:
-            permission_dict["install_permission"] = permission.install_permission
-            permission_dict["debug_permission"] = permission.debug_permission
-
-        auto_upgrade = PluginAutoUpgradeService.get_strategy(tenant_id)
-        auto_upgrade_dict = {
-            "strategy_setting": TenantPluginAutoUpgradeStrategy.StrategySetting.DISABLED,
-            "upgrade_time_of_day": 0,
-            "upgrade_mode": TenantPluginAutoUpgradeStrategy.UpgradeMode.EXCLUDE,
-            "exclude_plugins": [],
-            "include_plugins": [],
-        }
-
-        if auto_upgrade:
-            auto_upgrade_dict = {
-                "strategy_setting": auto_upgrade.strategy_setting,
-                "upgrade_time_of_day": auto_upgrade.upgrade_time_of_day,
-                "upgrade_mode": auto_upgrade.upgrade_mode,
-                "exclude_plugins": auto_upgrade.exclude_plugins,
-                "include_plugins": auto_upgrade.include_plugins,
+        return jsonable_encoder(
+            {
+                "category": args.category,
+                "auto_upgrade": auto_upgrade_dict,
             }
-
-        return jsonable_encoder({"permission": permission_dict, "auto_upgrade": auto_upgrade_dict})
+        )
 
 
-@console_ns.route("/workspaces/current/plugin/preferences/autoupgrade/exclude")
+@console_ns.route("/workspaces/current/plugin/auto-upgrade/exclude")
 class PluginAutoUpgradeExcludePluginApi(Resource):
     @console_ns.expect(console_ns.models[ParserExcludePlugin.__name__])
-    @console_ns.response(200, "Success", console_ns.models[PluginOperationSuccessResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[SuccessResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -909,7 +1097,9 @@ class PluginAutoUpgradeExcludePluginApi(Resource):
         # exclude one single plugin
         args = ParserExcludePlugin.model_validate(console_ns.payload)
 
-        return jsonable_encoder({"success": PluginAutoUpgradeService.exclude_plugin(tenant_id, args.plugin_id)})
+        return jsonable_encoder(
+            {"success": PluginAutoUpgradeService.exclude_plugin(tenant_id, args.plugin_id, args.category)}
+        )
 
 
 @console_ns.route("/workspaces/current/plugin/readme")