mirror of
https://github.com/langgenius/dify.git
synced 2026-11-10 21:35:45 +08:00
feat(knowledge-fs): add GitHub tree preview in connected sources
(cherry picked from commit 8e077ab5a3)
This commit is contained in:
FFXN committed
1 parent
a4d64bcddd
commit
889fac57d0
14 files changed
+541
-12
No files matched your search
@@ -3149,6 +3149,8 @@ class KnowledgeFSSpaceSourcePagesApi(Resource):
|
||||
source_id=source_id,
|
||||
cursor=query.cursor,
|
||||
limit=query.limit,
|
||||
parent_id=query.parent_id,
|
||||
view=query.view,
|
||||
)
|
||||
return dump_response(KnowledgeFSSourcePagesResponse, result)
|
||||
|
||||
|
||||
Generated
+5
@@ -10643,6 +10643,8 @@ Claim a workspace-staged upload. Multipart file bodies remain accepted as a lega
|
||||
| ---- | ---------- | ----------- | -------- | ------ |
|
||||
| cursor | query | | No | string |
|
||||
| limit | query | | No | integer, <br>**Default:** 50 |
|
||||
| parentId | query | | No | string |
|
||||
| view | query | | No | string |
|
||||
| control_space_id | path | | Yes | string |
|
||||
| source_id | path | | Yes | string |
|
||||
|
||||
@@ -29817,6 +29819,7 @@ the file name and a short-lived signed preview URL for files the caller still ow
|
||||
|
||||
| Name | Type | Description | Required |
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| has_children | boolean | | No |
|
||||
| last_edited_time | string | | No |
|
||||
| page_id | string | | Yes |
|
||||
| page_name | string | | Yes |
|
||||
@@ -29829,6 +29832,8 @@ the file name and a short-lived signed preview URL for files the caller still ow
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| cursor | string | | No |
|
||||
| limit | integer, <br>**Default:** 50 | | No |
|
||||
| parentId | string | | No |
|
||||
| view | string | | No |
|
||||
|
||||
#### KnowledgeFSSourcePagesResponse
|
||||
|
||||
|
||||
Generated
+5
@@ -1504,6 +1504,8 @@ Delete a logical document and its historical revisions. Use the logical document
|
||||
| ---- | ---------- | ----------- | -------- | ------ |
|
||||
| cursor | query | | No | string |
|
||||
| limit | query | | No | integer, <br>**Default:** 50 |
|
||||
| parentId | query | | No | string |
|
||||
| view | query | | No | string |
|
||||
| control_space_id | path | | Yes | string |
|
||||
| source_id | path | | Yes | string |
|
||||
|
||||
@@ -6780,6 +6782,7 @@ Dify owns datasource secrets; clients supply an opaque binding in configuration.
|
||||
|
||||
| Name | Type | Description | Required |
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| has_children | boolean | | No |
|
||||
| last_edited_time | string | | No |
|
||||
| page_id | string | | Yes |
|
||||
| page_name | string | | Yes |
|
||||
@@ -6792,6 +6795,8 @@ Dify owns datasource secrets; clients supply an opaque binding in configuration.
|
||||
| ---- | ---- | ----------- | -------- |
|
||||
| cursor | string | | No |
|
||||
| limit | integer, <br>**Default:** 50 | | No |
|
||||
| parentId | string | | No |
|
||||
| view | string | | No |
|
||||
|
||||
#### KnowledgeFSSourcePagesResponse
|
||||
|
||||
|
||||
@@ -1894,8 +1894,21 @@ class KnowledgeFSDataFacade:
|
||||
source_id: str,
|
||||
cursor: str | None = None,
|
||||
limit: int = 50,
|
||||
parent_id: str | None = None,
|
||||
view: Literal["tree"] | None = None,
|
||||
) -> KnowledgeFSSourcePagesResponse:
|
||||
query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ())
|
||||
"""List preview pages, optionally fetching a single tree branch.
|
||||
|
||||
Omitting tree arguments preserves flat previews. Background scans use a
|
||||
separate path. Tree requests forward only the requested branch and cursor.
|
||||
"""
|
||||
query = (("limit", str(limit)),)
|
||||
if cursor:
|
||||
query += (("cursor", cursor),)
|
||||
if parent_id:
|
||||
query += (("parentId", parent_id),)
|
||||
if view:
|
||||
query += (("view", view),)
|
||||
raw = self._interactive_child(
|
||||
tenant_id=tenant_id,
|
||||
account_id=account_id,
|
||||
|
||||
@@ -2663,11 +2663,14 @@ class KnowledgeFSSourceCrawlResponse(ResponseModel):
|
||||
class KnowledgeFSSourcePagesQuery(BaseModel):
|
||||
cursor: str | None = Field(default=None, min_length=1, max_length=4_096)
|
||||
limit: int = Field(default=50, ge=1, le=200)
|
||||
parent_id: str | None = Field(default=None, min_length=1, max_length=4_096, alias="parentId")
|
||||
view: Literal["tree"] | None = None
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class KnowledgeFSSourcePageResponse(ResponseModel):
|
||||
has_children: bool | None = Field(default=None, validation_alias=AliasChoices("has_children", "hasChildren"))
|
||||
last_edited_time: str | None = Field(
|
||||
default=None, validation_alias=AliasChoices("last_edited_time", "lastEditedTime")
|
||||
)
|
||||
|
||||
@@ -2979,6 +2979,60 @@ class TestKnowledgeFsResourceDelegationCases:
|
||||
else:
|
||||
assert result == ("", HTTPStatus.NO_CONTENT)
|
||||
|
||||
@staticmethod
|
||||
def test_console_source_pages_pass_tree_branch_query(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
list_source_pages = Recorder(_RAW_RESULT)
|
||||
monkeypatch.setattr(console_resources, "_actor", lambda: ("account-1", "tenant-1"))
|
||||
monkeypatch.setattr(
|
||||
console_resources,
|
||||
"_console_services",
|
||||
lambda: FakeRuntime(facade=FakeComponent(list_source_pages=list_source_pages)),
|
||||
)
|
||||
monkeypatch.setattr(console_resources, "dump_response", lambda _schema, raw: raw)
|
||||
app = Flask(__name__)
|
||||
|
||||
with app.test_request_context("/?view=tree&parentId=repo:FFXN/dify&cursor=next&limit=10"):
|
||||
result = _invoke(
|
||||
console_resources.KnowledgeFSSpaceSourcePagesApi,
|
||||
console_resources.KnowledgeFSSpaceSourcePagesApi.get,
|
||||
"space-1",
|
||||
"source-1",
|
||||
)
|
||||
|
||||
assert result is _RAW_RESULT
|
||||
assert list_source_pages.last_call.kwargs == {
|
||||
"tenant_id": "tenant-1",
|
||||
"account_id": "account-1",
|
||||
"control_space_id": "space-1",
|
||||
"source_id": "source-1",
|
||||
"cursor": "next",
|
||||
"limit": 10,
|
||||
"parent_id": "repo:FFXN/dify",
|
||||
"view": "tree",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def test_service_source_pages_pass_tree_branch_query(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
execute = Recorder(_RAW_RESULT)
|
||||
monkeypatch.setattr(service_resources, "_execute_service_operation", execute)
|
||||
monkeypatch.setattr(service_resources, "_dump_response", lambda _schema, raw: raw)
|
||||
app = Flask(__name__)
|
||||
|
||||
with app.test_request_context("/?view=tree&parentId=repo:FFXN/dify&limit=10"):
|
||||
result = _invoke(
|
||||
service_resources.KnowledgeFSServiceSourcePagesApi,
|
||||
service_resources.KnowledgeFSServiceSourcePagesApi.get,
|
||||
"space-1",
|
||||
"source-1",
|
||||
)
|
||||
|
||||
assert result is _RAW_RESULT
|
||||
assert execute.last_call.kwargs["query"] == (
|
||||
("limit", "10"),
|
||||
("parentId", "repo:FFXN/dify"),
|
||||
("view", "tree"),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def test_console_document_reference_delegates_exact_asset_identity(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
facade = FakeComponent(resolve_document_reference=Recorder(_RAW_RESULT))
|
||||
|
||||
@@ -2880,6 +2880,36 @@ def test_facade_public_methods_preserve_the_registered_operation_and_child_bindi
|
||||
assert delegated_kwargs["headers"] == (("Idempotency-Key", "import-source-once"),)
|
||||
|
||||
|
||||
def test_list_source_pages_forwards_tree_branch_without_changing_flat_default() -> None:
|
||||
facade = _offline_facade()
|
||||
call = CallRecorder(result={"workspaces": []})
|
||||
with patch.object(facade, "_interactive_child", call):
|
||||
facade.list_source_pages(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="space-1",
|
||||
source_id="source-1",
|
||||
)
|
||||
assert call.last.kwargs["query"] == (("limit", "50"),)
|
||||
|
||||
facade.list_source_pages(
|
||||
tenant_id="tenant-1",
|
||||
account_id="account-1",
|
||||
control_space_id="space-1",
|
||||
source_id="source-1",
|
||||
cursor="next",
|
||||
limit=10,
|
||||
parent_id="repo:FFXN/dify",
|
||||
view="tree",
|
||||
)
|
||||
assert call.last.kwargs["query"] == (
|
||||
("limit", "10"),
|
||||
("cursor", "next"),
|
||||
("parentId", "repo:FFXN/dify"),
|
||||
("view", "tree"),
|
||||
)
|
||||
|
||||
|
||||
def test_service_settings_migration_preserves_service_identity_on_every_operation() -> None:
|
||||
profile = KnowledgeFSServiceApiProfile(
|
||||
tenant_id="tenant-1",
|
||||
|
||||
@@ -61,6 +61,8 @@ from services.knowledge_fs.product_dto import (
|
||||
KnowledgeFSSourceEditSyncPolicyPayload,
|
||||
KnowledgeFSSourceImportFailureResponse,
|
||||
KnowledgeFSSourceListQuery,
|
||||
KnowledgeFSSourcePageResponse,
|
||||
KnowledgeFSSourcePagesQuery,
|
||||
KnowledgeFSSourceSyncPolicyPayload,
|
||||
KnowledgeFSSourceUpdatePayload,
|
||||
KnowledgeFSSourceWorkflowImportPayload,
|
||||
@@ -79,6 +81,22 @@ from services.knowledge_fs.product_dto import (
|
||||
)
|
||||
|
||||
|
||||
def test_source_pages_tree_preview_aliases() -> None:
|
||||
query = KnowledgeFSSourcePagesQuery.model_validate(
|
||||
{"view": "tree", "parentId": "repo:FFXN/dify", "cursor": "next", "limit": "10"}
|
||||
)
|
||||
assert query.model_dump(mode="json", by_alias=True, exclude_none=True) == {
|
||||
"cursor": "next",
|
||||
"limit": 10,
|
||||
"parentId": "repo:FFXN/dify",
|
||||
"view": "tree",
|
||||
}
|
||||
page = KnowledgeFSSourcePageResponse.model_validate(
|
||||
{"pageId": "repo:FFXN/dify", "pageName": "dify", "hasChildren": True, "type": "repository"}
|
||||
)
|
||||
assert page.model_dump(mode="json", exclude_none=True)["has_children"] is True
|
||||
|
||||
|
||||
def test_background_task_response_accepts_source_title_alias() -> None:
|
||||
task = KnowledgeFSBackgroundTaskResponse.model_validate(
|
||||
{
|
||||
|
||||
@@ -2595,6 +2595,7 @@ export type KnowledgeFsSourceFileResponse = {
|
||||
}
|
||||
|
||||
export type KnowledgeFsSourcePageResponse = {
|
||||
has_children?: boolean | null
|
||||
last_edited_time?: string | null
|
||||
page_id: string
|
||||
page_name: string
|
||||
@@ -5716,6 +5717,8 @@ export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesData = {
|
||||
query?: {
|
||||
cursor?: string
|
||||
limit?: number
|
||||
parentId?: string
|
||||
view?: string
|
||||
}
|
||||
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/pages'
|
||||
}
|
||||
|
||||
@@ -3322,6 +3322,7 @@ export const zKnowledgeFsSourceFilesResponse = z.object({
|
||||
* KnowledgeFSSourcePageResponse
|
||||
*/
|
||||
export const zKnowledgeFsSourcePageResponse = z.object({
|
||||
has_children: z.boolean().nullish(),
|
||||
last_edited_time: z.string().nullish(),
|
||||
page_id: z.string(),
|
||||
page_name: z.string(),
|
||||
@@ -4918,6 +4919,8 @@ export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesPath = z
|
||||
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesQuery = z.object({
|
||||
cursor: z.string().min(1).max(4096).optional(),
|
||||
limit: z.int().gte(1).lte(200).optional().default(50),
|
||||
parentId: z.string().min(1).max(4096).optional(),
|
||||
view: z.string().optional(),
|
||||
})
|
||||
|
||||
/**
|
||||
|
||||
@@ -2881,6 +2881,7 @@ export type KnowledgeFsSourceListResponse = {
|
||||
}
|
||||
|
||||
export type KnowledgeFsSourcePageResponse = {
|
||||
has_children?: boolean | null
|
||||
last_edited_time?: string | null
|
||||
page_id: string
|
||||
page_name: string
|
||||
@@ -2891,6 +2892,8 @@ export type KnowledgeFsSourcePageResponse = {
|
||||
export type KnowledgeFsSourcePagesQuery = {
|
||||
cursor?: string | null
|
||||
limit?: number
|
||||
parentId?: string | null
|
||||
view?: 'tree' | null
|
||||
}
|
||||
|
||||
export type KnowledgeFsSourcePagesResponse = {
|
||||
@@ -8622,6 +8625,8 @@ export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesData = {
|
||||
query?: {
|
||||
cursor?: string
|
||||
limit?: number
|
||||
parentId?: string
|
||||
view?: string
|
||||
}
|
||||
url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/pages'
|
||||
}
|
||||
|
||||
@@ -3339,6 +3339,7 @@ export const zKnowledgeFsSourceImportResponse = z.object({
|
||||
* KnowledgeFSSourcePageResponse
|
||||
*/
|
||||
export const zKnowledgeFsSourcePageResponse = z.object({
|
||||
has_children: z.boolean().nullish(),
|
||||
last_edited_time: z.string().nullish(),
|
||||
page_id: z.string(),
|
||||
page_name: z.string(),
|
||||
@@ -3352,6 +3353,8 @@ export const zKnowledgeFsSourcePageResponse = z.object({
|
||||
export const zKnowledgeFsSourcePagesQuery = z.object({
|
||||
cursor: z.string().min(1).max(4096).nullish(),
|
||||
limit: z.int().gte(1).lte(200).optional().default(50),
|
||||
parentId: z.string().min(1).max(4096).nullish(),
|
||||
view: z.literal('tree').nullish(),
|
||||
})
|
||||
|
||||
/**
|
||||
@@ -7788,6 +7791,8 @@ export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesPath = z
|
||||
export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesQuery = z.object({
|
||||
cursor: z.string().min(1).max(4096).optional(),
|
||||
limit: z.int().gte(1).lte(200).optional().default(50),
|
||||
parentId: z.string().min(1).max(4096).optional(),
|
||||
view: z.string().optional(),
|
||||
})
|
||||
|
||||
/**
|
||||
|
||||
@@ -193,6 +193,44 @@ const notionDatasourcePluginWithParameters: DataSourceItem = {
|
||||
},
|
||||
}
|
||||
|
||||
const githubProvider: KnowledgeFsSourceProviderResponse = {
|
||||
...notionProvider,
|
||||
display_name: 'GitHub Repository',
|
||||
id: 'github-provider',
|
||||
}
|
||||
|
||||
const githubCredential: DatasourceCredentialResponse = {
|
||||
...notionCredential,
|
||||
id: 'github-credential-1',
|
||||
name: 'GitHub token',
|
||||
}
|
||||
|
||||
const githubDatasourcePlugin: DataSourceItem = {
|
||||
...notionDatasourcePlugin,
|
||||
declaration: {
|
||||
...notionDatasourcePlugin.declaration,
|
||||
datasources: [
|
||||
{
|
||||
...notionDatasourcePlugin.declaration.datasources![0]!,
|
||||
identity: {
|
||||
...notionDatasourcePlugin.declaration.datasources![0]!.identity,
|
||||
label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' },
|
||||
name: 'github',
|
||||
provider: 'github_datasource',
|
||||
},
|
||||
},
|
||||
],
|
||||
identity: {
|
||||
...notionDatasourcePlugin.declaration.identity,
|
||||
label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' },
|
||||
name: 'github_datasource',
|
||||
},
|
||||
},
|
||||
plugin_id: 'langgenius/github_datasource',
|
||||
plugin_unique_identifier: 'langgenius/github_datasource:0.4.12@local',
|
||||
provider: 'github_datasource',
|
||||
}
|
||||
|
||||
const s3Provider: KnowledgeFsSourceProviderResponse = {
|
||||
auth_kinds: ['endpoint'],
|
||||
available: true,
|
||||
@@ -307,6 +345,17 @@ function notionDatasourceAuth(
|
||||
}
|
||||
}
|
||||
|
||||
function githubDatasourceAuth(): DatasourceProviderAuthResponse {
|
||||
return {
|
||||
...notionDatasourceAuth([githubCredential]),
|
||||
label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' },
|
||||
name: 'github_datasource',
|
||||
plugin_id: 'langgenius/github_datasource',
|
||||
plugin_unique_identifier: 'langgenius/github_datasource:0.4.12@local',
|
||||
provider: 'github_datasource',
|
||||
}
|
||||
}
|
||||
|
||||
function connectionResponse(
|
||||
overrides: Partial<KnowledgeFsSourceConnectionResponse> = {},
|
||||
): KnowledgeFsSourceConnectionResponse {
|
||||
@@ -853,6 +902,131 @@ describe('ConnectedSourceWorkflow', () => {
|
||||
expect(clientMock.deleteSource).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('loads GitHub tree branches on expansion and imports only explicitly selected pages', async () => {
|
||||
const user = userEvent.setup()
|
||||
const connection = connectionResponse({
|
||||
configuration: {
|
||||
credentialId: githubCredential.id,
|
||||
datasource: 'github',
|
||||
pluginId: 'langgenius/github_datasource',
|
||||
provider: 'github_datasource',
|
||||
providerKind: 'online-document',
|
||||
},
|
||||
name: 'GitHub token',
|
||||
provider_id: githubProvider.id,
|
||||
})
|
||||
clientMock.listProviders.mockResolvedValue({ data: [githubProvider] })
|
||||
clientMock.listDatasourcePlugins.mockResolvedValue([githubDatasourcePlugin])
|
||||
clientMock.listDatasourceAuth.mockResolvedValue({ result: [githubDatasourceAuth()] })
|
||||
clientMock.listConnections.mockResolvedValue({ data: [connection], next_cursor: null })
|
||||
clientMock.getPages.mockImplementation(
|
||||
({ query }: { query: { cursor?: string; parentId?: string } }) => {
|
||||
const pages = query.parentId
|
||||
? query.cursor
|
||||
? [
|
||||
{
|
||||
has_children: false,
|
||||
page_id: 'issue:FFXN/dify:1',
|
||||
page_name: 'Issue #1',
|
||||
parent_id: 'repo:FFXN/dify',
|
||||
type: 'issue',
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
has_children: false,
|
||||
page_id: 'file:FFXN/dify:README.md',
|
||||
page_name: 'README',
|
||||
parent_id: 'repo:FFXN/dify',
|
||||
type: 'file',
|
||||
},
|
||||
]
|
||||
: query.cursor
|
||||
? [
|
||||
{
|
||||
has_children: true,
|
||||
page_id: 'repo:FFXN/checkin',
|
||||
page_name: 'checkin',
|
||||
parent_id: null,
|
||||
type: 'repository',
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
has_children: true,
|
||||
page_id: 'repo:FFXN/dify',
|
||||
page_name: 'dify',
|
||||
parent_id: null,
|
||||
type: 'repository',
|
||||
},
|
||||
]
|
||||
return Promise.resolve({
|
||||
next_cursor: !query.cursor ? (query.parentId ? 'branch-next' : 'root-next') : null,
|
||||
workspaces: [
|
||||
{
|
||||
pages,
|
||||
total: pages.length,
|
||||
workspace_id: 'FFXN',
|
||||
workspace_name: "FFXN's GitHub",
|
||||
},
|
||||
],
|
||||
})
|
||||
},
|
||||
)
|
||||
|
||||
const view = await renderSetup({
|
||||
...defaultDraft,
|
||||
provider: 'GitHub Repository',
|
||||
sourceName: 'GitHub docs',
|
||||
})
|
||||
const repository = await screen.findByRole('checkbox', { name: 'dify' })
|
||||
expect(clientMock.getPages).toHaveBeenCalledExactlyOnceWith({
|
||||
params: { control_space_id: 'space-1', source_id: 'preview-source' },
|
||||
query: { limit: 10, view: 'tree' },
|
||||
})
|
||||
expect(screen.queryByRole('checkbox', { name: 'README' })).not.toBeInTheDocument()
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'knowledgeSpace.loadMore' }))
|
||||
expect(await screen.findByRole('checkbox', { name: 'checkin' })).not.toBeChecked()
|
||||
expect(clientMock.getPages).toHaveBeenCalledWith({
|
||||
params: { control_space_id: 'space-1', source_id: 'preview-source' },
|
||||
query: { cursor: 'root-next', limit: 10, view: 'tree' },
|
||||
})
|
||||
|
||||
await user.click(repository)
|
||||
await user.click(screen.getByRole('button', { name: 'dify' }))
|
||||
const readme = await screen.findByRole('checkbox', { name: 'README' })
|
||||
expect(clientMock.getPages).toHaveBeenCalledWith({
|
||||
params: { control_space_id: 'space-1', source_id: 'preview-source' },
|
||||
query: { limit: 200, parentId: 'repo:FFXN/dify', view: 'tree' },
|
||||
})
|
||||
expect(readme).not.toBeChecked()
|
||||
expect(repository).toBePartiallyChecked()
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'knowledgeSpace.loadMore' }))
|
||||
const issue = await screen.findByRole('checkbox', { name: 'Issue #1' })
|
||||
expect(clientMock.getPages).toHaveBeenCalledWith({
|
||||
params: { control_space_id: 'space-1', source_id: 'preview-source' },
|
||||
query: { cursor: 'branch-next', limit: 200, parentId: 'repo:FFXN/dify', view: 'tree' },
|
||||
})
|
||||
expect(issue).not.toBeChecked()
|
||||
|
||||
await user.click(repository)
|
||||
expect(readme).toBeChecked()
|
||||
expect(issue).toBeChecked()
|
||||
await user.click(readme)
|
||||
await user.click(issue)
|
||||
await user.click(screen.getByRole('button', { name: 'knowledgeSpace.addSource' }))
|
||||
await waitFor(() => expect(view.onCompleted).toHaveBeenCalledOnce())
|
||||
expect(clientMock.createWorkflowImport).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
items: [expect.objectContaining({ pageId: 'repo:FFXN/dify' })],
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('stops the import and cleans up the renamed source when canceled while the rename is pending', async () => {
|
||||
const user = userEvent.setup()
|
||||
clientMock.listDatasourceAuth.mockResolvedValue({
|
||||
|
||||
@@ -24,7 +24,9 @@ import { Checkbox } from '@langgenius/dify-ui/checkbox'
|
||||
import { cn } from '@langgenius/dify-ui/cn'
|
||||
import {
|
||||
infiniteQueryOptions,
|
||||
queryOptions,
|
||||
useInfiniteQuery,
|
||||
useQueries,
|
||||
useQuery,
|
||||
useQueryClient,
|
||||
} from '@tanstack/react-query'
|
||||
@@ -107,9 +109,16 @@ type DriveBranch = {
|
||||
loaded: boolean
|
||||
loading: boolean
|
||||
}
|
||||
type PageBranchRequest = { parentId: string; cursors: (string | null)[] }
|
||||
type PageBranch = {
|
||||
error: boolean
|
||||
loading: boolean
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
const CONNECTION_PAGE_SIZE = 200
|
||||
const RESOURCE_PAGE_SIZE = 200
|
||||
const GITHUB_ROOT_PAGE_SIZE = 10
|
||||
const MAX_SELECTION = 200
|
||||
const MAX_SOURCE_CURSOR_PAGES = 100
|
||||
|
||||
@@ -324,6 +333,8 @@ function hasDriveFolderCheckbox(resource: SelectableResource) {
|
||||
}
|
||||
|
||||
function resourceIcon(resource: SelectableResource) {
|
||||
if (resource.kind === 'page' && resource.page.type === 'repository')
|
||||
return 'i-ri-folder-3-fill text-text-warning'
|
||||
if (isFolder(resource) || isBucket(resource)) return 'i-ri-folder-3-fill text-text-warning'
|
||||
if (resource.kind === 'page' && /database/i.test(resource.page.type))
|
||||
return 'i-ri-database-2-line text-text-accent'
|
||||
@@ -337,14 +348,18 @@ function ResourceList({
|
||||
driveBranches,
|
||||
driveDescendantLeafKeys,
|
||||
expandedDriveContainers,
|
||||
expandedPageKeys,
|
||||
folderSelectionIntents,
|
||||
hasMore,
|
||||
loadingMore,
|
||||
onLoadMore,
|
||||
onLoadMorePageBranch,
|
||||
onRetryPageBranch,
|
||||
onToggle,
|
||||
onToggleAll,
|
||||
onToggleExpanded,
|
||||
providerRegion,
|
||||
pageBranches,
|
||||
resources,
|
||||
selectionAtLimit,
|
||||
selectionLimitId,
|
||||
@@ -352,20 +367,25 @@ function ResourceList({
|
||||
selectionPending,
|
||||
selectionScope,
|
||||
selected,
|
||||
treePreview,
|
||||
}: {
|
||||
collapsed: ReadonlySet<string>
|
||||
connection: SourceConnection
|
||||
driveBranches: ReadonlyMap<string, DriveBranch>
|
||||
driveDescendantLeafKeys: ReadonlyMap<string, string[]>
|
||||
expandedDriveContainers: ReadonlySet<string>
|
||||
expandedPageKeys: ReadonlySet<string>
|
||||
folderSelectionIntents: ReadonlySet<string>
|
||||
hasMore: boolean
|
||||
loadingMore: boolean
|
||||
onLoadMore: () => void
|
||||
onLoadMorePageBranch: (resource: PageResource) => void
|
||||
onRetryPageBranch: (resource: PageResource) => void
|
||||
onToggle: (resource: SelectableResource) => void
|
||||
onToggleAll: () => void
|
||||
onToggleExpanded: (resource: SelectableResource) => void
|
||||
providerRegion?: string
|
||||
pageBranches: ReadonlyMap<string, PageBranch>
|
||||
resources: SelectableResource[]
|
||||
selectionAtLimit: boolean
|
||||
selectionLimitId: string
|
||||
@@ -373,8 +393,9 @@ function ResourceList({
|
||||
selectionPending: boolean
|
||||
selectionScope: SelectableResource[]
|
||||
selected: ReadonlyMap<string, SelectableResource>
|
||||
treePreview: boolean
|
||||
}) {
|
||||
const { t } = useTranslation(['knowledgeSpace', 'knowledgeSources'])
|
||||
const { t } = useTranslation(['knowledgeSpace', 'knowledgeSources', 'common'])
|
||||
const selectableResources = selectionScope.filter(isSelectableResource)
|
||||
const driveContainers = selectionScope.filter(isDriveContainer)
|
||||
const allDriveContainersLoaded = driveContainers.every(
|
||||
@@ -426,6 +447,7 @@ function ResourceList({
|
||||
(hasSelectionKeys && selectionKeys.every((key) => selected.has(key)))
|
||||
const indeterminate = !checked && selectionKeys.some((key) => selected.has(key))
|
||||
const driveBranch = resource.kind === 'file' ? driveBranches.get(resource.key) : undefined
|
||||
const pageBranch = resource.kind === 'page' ? pageBranches.get(resource.key) : undefined
|
||||
const emptyDriveFolder =
|
||||
hasDriveFolderCheckbox(resource) && driveBranch?.loaded && !selectionKeys.length
|
||||
const selectionBlocked = selectionAtLimit && !checked
|
||||
@@ -433,7 +455,9 @@ function ResourceList({
|
||||
selectionBlocked || emptyDriveFolder || Boolean(driveBranch?.loading)
|
||||
const expanded =
|
||||
resource.kind === 'page'
|
||||
? !collapsed.has(resource.key)
|
||||
? treePreview
|
||||
? expandedPageKeys.has(resource.key)
|
||||
: !collapsed.has(resource.key)
|
||||
: expandedDriveContainers.has(resource.key)
|
||||
const rendersCheckbox =
|
||||
resource.kind === 'page' ||
|
||||
@@ -484,6 +508,37 @@ function ResourceList({
|
||||
<span className="min-w-0 flex-1 truncate system-xs-regular text-text-primary">
|
||||
{resourceName(resource)}
|
||||
</span>
|
||||
{expanded && pageBranch?.loading && (
|
||||
<span
|
||||
role="status"
|
||||
aria-label={t(($) => $.loading, { ns: 'common' })}
|
||||
className="i-ri-loader-4-line size-4 animate-spin text-text-tertiary"
|
||||
/>
|
||||
)}
|
||||
{expanded && pageBranch?.error && resource.kind === 'page' && (
|
||||
<Button
|
||||
type="button"
|
||||
size="small"
|
||||
variant="ghost"
|
||||
onClick={() => onRetryPageBranch(resource)}
|
||||
>
|
||||
{t(($) => $.retryProviderLoad)}
|
||||
</Button>
|
||||
)}
|
||||
{expanded &&
|
||||
!pageBranch?.loading &&
|
||||
!pageBranch?.error &&
|
||||
pageBranch?.nextCursor &&
|
||||
resource.kind === 'page' && (
|
||||
<Button
|
||||
type="button"
|
||||
size="small"
|
||||
variant="ghost"
|
||||
onClick={() => onLoadMorePageBranch(resource)}
|
||||
>
|
||||
{t(($) => $.loadMore)}
|
||||
</Button>
|
||||
)}
|
||||
{resource.kind === 'file' && resource.file.size != null && (
|
||||
<span className="system-xs-regular text-text-quaternary">
|
||||
{formatFileSize(resource.file.size)}
|
||||
@@ -532,6 +587,7 @@ type PreviewSourceSession = {
|
||||
function ResourceConfiguration({
|
||||
connection,
|
||||
draft,
|
||||
githubTreePreview,
|
||||
knowledgeSpaceId,
|
||||
onCompleted,
|
||||
onDraftChange,
|
||||
@@ -542,6 +598,7 @@ function ResourceConfiguration({
|
||||
}: {
|
||||
connection: SourceConnection
|
||||
draft: ConnectedSourceDraft
|
||||
githubTreePreview: boolean
|
||||
knowledgeSpaceId: string
|
||||
onCompleted: () => void
|
||||
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
|
||||
@@ -561,6 +618,10 @@ function ResourceConfiguration({
|
||||
const [selected, setSelected] = useState<Map<string, SelectableResource>>(() => new Map())
|
||||
const selectedRef = useRef<Map<string, SelectableResource>>(new Map())
|
||||
const [collapsed, setCollapsed] = useState<Set<string>>(() => new Set())
|
||||
const [expandedPageKeys, setExpandedPageKeys] = useState<Set<string>>(() => new Set())
|
||||
const [pageBranchRequests, setPageBranchRequests] = useState<Map<string, PageBranchRequest>>(
|
||||
() => new Map(),
|
||||
)
|
||||
const [driveBranches, setDriveBranches] = useState<Map<string, DriveBranch>>(() => new Map())
|
||||
const [expandedDriveContainers, setExpandedDriveContainers] = useState<Set<string>>(
|
||||
() => new Set(),
|
||||
@@ -663,7 +724,13 @@ function ResourceConfiguration({
|
||||
|
||||
const pagesQuery = useInfiniteQuery(
|
||||
infiniteQueryOptions({
|
||||
queryKey: ['new-rag', 'connected-source-pages', knowledgeSpaceId, previewSource?.id],
|
||||
queryKey: [
|
||||
'new-rag',
|
||||
'connected-source-pages',
|
||||
knowledgeSpaceId,
|
||||
previewSource?.id,
|
||||
githubTreePreview,
|
||||
],
|
||||
queryFn: ({ pageParam }) =>
|
||||
consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.pages.get({
|
||||
params: {
|
||||
@@ -671,7 +738,8 @@ function ResourceConfiguration({
|
||||
source_id: previewSource?.id ?? '',
|
||||
},
|
||||
query: {
|
||||
limit: RESOURCE_PAGE_SIZE,
|
||||
limit: githubTreePreview ? GITHUB_ROOT_PAGE_SIZE : RESOURCE_PAGE_SIZE,
|
||||
...(githubTreePreview ? { view: 'tree' as const } : {}),
|
||||
...(typeof pageParam === 'string' ? { cursor: pageParam } : {}),
|
||||
},
|
||||
}),
|
||||
@@ -681,6 +749,62 @@ function ResourceConfiguration({
|
||||
retry: false,
|
||||
}),
|
||||
)
|
||||
const pageBranchEntries = useMemo(
|
||||
() =>
|
||||
[...pageBranchRequests].flatMap(([resourceKey, request]) =>
|
||||
request.cursors.map((cursor) => ({ cursor, parentId: request.parentId, resourceKey })),
|
||||
),
|
||||
[pageBranchRequests],
|
||||
)
|
||||
const pageBranchQueries = useQueries({
|
||||
queries: pageBranchEntries.map(({ cursor, parentId }) =>
|
||||
queryOptions({
|
||||
queryKey: [
|
||||
'new-rag',
|
||||
'connected-source-page-branch',
|
||||
knowledgeSpaceId,
|
||||
previewSource?.id,
|
||||
parentId,
|
||||
cursor,
|
||||
],
|
||||
queryFn: () =>
|
||||
consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.pages.get({
|
||||
params: {
|
||||
control_space_id: knowledgeSpaceId,
|
||||
source_id: previewSource?.id ?? '',
|
||||
},
|
||||
query: {
|
||||
limit: RESOURCE_PAGE_SIZE,
|
||||
parentId,
|
||||
view: 'tree',
|
||||
...(cursor ? { cursor } : {}),
|
||||
},
|
||||
}),
|
||||
enabled: githubTreePreview && Boolean(previewSource),
|
||||
retry: false,
|
||||
}),
|
||||
),
|
||||
})
|
||||
const pageBranches = (() => {
|
||||
const branches = new Map<string, PageBranch>()
|
||||
pageBranchEntries.forEach(({ resourceKey }, index) => {
|
||||
const query = pageBranchQueries[index]
|
||||
if (!query) return
|
||||
const current = branches.get(resourceKey)
|
||||
const nextCursor = query.data
|
||||
? (query.data.next_cursor ?? null)
|
||||
: (current?.nextCursor ?? null)
|
||||
branches.set(resourceKey, {
|
||||
error: Boolean(current?.error || query.isError),
|
||||
loading: Boolean(current?.loading || query.isPending || query.isFetching),
|
||||
nextCursor:
|
||||
nextCursor && !pageBranchRequests.get(resourceKey)?.cursors.includes(nextCursor)
|
||||
? nextCursor
|
||||
: null,
|
||||
})
|
||||
})
|
||||
return branches
|
||||
})()
|
||||
const filesQuery = useInfiniteQuery(
|
||||
infiniteQueryOptions({
|
||||
queryKey: ['new-rag', 'connected-source-files', knowledgeSpaceId, previewSource?.id],
|
||||
@@ -720,12 +844,16 @@ function ResourceConfiguration({
|
||||
}),
|
||||
)
|
||||
|
||||
const pageResources = useMemo<PageResource[]>(() => {
|
||||
const pageResources: PageResource[] = (() => {
|
||||
const groups = new Map<
|
||||
string,
|
||||
{ name: string; pages: Map<string, KnowledgeFsSourcePageResponse> }
|
||||
>()
|
||||
for (const response of pagesQuery.data?.pages ?? []) {
|
||||
const responses = [
|
||||
...(pagesQuery.data?.pages ?? []),
|
||||
...pageBranchQueries.flatMap((query) => (query.data ? [query.data] : [])),
|
||||
]
|
||||
for (const response of responses) {
|
||||
for (const workspace of response.workspaces) {
|
||||
const groupId = workspace.workspace_id?.trim() || 'workspace'
|
||||
const group = groups.get(groupId) ?? {
|
||||
@@ -768,7 +896,27 @@ function ResourceConfiguration({
|
||||
}
|
||||
return descendants
|
||||
}
|
||||
return [...group.pages.values()].map((page) => {
|
||||
const orderedPages = [...group.pages.values()]
|
||||
if (githubTreePreview) {
|
||||
const byId = group.pages
|
||||
const visited = new Set<string>()
|
||||
const treeOrder: KnowledgeFsSourcePageResponse[] = []
|
||||
const visit = (page: KnowledgeFsSourcePageResponse) => {
|
||||
if (visited.has(page.page_id)) return
|
||||
visited.add(page.page_id)
|
||||
treeOrder.push(page)
|
||||
for (const childId of children.get(page.page_id) ?? []) {
|
||||
const child = byId.get(childId)
|
||||
if (child) visit(child)
|
||||
}
|
||||
}
|
||||
for (const page of orderedPages) {
|
||||
if (!page.parent_id || !byId.has(page.parent_id)) visit(page)
|
||||
}
|
||||
for (const page of orderedPages) visit(page)
|
||||
orderedPages.splice(0, orderedPages.length, ...treeOrder)
|
||||
}
|
||||
return orderedPages.map((page) => {
|
||||
const ancestorKeys = ancestorsFor(page)
|
||||
return {
|
||||
ancestorKeys,
|
||||
@@ -776,14 +924,14 @@ function ResourceConfiguration({
|
||||
descendantKeys: descendantsFor(page.page_id),
|
||||
groupId,
|
||||
groupName: group.name,
|
||||
hasChildren: Boolean(children.get(page.page_id)?.length),
|
||||
hasChildren: Boolean(page.has_children || children.get(page.page_id)?.length),
|
||||
key: keyFor(page.page_id),
|
||||
kind: 'page' as const,
|
||||
page,
|
||||
}
|
||||
})
|
||||
})
|
||||
}, [connection.name, pagesQuery.data?.pages])
|
||||
})()
|
||||
const rootFileResources = useMemo<FileResource[]>(() => {
|
||||
const resourcesByKey = new Map<string, FileResource>()
|
||||
for (const response of filesQuery.data?.pages ?? []) {
|
||||
@@ -966,9 +1114,19 @@ function ResourceConfiguration({
|
||||
driveTransport
|
||||
? visibleDriveResources
|
||||
: pageResources.filter(
|
||||
(resource) => !resource.ancestorKeys.some((ancestorKey) => collapsed.has(ancestorKey)),
|
||||
(resource) =>
|
||||
!resource.ancestorKeys.some((ancestorKey) =>
|
||||
githubTreePreview ? !expandedPageKeys.has(ancestorKey) : collapsed.has(ancestorKey),
|
||||
),
|
||||
),
|
||||
[collapsed, driveTransport, pageResources, visibleDriveResources],
|
||||
[
|
||||
collapsed,
|
||||
driveTransport,
|
||||
expandedPageKeys,
|
||||
githubTreePreview,
|
||||
pageResources,
|
||||
visibleDriveResources,
|
||||
],
|
||||
)
|
||||
const queryPending = driveTransport ? filesQuery.isPending : pagesQuery.isPending
|
||||
const queryError = driveTransport ? filesQuery.error : pagesQuery.error
|
||||
@@ -1161,6 +1319,25 @@ function ResourceConfiguration({
|
||||
|
||||
const toggleExpanded = (resource: SelectableResource) => {
|
||||
if (resource.kind === 'page') {
|
||||
if (githubTreePreview) {
|
||||
const expanding = !expandedPageKeys.has(resource.key)
|
||||
setExpandedPageKeys((current) => {
|
||||
const next = new Set(current)
|
||||
if (expanding) next.add(resource.key)
|
||||
else next.delete(resource.key)
|
||||
return next
|
||||
})
|
||||
if (expanding) {
|
||||
setPageBranchRequests((current) => {
|
||||
if (current.has(resource.key)) return current
|
||||
return new Map(current).set(resource.key, {
|
||||
parentId: resource.page.page_id,
|
||||
cursors: [null],
|
||||
})
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
setCollapsed((current) => {
|
||||
const next = new Set(current)
|
||||
if (next.has(resource.key)) next.delete(resource.key)
|
||||
@@ -1179,6 +1356,26 @@ function ResourceConfiguration({
|
||||
if (expanding) void loadDriveChildren(resource).catch(() => undefined)
|
||||
}
|
||||
|
||||
const loadMorePageBranch = (resource: PageResource) => {
|
||||
const nextCursor = pageBranches.get(resource.key)?.nextCursor
|
||||
if (!nextCursor) return
|
||||
setPageBranchRequests((current) => {
|
||||
const request = current.get(resource.key)
|
||||
if (!request || request.cursors.includes(nextCursor)) return current
|
||||
return new Map(current).set(resource.key, {
|
||||
...request,
|
||||
cursors: [...request.cursors, nextCursor],
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const retryPageBranch = (resource: PageResource) => {
|
||||
pageBranchEntries.forEach(({ resourceKey }, index) => {
|
||||
if (resourceKey === resource.key && pageBranchQueries[index]?.isError)
|
||||
void pageBranchQueries[index].refetch()
|
||||
})
|
||||
}
|
||||
|
||||
const loadMore = () => {
|
||||
if (driveTransport) void filesQuery.fetchNextPage()
|
||||
else void pagesQuery.fetchNextPage()
|
||||
@@ -1422,14 +1619,18 @@ function ResourceConfiguration({
|
||||
driveBranches={driveBranches}
|
||||
driveDescendantLeafKeys={driveDescendantLeafKeys}
|
||||
expandedDriveContainers={expandedDriveContainers}
|
||||
expandedPageKeys={expandedPageKeys}
|
||||
folderSelectionIntents={folderSelectionIntents}
|
||||
hasMore={hasMore}
|
||||
loadingMore={loadingMore}
|
||||
onLoadMore={loadMore}
|
||||
onLoadMorePageBranch={loadMorePageBranch}
|
||||
onRetryPageBranch={retryPageBranch}
|
||||
onToggle={toggle}
|
||||
onToggleAll={toggleAll}
|
||||
onToggleExpanded={toggleExpanded}
|
||||
providerRegion={providerRegion}
|
||||
pageBranches={pageBranches}
|
||||
resources={visibleResources}
|
||||
selectionAtLimit={selectionAtLimit}
|
||||
selectionLimitId={selectionLimitId}
|
||||
@@ -1437,6 +1638,7 @@ function ResourceConfiguration({
|
||||
selectionPending={selectionPending}
|
||||
selectionScope={resources}
|
||||
selected={selected}
|
||||
treePreview={githubTreePreview}
|
||||
/>
|
||||
)}
|
||||
</section>
|
||||
@@ -1475,6 +1677,7 @@ function ResourceConfiguration({
|
||||
function AppliedResourceConfiguration({
|
||||
connection,
|
||||
draft,
|
||||
githubTreePreview,
|
||||
knowledgeSpaceId,
|
||||
onCompleted,
|
||||
onDraftChange,
|
||||
@@ -1487,6 +1690,7 @@ function AppliedResourceConfiguration({
|
||||
}: {
|
||||
connection: SourceConnection
|
||||
draft: ConnectedSourceDraft
|
||||
githubTreePreview: boolean
|
||||
knowledgeSpaceId: string
|
||||
onCompleted: () => void
|
||||
onDraftChange: (draft: NewKnowledgeSourceDraft) => void
|
||||
@@ -1534,6 +1738,7 @@ function AppliedResourceConfiguration({
|
||||
<ResourceConfiguration
|
||||
connection={connection}
|
||||
draft={draft}
|
||||
githubTreePreview={githubTreePreview}
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onCompleted={onCompleted}
|
||||
onDraftChange={onDraftChange}
|
||||
@@ -1884,6 +2089,10 @@ function ConnectedSourceWorkflowSession({
|
||||
key={`${provider.id}:${connection.id}`}
|
||||
connection={connection}
|
||||
draft={providerDraft}
|
||||
githubTreePreview={
|
||||
datasourceProvider?.plugin.plugin_id === 'langgenius/github_datasource' &&
|
||||
datasourceProvider.datasource.identity.name === 'github'
|
||||
}
|
||||
knowledgeSpaceId={knowledgeSpaceId}
|
||||
onCompleted={onCompleted}
|
||||
onDraftChange={onDraftChange}
|
||||
|
||||
Reference in new issue
Block a user