diff --git a/api/controllers/console/knowledge_fs/resources.py b/api/controllers/console/knowledge_fs/resources.py index e2941773295..061b215b0ed 100644 --- a/api/controllers/console/knowledge_fs/resources.py +++ b/api/controllers/console/knowledge_fs/resources.py @@ -3149,6 +3149,8 @@ class KnowledgeFSSpaceSourcePagesApi(Resource): source_id=source_id, cursor=query.cursor, limit=query.limit, + parent_id=query.parent_id, + view=query.view, ) return dump_response(KnowledgeFSSourcePagesResponse, result) diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 2c8b2f6032a..22684601935 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -10643,6 +10643,8 @@ Claim a workspace-staged upload. Multipart file bodies remain accepted as a lega | ---- | ---------- | ----------- | -------- | ------ | | cursor | query | | No | string | | limit | query | | No | integer,
**Default:** 50 | +| parentId | query | | No | string | +| view | query | | No | string | | control_space_id | path | | Yes | string | | source_id | path | | Yes | string | @@ -29817,6 +29819,7 @@ the file name and a short-lived signed preview URL for files the caller still ow | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | +| has_children | boolean | | No | | last_edited_time | string | | No | | page_id | string | | Yes | | page_name | string | | Yes | @@ -29829,6 +29832,8 @@ the file name and a short-lived signed preview URL for files the caller still ow | ---- | ---- | ----------- | -------- | | cursor | string | | No | | limit | integer,
**Default:** 50 | | No | +| parentId | string | | No | +| view | string | | No | #### KnowledgeFSSourcePagesResponse diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index 10221c72cc0..b751ba63e24 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -1504,6 +1504,8 @@ Delete a logical document and its historical revisions. Use the logical document | ---- | ---------- | ----------- | -------- | ------ | | cursor | query | | No | string | | limit | query | | No | integer,
**Default:** 50 | +| parentId | query | | No | string | +| view | query | | No | string | | control_space_id | path | | Yes | string | | source_id | path | | Yes | string | @@ -6780,6 +6782,7 @@ Dify owns datasource secrets; clients supply an opaque binding in configuration. | Name | Type | Description | Required | | ---- | ---- | ----------- | -------- | +| has_children | boolean | | No | | last_edited_time | string | | No | | page_id | string | | Yes | | page_name | string | | Yes | @@ -6792,6 +6795,8 @@ Dify owns datasource secrets; clients supply an opaque binding in configuration. | ---- | ---- | ----------- | -------- | | cursor | string | | No | | limit | integer,
**Default:** 50 | | No | +| parentId | string | | No | +| view | string | | No | #### KnowledgeFSSourcePagesResponse diff --git a/api/services/knowledge_fs/data_facade.py b/api/services/knowledge_fs/data_facade.py index 84e046afaff..529b019f8ce 100644 --- a/api/services/knowledge_fs/data_facade.py +++ b/api/services/knowledge_fs/data_facade.py @@ -1894,8 +1894,21 @@ class KnowledgeFSDataFacade: source_id: str, cursor: str | None = None, limit: int = 50, + parent_id: str | None = None, + view: Literal["tree"] | None = None, ) -> KnowledgeFSSourcePagesResponse: - query = (("limit", str(limit)),) + ((("cursor", cursor),) if cursor else ()) + """List preview pages, optionally fetching a single tree branch. + + Omitting tree arguments preserves flat previews. Background scans use a + separate path. Tree requests forward only the requested branch and cursor. + """ + query = (("limit", str(limit)),) + if cursor: + query += (("cursor", cursor),) + if parent_id: + query += (("parentId", parent_id),) + if view: + query += (("view", view),) raw = self._interactive_child( tenant_id=tenant_id, account_id=account_id, diff --git a/api/services/knowledge_fs/product_dto.py b/api/services/knowledge_fs/product_dto.py index 69e84e4176e..ac23effe260 100644 --- a/api/services/knowledge_fs/product_dto.py +++ b/api/services/knowledge_fs/product_dto.py @@ -2663,11 +2663,14 @@ class KnowledgeFSSourceCrawlResponse(ResponseModel): class KnowledgeFSSourcePagesQuery(BaseModel): cursor: str | None = Field(default=None, min_length=1, max_length=4_096) limit: int = Field(default=50, ge=1, le=200) + parent_id: str | None = Field(default=None, min_length=1, max_length=4_096, alias="parentId") + view: Literal["tree"] | None = None model_config = ConfigDict(extra="forbid") class KnowledgeFSSourcePageResponse(ResponseModel): + has_children: bool | None = Field(default=None, validation_alias=AliasChoices("has_children", "hasChildren")) last_edited_time: str | None = Field( default=None, validation_alias=AliasChoices("last_edited_time", "lastEditedTime") ) diff --git a/api/tests/unit_tests/controllers/console/knowledge_fs/test_resources.py b/api/tests/unit_tests/controllers/console/knowledge_fs/test_resources.py index cdc08c4c7fe..c7e946c19d9 100644 --- a/api/tests/unit_tests/controllers/console/knowledge_fs/test_resources.py +++ b/api/tests/unit_tests/controllers/console/knowledge_fs/test_resources.py @@ -2979,6 +2979,60 @@ class TestKnowledgeFsResourceDelegationCases: else: assert result == ("", HTTPStatus.NO_CONTENT) + @staticmethod + def test_console_source_pages_pass_tree_branch_query(monkeypatch: pytest.MonkeyPatch) -> None: + list_source_pages = Recorder(_RAW_RESULT) + monkeypatch.setattr(console_resources, "_actor", lambda: ("account-1", "tenant-1")) + monkeypatch.setattr( + console_resources, + "_console_services", + lambda: FakeRuntime(facade=FakeComponent(list_source_pages=list_source_pages)), + ) + monkeypatch.setattr(console_resources, "dump_response", lambda _schema, raw: raw) + app = Flask(__name__) + + with app.test_request_context("/?view=tree&parentId=repo:FFXN/dify&cursor=next&limit=10"): + result = _invoke( + console_resources.KnowledgeFSSpaceSourcePagesApi, + console_resources.KnowledgeFSSpaceSourcePagesApi.get, + "space-1", + "source-1", + ) + + assert result is _RAW_RESULT + assert list_source_pages.last_call.kwargs == { + "tenant_id": "tenant-1", + "account_id": "account-1", + "control_space_id": "space-1", + "source_id": "source-1", + "cursor": "next", + "limit": 10, + "parent_id": "repo:FFXN/dify", + "view": "tree", + } + + @staticmethod + def test_service_source_pages_pass_tree_branch_query(monkeypatch: pytest.MonkeyPatch) -> None: + execute = Recorder(_RAW_RESULT) + monkeypatch.setattr(service_resources, "_execute_service_operation", execute) + monkeypatch.setattr(service_resources, "_dump_response", lambda _schema, raw: raw) + app = Flask(__name__) + + with app.test_request_context("/?view=tree&parentId=repo:FFXN/dify&limit=10"): + result = _invoke( + service_resources.KnowledgeFSServiceSourcePagesApi, + service_resources.KnowledgeFSServiceSourcePagesApi.get, + "space-1", + "source-1", + ) + + assert result is _RAW_RESULT + assert execute.last_call.kwargs["query"] == ( + ("limit", "10"), + ("parentId", "repo:FFXN/dify"), + ("view", "tree"), + ) + @staticmethod def test_console_document_reference_delegates_exact_asset_identity(monkeypatch: pytest.MonkeyPatch) -> None: facade = FakeComponent(resolve_document_reference=Recorder(_RAW_RESULT)) diff --git a/api/tests/unit_tests/services/knowledge_fs/test_data_facade.py b/api/tests/unit_tests/services/knowledge_fs/test_data_facade.py index c12e6ad2a3f..ed23cc6ac96 100644 --- a/api/tests/unit_tests/services/knowledge_fs/test_data_facade.py +++ b/api/tests/unit_tests/services/knowledge_fs/test_data_facade.py @@ -2880,6 +2880,36 @@ def test_facade_public_methods_preserve_the_registered_operation_and_child_bindi assert delegated_kwargs["headers"] == (("Idempotency-Key", "import-source-once"),) +def test_list_source_pages_forwards_tree_branch_without_changing_flat_default() -> None: + facade = _offline_facade() + call = CallRecorder(result={"workspaces": []}) + with patch.object(facade, "_interactive_child", call): + facade.list_source_pages( + tenant_id="tenant-1", + account_id="account-1", + control_space_id="space-1", + source_id="source-1", + ) + assert call.last.kwargs["query"] == (("limit", "50"),) + + facade.list_source_pages( + tenant_id="tenant-1", + account_id="account-1", + control_space_id="space-1", + source_id="source-1", + cursor="next", + limit=10, + parent_id="repo:FFXN/dify", + view="tree", + ) + assert call.last.kwargs["query"] == ( + ("limit", "10"), + ("cursor", "next"), + ("parentId", "repo:FFXN/dify"), + ("view", "tree"), + ) + + def test_service_settings_migration_preserves_service_identity_on_every_operation() -> None: profile = KnowledgeFSServiceApiProfile( tenant_id="tenant-1", diff --git a/api/tests/unit_tests/services/knowledge_fs/test_product_dto.py b/api/tests/unit_tests/services/knowledge_fs/test_product_dto.py index fdc24863514..43c62477370 100644 --- a/api/tests/unit_tests/services/knowledge_fs/test_product_dto.py +++ b/api/tests/unit_tests/services/knowledge_fs/test_product_dto.py @@ -61,6 +61,8 @@ from services.knowledge_fs.product_dto import ( KnowledgeFSSourceEditSyncPolicyPayload, KnowledgeFSSourceImportFailureResponse, KnowledgeFSSourceListQuery, + KnowledgeFSSourcePageResponse, + KnowledgeFSSourcePagesQuery, KnowledgeFSSourceSyncPolicyPayload, KnowledgeFSSourceUpdatePayload, KnowledgeFSSourceWorkflowImportPayload, @@ -79,6 +81,22 @@ from services.knowledge_fs.product_dto import ( ) +def test_source_pages_tree_preview_aliases() -> None: + query = KnowledgeFSSourcePagesQuery.model_validate( + {"view": "tree", "parentId": "repo:FFXN/dify", "cursor": "next", "limit": "10"} + ) + assert query.model_dump(mode="json", by_alias=True, exclude_none=True) == { + "cursor": "next", + "limit": 10, + "parentId": "repo:FFXN/dify", + "view": "tree", + } + page = KnowledgeFSSourcePageResponse.model_validate( + {"pageId": "repo:FFXN/dify", "pageName": "dify", "hasChildren": True, "type": "repository"} + ) + assert page.model_dump(mode="json", exclude_none=True)["has_children"] is True + + def test_background_task_response_accepts_source_title_alias() -> None: task = KnowledgeFSBackgroundTaskResponse.model_validate( { diff --git a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts index 365faf6630f..d54b2bac152 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/types.gen.ts @@ -2595,6 +2595,7 @@ export type KnowledgeFsSourceFileResponse = { } export type KnowledgeFsSourcePageResponse = { + has_children?: boolean | null last_edited_time?: string | null page_id: string page_name: string @@ -5716,6 +5717,8 @@ export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesData = { query?: { cursor?: string limit?: number + parentId?: string + view?: string } url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/pages' } diff --git a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts index 182c5b83107..937e23c5289 100644 --- a/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts +++ b/packages/contracts/generated/api/console/knowledge-fs/zod.gen.ts @@ -3322,6 +3322,7 @@ export const zKnowledgeFsSourceFilesResponse = z.object({ * KnowledgeFSSourcePageResponse */ export const zKnowledgeFsSourcePageResponse = z.object({ + has_children: z.boolean().nullish(), last_edited_time: z.string().nullish(), page_id: z.string(), page_name: z.string(), @@ -4918,6 +4919,8 @@ export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesPath = z export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesQuery = z.object({ cursor: z.string().min(1).max(4096).optional(), limit: z.int().gte(1).lte(200).optional().default(50), + parentId: z.string().min(1).max(4096).optional(), + view: z.string().optional(), }) /** diff --git a/packages/contracts/generated/api/service/types.gen.ts b/packages/contracts/generated/api/service/types.gen.ts index 8ee9c8af723..91cdaa8a317 100644 --- a/packages/contracts/generated/api/service/types.gen.ts +++ b/packages/contracts/generated/api/service/types.gen.ts @@ -2881,6 +2881,7 @@ export type KnowledgeFsSourceListResponse = { } export type KnowledgeFsSourcePageResponse = { + has_children?: boolean | null last_edited_time?: string | null page_id: string page_name: string @@ -2891,6 +2892,8 @@ export type KnowledgeFsSourcePageResponse = { export type KnowledgeFsSourcePagesQuery = { cursor?: string | null limit?: number + parentId?: string | null + view?: 'tree' | null } export type KnowledgeFsSourcePagesResponse = { @@ -8622,6 +8625,8 @@ export type GetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesData = { query?: { cursor?: string limit?: number + parentId?: string + view?: string } url: '/knowledge-fs/spaces/{control_space_id}/sources/{source_id}/pages' } diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts index e8849681f01..66a45d4e0d4 100644 --- a/packages/contracts/generated/api/service/zod.gen.ts +++ b/packages/contracts/generated/api/service/zod.gen.ts @@ -3339,6 +3339,7 @@ export const zKnowledgeFsSourceImportResponse = z.object({ * KnowledgeFSSourcePageResponse */ export const zKnowledgeFsSourcePageResponse = z.object({ + has_children: z.boolean().nullish(), last_edited_time: z.string().nullish(), page_id: z.string(), page_name: z.string(), @@ -3352,6 +3353,8 @@ export const zKnowledgeFsSourcePageResponse = z.object({ export const zKnowledgeFsSourcePagesQuery = z.object({ cursor: z.string().min(1).max(4096).nullish(), limit: z.int().gte(1).lte(200).optional().default(50), + parentId: z.string().min(1).max(4096).nullish(), + view: z.literal('tree').nullish(), }) /** @@ -7788,6 +7791,8 @@ export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesPath = z export const zGetKnowledgeFsSpacesByControlSpaceIdSourcesBySourceIdPagesQuery = z.object({ cursor: z.string().min(1).max(4096).optional(), limit: z.int().gte(1).lte(200).optional().default(50), + parentId: z.string().min(1).max(4096).optional(), + view: z.string().optional(), }) /** diff --git a/web/features/new-rag/sources/create/__tests__/connected-source-workflow.spec.tsx b/web/features/new-rag/sources/create/__tests__/connected-source-workflow.spec.tsx index 66fdca64812..6b8d05b20b0 100644 --- a/web/features/new-rag/sources/create/__tests__/connected-source-workflow.spec.tsx +++ b/web/features/new-rag/sources/create/__tests__/connected-source-workflow.spec.tsx @@ -193,6 +193,44 @@ const notionDatasourcePluginWithParameters: DataSourceItem = { }, } +const githubProvider: KnowledgeFsSourceProviderResponse = { + ...notionProvider, + display_name: 'GitHub Repository', + id: 'github-provider', +} + +const githubCredential: DatasourceCredentialResponse = { + ...notionCredential, + id: 'github-credential-1', + name: 'GitHub token', +} + +const githubDatasourcePlugin: DataSourceItem = { + ...notionDatasourcePlugin, + declaration: { + ...notionDatasourcePlugin.declaration, + datasources: [ + { + ...notionDatasourcePlugin.declaration.datasources![0]!, + identity: { + ...notionDatasourcePlugin.declaration.datasources![0]!.identity, + label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' }, + name: 'github', + provider: 'github_datasource', + }, + }, + ], + identity: { + ...notionDatasourcePlugin.declaration.identity, + label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' }, + name: 'github_datasource', + }, + }, + plugin_id: 'langgenius/github_datasource', + plugin_unique_identifier: 'langgenius/github_datasource:0.4.12@local', + provider: 'github_datasource', +} + const s3Provider: KnowledgeFsSourceProviderResponse = { auth_kinds: ['endpoint'], available: true, @@ -307,6 +345,17 @@ function notionDatasourceAuth( } } +function githubDatasourceAuth(): DatasourceProviderAuthResponse { + return { + ...notionDatasourceAuth([githubCredential]), + label: { en_US: 'GitHub Repository', zh_Hans: 'GitHub 仓库' }, + name: 'github_datasource', + plugin_id: 'langgenius/github_datasource', + plugin_unique_identifier: 'langgenius/github_datasource:0.4.12@local', + provider: 'github_datasource', + } +} + function connectionResponse( overrides: Partial = {}, ): KnowledgeFsSourceConnectionResponse { @@ -853,6 +902,131 @@ describe('ConnectedSourceWorkflow', () => { expect(clientMock.deleteSource).not.toHaveBeenCalled() }) + it('loads GitHub tree branches on expansion and imports only explicitly selected pages', async () => { + const user = userEvent.setup() + const connection = connectionResponse({ + configuration: { + credentialId: githubCredential.id, + datasource: 'github', + pluginId: 'langgenius/github_datasource', + provider: 'github_datasource', + providerKind: 'online-document', + }, + name: 'GitHub token', + provider_id: githubProvider.id, + }) + clientMock.listProviders.mockResolvedValue({ data: [githubProvider] }) + clientMock.listDatasourcePlugins.mockResolvedValue([githubDatasourcePlugin]) + clientMock.listDatasourceAuth.mockResolvedValue({ result: [githubDatasourceAuth()] }) + clientMock.listConnections.mockResolvedValue({ data: [connection], next_cursor: null }) + clientMock.getPages.mockImplementation( + ({ query }: { query: { cursor?: string; parentId?: string } }) => { + const pages = query.parentId + ? query.cursor + ? [ + { + has_children: false, + page_id: 'issue:FFXN/dify:1', + page_name: 'Issue #1', + parent_id: 'repo:FFXN/dify', + type: 'issue', + }, + ] + : [ + { + has_children: false, + page_id: 'file:FFXN/dify:README.md', + page_name: 'README', + parent_id: 'repo:FFXN/dify', + type: 'file', + }, + ] + : query.cursor + ? [ + { + has_children: true, + page_id: 'repo:FFXN/checkin', + page_name: 'checkin', + parent_id: null, + type: 'repository', + }, + ] + : [ + { + has_children: true, + page_id: 'repo:FFXN/dify', + page_name: 'dify', + parent_id: null, + type: 'repository', + }, + ] + return Promise.resolve({ + next_cursor: !query.cursor ? (query.parentId ? 'branch-next' : 'root-next') : null, + workspaces: [ + { + pages, + total: pages.length, + workspace_id: 'FFXN', + workspace_name: "FFXN's GitHub", + }, + ], + }) + }, + ) + + const view = await renderSetup({ + ...defaultDraft, + provider: 'GitHub Repository', + sourceName: 'GitHub docs', + }) + const repository = await screen.findByRole('checkbox', { name: 'dify' }) + expect(clientMock.getPages).toHaveBeenCalledExactlyOnceWith({ + params: { control_space_id: 'space-1', source_id: 'preview-source' }, + query: { limit: 10, view: 'tree' }, + }) + expect(screen.queryByRole('checkbox', { name: 'README' })).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'knowledgeSpace.loadMore' })) + expect(await screen.findByRole('checkbox', { name: 'checkin' })).not.toBeChecked() + expect(clientMock.getPages).toHaveBeenCalledWith({ + params: { control_space_id: 'space-1', source_id: 'preview-source' }, + query: { cursor: 'root-next', limit: 10, view: 'tree' }, + }) + + await user.click(repository) + await user.click(screen.getByRole('button', { name: 'dify' })) + const readme = await screen.findByRole('checkbox', { name: 'README' }) + expect(clientMock.getPages).toHaveBeenCalledWith({ + params: { control_space_id: 'space-1', source_id: 'preview-source' }, + query: { limit: 200, parentId: 'repo:FFXN/dify', view: 'tree' }, + }) + expect(readme).not.toBeChecked() + expect(repository).toBePartiallyChecked() + + await user.click(screen.getByRole('button', { name: 'knowledgeSpace.loadMore' })) + const issue = await screen.findByRole('checkbox', { name: 'Issue #1' }) + expect(clientMock.getPages).toHaveBeenCalledWith({ + params: { control_space_id: 'space-1', source_id: 'preview-source' }, + query: { cursor: 'branch-next', limit: 200, parentId: 'repo:FFXN/dify', view: 'tree' }, + }) + expect(issue).not.toBeChecked() + + await user.click(repository) + expect(readme).toBeChecked() + expect(issue).toBeChecked() + await user.click(readme) + await user.click(issue) + await user.click(screen.getByRole('button', { name: 'knowledgeSpace.addSource' })) + await waitFor(() => expect(view.onCompleted).toHaveBeenCalledOnce()) + expect(clientMock.createWorkflowImport).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + items: [expect.objectContaining({ pageId: 'repo:FFXN/dify' })], + }), + }), + ) + }) + it('stops the import and cleans up the renamed source when canceled while the rename is pending', async () => { const user = userEvent.setup() clientMock.listDatasourceAuth.mockResolvedValue({ diff --git a/web/features/new-rag/sources/create/connected-source-workflow.tsx b/web/features/new-rag/sources/create/connected-source-workflow.tsx index f8b55a9fff0..d3964e4537f 100644 --- a/web/features/new-rag/sources/create/connected-source-workflow.tsx +++ b/web/features/new-rag/sources/create/connected-source-workflow.tsx @@ -24,7 +24,9 @@ import { Checkbox } from '@langgenius/dify-ui/checkbox' import { cn } from '@langgenius/dify-ui/cn' import { infiniteQueryOptions, + queryOptions, useInfiniteQuery, + useQueries, useQuery, useQueryClient, } from '@tanstack/react-query' @@ -107,9 +109,16 @@ type DriveBranch = { loaded: boolean loading: boolean } +type PageBranchRequest = { parentId: string; cursors: (string | null)[] } +type PageBranch = { + error: boolean + loading: boolean + nextCursor: string | null +} const CONNECTION_PAGE_SIZE = 200 const RESOURCE_PAGE_SIZE = 200 +const GITHUB_ROOT_PAGE_SIZE = 10 const MAX_SELECTION = 200 const MAX_SOURCE_CURSOR_PAGES = 100 @@ -324,6 +333,8 @@ function hasDriveFolderCheckbox(resource: SelectableResource) { } function resourceIcon(resource: SelectableResource) { + if (resource.kind === 'page' && resource.page.type === 'repository') + return 'i-ri-folder-3-fill text-text-warning' if (isFolder(resource) || isBucket(resource)) return 'i-ri-folder-3-fill text-text-warning' if (resource.kind === 'page' && /database/i.test(resource.page.type)) return 'i-ri-database-2-line text-text-accent' @@ -337,14 +348,18 @@ function ResourceList({ driveBranches, driveDescendantLeafKeys, expandedDriveContainers, + expandedPageKeys, folderSelectionIntents, hasMore, loadingMore, onLoadMore, + onLoadMorePageBranch, + onRetryPageBranch, onToggle, onToggleAll, onToggleExpanded, providerRegion, + pageBranches, resources, selectionAtLimit, selectionLimitId, @@ -352,20 +367,25 @@ function ResourceList({ selectionPending, selectionScope, selected, + treePreview, }: { collapsed: ReadonlySet connection: SourceConnection driveBranches: ReadonlyMap driveDescendantLeafKeys: ReadonlyMap expandedDriveContainers: ReadonlySet + expandedPageKeys: ReadonlySet folderSelectionIntents: ReadonlySet hasMore: boolean loadingMore: boolean onLoadMore: () => void + onLoadMorePageBranch: (resource: PageResource) => void + onRetryPageBranch: (resource: PageResource) => void onToggle: (resource: SelectableResource) => void onToggleAll: () => void onToggleExpanded: (resource: SelectableResource) => void providerRegion?: string + pageBranches: ReadonlyMap resources: SelectableResource[] selectionAtLimit: boolean selectionLimitId: string @@ -373,8 +393,9 @@ function ResourceList({ selectionPending: boolean selectionScope: SelectableResource[] selected: ReadonlyMap + treePreview: boolean }) { - const { t } = useTranslation(['knowledgeSpace', 'knowledgeSources']) + const { t } = useTranslation(['knowledgeSpace', 'knowledgeSources', 'common']) const selectableResources = selectionScope.filter(isSelectableResource) const driveContainers = selectionScope.filter(isDriveContainer) const allDriveContainersLoaded = driveContainers.every( @@ -426,6 +447,7 @@ function ResourceList({ (hasSelectionKeys && selectionKeys.every((key) => selected.has(key))) const indeterminate = !checked && selectionKeys.some((key) => selected.has(key)) const driveBranch = resource.kind === 'file' ? driveBranches.get(resource.key) : undefined + const pageBranch = resource.kind === 'page' ? pageBranches.get(resource.key) : undefined const emptyDriveFolder = hasDriveFolderCheckbox(resource) && driveBranch?.loaded && !selectionKeys.length const selectionBlocked = selectionAtLimit && !checked @@ -433,7 +455,9 @@ function ResourceList({ selectionBlocked || emptyDriveFolder || Boolean(driveBranch?.loading) const expanded = resource.kind === 'page' - ? !collapsed.has(resource.key) + ? treePreview + ? expandedPageKeys.has(resource.key) + : !collapsed.has(resource.key) : expandedDriveContainers.has(resource.key) const rendersCheckbox = resource.kind === 'page' || @@ -484,6 +508,37 @@ function ResourceList({ {resourceName(resource)} + {expanded && pageBranch?.loading && ( + $.loading, { ns: 'common' })} + className="i-ri-loader-4-line size-4 animate-spin text-text-tertiary" + /> + )} + {expanded && pageBranch?.error && resource.kind === 'page' && ( + + )} + {expanded && + !pageBranch?.loading && + !pageBranch?.error && + pageBranch?.nextCursor && + resource.kind === 'page' && ( + + )} {resource.kind === 'file' && resource.file.size != null && ( {formatFileSize(resource.file.size)} @@ -532,6 +587,7 @@ type PreviewSourceSession = { function ResourceConfiguration({ connection, draft, + githubTreePreview, knowledgeSpaceId, onCompleted, onDraftChange, @@ -542,6 +598,7 @@ function ResourceConfiguration({ }: { connection: SourceConnection draft: ConnectedSourceDraft + githubTreePreview: boolean knowledgeSpaceId: string onCompleted: () => void onDraftChange: (draft: NewKnowledgeSourceDraft) => void @@ -561,6 +618,10 @@ function ResourceConfiguration({ const [selected, setSelected] = useState>(() => new Map()) const selectedRef = useRef>(new Map()) const [collapsed, setCollapsed] = useState>(() => new Set()) + const [expandedPageKeys, setExpandedPageKeys] = useState>(() => new Set()) + const [pageBranchRequests, setPageBranchRequests] = useState>( + () => new Map(), + ) const [driveBranches, setDriveBranches] = useState>(() => new Map()) const [expandedDriveContainers, setExpandedDriveContainers] = useState>( () => new Set(), @@ -663,7 +724,13 @@ function ResourceConfiguration({ const pagesQuery = useInfiniteQuery( infiniteQueryOptions({ - queryKey: ['new-rag', 'connected-source-pages', knowledgeSpaceId, previewSource?.id], + queryKey: [ + 'new-rag', + 'connected-source-pages', + knowledgeSpaceId, + previewSource?.id, + githubTreePreview, + ], queryFn: ({ pageParam }) => consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.pages.get({ params: { @@ -671,7 +738,8 @@ function ResourceConfiguration({ source_id: previewSource?.id ?? '', }, query: { - limit: RESOURCE_PAGE_SIZE, + limit: githubTreePreview ? GITHUB_ROOT_PAGE_SIZE : RESOURCE_PAGE_SIZE, + ...(githubTreePreview ? { view: 'tree' as const } : {}), ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), }, }), @@ -681,6 +749,62 @@ function ResourceConfiguration({ retry: false, }), ) + const pageBranchEntries = useMemo( + () => + [...pageBranchRequests].flatMap(([resourceKey, request]) => + request.cursors.map((cursor) => ({ cursor, parentId: request.parentId, resourceKey })), + ), + [pageBranchRequests], + ) + const pageBranchQueries = useQueries({ + queries: pageBranchEntries.map(({ cursor, parentId }) => + queryOptions({ + queryKey: [ + 'new-rag', + 'connected-source-page-branch', + knowledgeSpaceId, + previewSource?.id, + parentId, + cursor, + ], + queryFn: () => + consoleClient.knowledgeFs.spaces.byControlSpaceId.sources.bySourceId.pages.get({ + params: { + control_space_id: knowledgeSpaceId, + source_id: previewSource?.id ?? '', + }, + query: { + limit: RESOURCE_PAGE_SIZE, + parentId, + view: 'tree', + ...(cursor ? { cursor } : {}), + }, + }), + enabled: githubTreePreview && Boolean(previewSource), + retry: false, + }), + ), + }) + const pageBranches = (() => { + const branches = new Map() + pageBranchEntries.forEach(({ resourceKey }, index) => { + const query = pageBranchQueries[index] + if (!query) return + const current = branches.get(resourceKey) + const nextCursor = query.data + ? (query.data.next_cursor ?? null) + : (current?.nextCursor ?? null) + branches.set(resourceKey, { + error: Boolean(current?.error || query.isError), + loading: Boolean(current?.loading || query.isPending || query.isFetching), + nextCursor: + nextCursor && !pageBranchRequests.get(resourceKey)?.cursors.includes(nextCursor) + ? nextCursor + : null, + }) + }) + return branches + })() const filesQuery = useInfiniteQuery( infiniteQueryOptions({ queryKey: ['new-rag', 'connected-source-files', knowledgeSpaceId, previewSource?.id], @@ -720,12 +844,16 @@ function ResourceConfiguration({ }), ) - const pageResources = useMemo(() => { + const pageResources: PageResource[] = (() => { const groups = new Map< string, { name: string; pages: Map } >() - for (const response of pagesQuery.data?.pages ?? []) { + const responses = [ + ...(pagesQuery.data?.pages ?? []), + ...pageBranchQueries.flatMap((query) => (query.data ? [query.data] : [])), + ] + for (const response of responses) { for (const workspace of response.workspaces) { const groupId = workspace.workspace_id?.trim() || 'workspace' const group = groups.get(groupId) ?? { @@ -768,7 +896,27 @@ function ResourceConfiguration({ } return descendants } - return [...group.pages.values()].map((page) => { + const orderedPages = [...group.pages.values()] + if (githubTreePreview) { + const byId = group.pages + const visited = new Set() + const treeOrder: KnowledgeFsSourcePageResponse[] = [] + const visit = (page: KnowledgeFsSourcePageResponse) => { + if (visited.has(page.page_id)) return + visited.add(page.page_id) + treeOrder.push(page) + for (const childId of children.get(page.page_id) ?? []) { + const child = byId.get(childId) + if (child) visit(child) + } + } + for (const page of orderedPages) { + if (!page.parent_id || !byId.has(page.parent_id)) visit(page) + } + for (const page of orderedPages) visit(page) + orderedPages.splice(0, orderedPages.length, ...treeOrder) + } + return orderedPages.map((page) => { const ancestorKeys = ancestorsFor(page) return { ancestorKeys, @@ -776,14 +924,14 @@ function ResourceConfiguration({ descendantKeys: descendantsFor(page.page_id), groupId, groupName: group.name, - hasChildren: Boolean(children.get(page.page_id)?.length), + hasChildren: Boolean(page.has_children || children.get(page.page_id)?.length), key: keyFor(page.page_id), kind: 'page' as const, page, } }) }) - }, [connection.name, pagesQuery.data?.pages]) + })() const rootFileResources = useMemo(() => { const resourcesByKey = new Map() for (const response of filesQuery.data?.pages ?? []) { @@ -966,9 +1114,19 @@ function ResourceConfiguration({ driveTransport ? visibleDriveResources : pageResources.filter( - (resource) => !resource.ancestorKeys.some((ancestorKey) => collapsed.has(ancestorKey)), + (resource) => + !resource.ancestorKeys.some((ancestorKey) => + githubTreePreview ? !expandedPageKeys.has(ancestorKey) : collapsed.has(ancestorKey), + ), ), - [collapsed, driveTransport, pageResources, visibleDriveResources], + [ + collapsed, + driveTransport, + expandedPageKeys, + githubTreePreview, + pageResources, + visibleDriveResources, + ], ) const queryPending = driveTransport ? filesQuery.isPending : pagesQuery.isPending const queryError = driveTransport ? filesQuery.error : pagesQuery.error @@ -1161,6 +1319,25 @@ function ResourceConfiguration({ const toggleExpanded = (resource: SelectableResource) => { if (resource.kind === 'page') { + if (githubTreePreview) { + const expanding = !expandedPageKeys.has(resource.key) + setExpandedPageKeys((current) => { + const next = new Set(current) + if (expanding) next.add(resource.key) + else next.delete(resource.key) + return next + }) + if (expanding) { + setPageBranchRequests((current) => { + if (current.has(resource.key)) return current + return new Map(current).set(resource.key, { + parentId: resource.page.page_id, + cursors: [null], + }) + }) + } + return + } setCollapsed((current) => { const next = new Set(current) if (next.has(resource.key)) next.delete(resource.key) @@ -1179,6 +1356,26 @@ function ResourceConfiguration({ if (expanding) void loadDriveChildren(resource).catch(() => undefined) } + const loadMorePageBranch = (resource: PageResource) => { + const nextCursor = pageBranches.get(resource.key)?.nextCursor + if (!nextCursor) return + setPageBranchRequests((current) => { + const request = current.get(resource.key) + if (!request || request.cursors.includes(nextCursor)) return current + return new Map(current).set(resource.key, { + ...request, + cursors: [...request.cursors, nextCursor], + }) + }) + } + + const retryPageBranch = (resource: PageResource) => { + pageBranchEntries.forEach(({ resourceKey }, index) => { + if (resourceKey === resource.key && pageBranchQueries[index]?.isError) + void pageBranchQueries[index].refetch() + }) + } + const loadMore = () => { if (driveTransport) void filesQuery.fetchNextPage() else void pagesQuery.fetchNextPage() @@ -1422,14 +1619,18 @@ function ResourceConfiguration({ driveBranches={driveBranches} driveDescendantLeafKeys={driveDescendantLeafKeys} expandedDriveContainers={expandedDriveContainers} + expandedPageKeys={expandedPageKeys} folderSelectionIntents={folderSelectionIntents} hasMore={hasMore} loadingMore={loadingMore} onLoadMore={loadMore} + onLoadMorePageBranch={loadMorePageBranch} + onRetryPageBranch={retryPageBranch} onToggle={toggle} onToggleAll={toggleAll} onToggleExpanded={toggleExpanded} providerRegion={providerRegion} + pageBranches={pageBranches} resources={visibleResources} selectionAtLimit={selectionAtLimit} selectionLimitId={selectionLimitId} @@ -1437,6 +1638,7 @@ function ResourceConfiguration({ selectionPending={selectionPending} selectionScope={resources} selected={selected} + treePreview={githubTreePreview} /> )} @@ -1475,6 +1677,7 @@ function ResourceConfiguration({ function AppliedResourceConfiguration({ connection, draft, + githubTreePreview, knowledgeSpaceId, onCompleted, onDraftChange, @@ -1487,6 +1690,7 @@ function AppliedResourceConfiguration({ }: { connection: SourceConnection draft: ConnectedSourceDraft + githubTreePreview: boolean knowledgeSpaceId: string onCompleted: () => void onDraftChange: (draft: NewKnowledgeSourceDraft) => void @@ -1534,6 +1738,7 @@ function AppliedResourceConfiguration({