diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go index 9e0d02bb96..16ea8d1d9a 100644 --- a/internal/handler/dataset_artifact.go +++ b/internal/handler/dataset_artifact.go @@ -255,13 +255,25 @@ func (h *DatasetArtifactHandler) GetArtifactAlteration(c *gin.Context) { } // GetArtifactGraph handles GET /artifacts/graph — wiki entity/relation graph. +// keywords seeds the graph with BM25 matches; top_n/topN controls the entity cap. func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) { _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) if tenantID == "" { return } datasetID := c.Param("dataset_id") - graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID) + keywords := strings.TrimSpace(c.Query("keywords")) + topNValue := c.Query("top_n") + if topNValue == "" { + topNValue = c.Query("topN") + } + var topN *int + if topNValue != "" { + if value, parseErr := strconv.Atoi(topNValue); parseErr == nil { + topN = &value + } + } + graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID, keywords, topN) if err != nil { common.ErrorWithCode(c, common.CodeDataError, err.Error()) return diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go index 8b53367891..ce0054b767 100644 --- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go +++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go @@ -714,12 +714,108 @@ func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) { } func (p *wikiPipeline) runPlan() (wikiPlan, error) { + if p.wikiMode() == "entity" { + return p.runEntityPlan() + } if p.wikiMode() == "topic" && p.deps.Embed != nil { return p.runTopicPlan() } return p.runLegacyPlan() } +// runEntityPlan builds one page for every reduced entity or concept. Entity +// mode already has a canonical identity from MAP/REDUCE, so a PLAN LLM call +// would only repeat fields that the compiler can derive deterministically. +func (p *wikiPipeline) runEntityPlan() (wikiPlan, error) { + if err := p.ctx.Err(); err != nil { + return wikiPlan{}, err + } + + plan := wikiPlan{Pages: buildWikiEntityPlanPages(p.reduced)} + plan.Pages = assembleWikiPlanRelatedPages(plan.Pages, p.reduced.Relations) + plan.Pages = normalizeWikiPlanPages(plan.Pages, p.reduced) + plan.Pages = normalizeWikiPlanPageLinks(plan.Pages) + reconciled, err := p.reconcilePlan(plan) + if err != nil { + return wikiPlan{}, err + } + reconciled.Pages = normalizeWikiPlanPageLinks(reconciled.Pages) + return reconciled, nil +} + +// buildWikiEntityPlanPages projects MAP/REDUCE identities into deterministic +// page metadata. Topic assignment uses source-chunk overlap with MAP topics; +// the page itself remains one-to-one with the extracted entity or concept. +func buildWikiEntityPlanPages(reduced wikiExtract) []wikiPlanPage { + pages := make([]wikiPlanPage, 0, len(reduced.Entities)+len(reduced.Concepts)) + for _, entity := range reduced.Entities { + name := strings.TrimSpace(entity.Name) + if name == "" { + continue + } + pages = append(pages, wikiPlanPage{ + Action: "CREATE", + Slug: entityPageSlug(name, entity.Type), + Title: name, + PageType: "entity", + Topic: selectWikiIdentityTopic(name, "entity", sourceChunkIDsForWikiIdentity(name, entity.SourceChunkIDs, reduced.Claims), reduced.Topics), + EntityNames: uniqueStrings(append([]string{name}, entity.Aliases...)), + Priority: len(pages) + 1, + }) + } + for _, concept := range reduced.Concepts { + term := strings.TrimSpace(concept.Term) + if term == "" { + continue + } + pages = append(pages, wikiPlanPage{ + Action: "CREATE", + Slug: "concept/" + slugify(term), + Title: term, + PageType: "concept", + Topic: selectWikiIdentityTopic(term, "concept", sourceChunkIDsForWikiIdentity(term, concept.SourceChunkIDs, reduced.Claims), reduced.Topics), + EntityNames: []string{term}, + Priority: len(pages) + 1, + }) + } + return pages +} + +func sourceChunkIDsForWikiIdentity(name string, sourceChunkIDs []string, claims []wikiClaim) []string { + ids := append([]string(nil), sourceChunkIDs...) + for _, claim := range claims { + if normKey(claim.Subject) == normKey(name) { + ids = append(ids, claim.SourceChunkIDs...) + } + } + return uniqueStrings(ids) +} + +func selectWikiIdentityTopic(name, pageType string, sourceChunkIDs []string, topics []wikiTopic) string { + identityChunks := normalizedStringSet(sourceChunkIDs) + bestPath := "" + bestOverlap := 0 + for _, topic := range topics { + path := common.NormalizeWikiTopicPath(topic.Path) + if path == "" || (pageType != "topic" && normKey(common.WikiTopicLeaf(path)) == normKey(name)) { + continue + } + overlap := normalizedOverlapCount(identityChunks, topic.SourceChunkIDs) + if overlap == 0 || overlap < bestOverlap { + continue + } + if overlap == bestOverlap && bestPath != "" && normKey(path) >= normKey(bestPath) { + continue + } + bestPath = path + bestOverlap = overlap + } + if bestPath == "" { + return common.GeneralWikiTopic + } + return bestPath +} + func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) { batches := packWikiPlanBatches(p.reduced, wikiPlanTokenBudget) if len(batches) == 0 { diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go index f67d2423ca..d3054f1250 100644 --- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go +++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go @@ -235,7 +235,8 @@ func TestRunPlan_PromptMaxPagesNeverExceedsCap(t *testing.T) { entities = append(entities, wikiEntity{Name: "Ent " + itoa(i) + big}) } p := &wikiPipeline{ - ctx: context.Background(), + ctx: context.Background(), + param: topicModeParam(), deps: common.Deps{ ModelContextLen: 1024, Chat: chatFunc(func(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) { @@ -352,7 +353,8 @@ func TestRunPlan_TruncatesToGlobalHardCap(t *testing.T) { } payload := map[string]any{"pages": pages} p := &wikiPipeline{ - ctx: context.Background(), + ctx: context.Background(), + param: topicModeParam(), deps: common.Deps{ Chat: reconcileChatStub{resp: mustJSON(payload)}, }, @@ -440,7 +442,8 @@ func TestRunPlan_ParallelBatchesMergeInOrder(t *testing.T) { // batch2 (token budget 3500). big := strings.Repeat("x", 7000) p := &wikiPipeline{ - ctx: context.Background(), + ctx: context.Background(), + param: topicModeParam(), deps: common.Deps{ Chat: batchPlanChatStub{}, }, @@ -499,7 +502,8 @@ func TestRunPlan_ParallelBatchesFirstError(t *testing.T) { big := strings.Repeat("x", 7000) boom := errors.New("planning failed") p := &wikiPipeline{ - ctx: context.Background(), + ctx: context.Background(), + param: topicModeParam(), deps: common.Deps{ Chat: failPlanChatStub{err: boom}, }, @@ -548,7 +552,8 @@ func TestRunPlan_CancelledCtxAborts(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() p := &wikiPipeline{ - ctx: ctx, + ctx: ctx, + param: topicModeParam(), deps: common.Deps{ Chat: batchPlanChatStub{}, }, @@ -565,3 +570,8 @@ func TestRunPlan_CancelledCtxAborts(t *testing.T) { t.Fatalf("runPlan err = nil, want context cancelled") } } + +func topicModeParam() common.Param { + enabled := true + return common.Param{Plan: &enabled} +} diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go index 1430c5abc5..d1a0ea5c27 100644 --- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go +++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go @@ -34,6 +34,69 @@ func TestReduceExtracts_MergesProvenance(t *testing.T) { } } +func TestRunPlan_EntityModeBuildsPagesFromReducedExtract(t *testing.T) { + p := &wikiPipeline{ + ctx: context.Background(), + reduced: wikiExtract{ + Entities: []wikiEntity{{ + Name: "刘备", + Type: "person", + Aliases: []string{"玄德"}, + SourceChunkIDs: []string{"c1"}, + }}, + Concepts: []wikiConcept{{ + Term: "三国演义", + Definition: "历史小说", + SourceChunkIDs: []string{"c2"}, + }}, + Claims: []wikiClaim{ + {Subject: "刘备", SourceChunkIDs: []string{"c1"}}, + {Subject: "三国演义", SourceChunkIDs: []string{"c2"}}, + }, + Relations: []wikiRelation{{From: "刘备", To: "三国演义", Type: "appears_in"}}, + Topics: []wikiTopic{ + {Path: "历史/人物", SourceChunkIDs: []string{"c1"}}, + {Path: "历史/作品", SourceChunkIDs: []string{"c2"}}, + }, + }, + docID: "doc-1", + } + + plan, err := p.runPlan() + if err != nil { + t.Fatalf("runPlan err = %v", err) + } + if len(plan.Pages) != 2 { + t.Fatalf("plan pages = %d, want 2", len(plan.Pages)) + } + bySlug := make(map[string]wikiPlanPage, len(plan.Pages)) + for _, page := range plan.Pages { + bySlug[page.Slug] = page + } + + entityPage, ok := bySlug["entity/person/刘备"] + if !ok { + t.Fatalf("entity page missing: %#v", plan.Pages) + } + if entityPage.Title != "刘备" || entityPage.PageType != "entity" || entityPage.Topic != "历史/人物" { + t.Fatalf("entity page metadata = %#v", entityPage) + } + if !slices.Equal(entityPage.EntityNames, []string{"刘备", "玄德"}) { + t.Fatalf("entity page names = %#v", entityPage.EntityNames) + } + if !slices.Equal(entityPage.RelatedKB, []string{"concept/三国演义"}) { + t.Fatalf("entity related pages = %#v", entityPage.RelatedKB) + } + + conceptPage, ok := bySlug["concept/三国演义"] + if !ok { + t.Fatalf("concept page missing: %#v", plan.Pages) + } + if conceptPage.Topic != "历史/作品" || !slices.Equal(conceptPage.RelatedKB, []string{"entity/person/刘备"}) { + t.Fatalf("concept page metadata = %#v", conceptPage) + } +} + func TestParseWikiExtractNormalizesTopicPathAndProvenance(t *testing.T) { extract := parseWikiExtract(map[string]any{ "topics": []any{map[string]any{ diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go index a0e39e3c82..deb3c537e8 100644 --- a/internal/ingestion/knowledge_compile/wiki_graph_test.go +++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go @@ -18,6 +18,7 @@ package knowledge_compile import ( "context" "encoding/json" + "strings" "testing" ) @@ -51,6 +52,27 @@ func findRelation(rows []map[string]interface{}, from, to string) map[string]int const kbForTest = "kb1" +func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) { + w := engineWriter{} + p := page("entity", "刘备") + p.Summary = "三国演义中的人物" + + rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{p}) + if err != nil { + t.Fatalf("project: %v", err) + } + entity := findEntity(rows, "entity/刘备") + if entity == nil { + t.Fatal("missing entity") + } + for _, field := range []string{"title_tks", "title_sm_tks", "content_ltks", "content_sm_ltks"} { + value, ok := entity[field].(string) + if !ok || strings.TrimSpace(value) == "" { + t.Fatalf("entity[%q] = %#v, want non-empty search content", field, entity[field]) + } + } +} + // TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two // endpoints land in different "batches" (here modeled as two pages) is still // materialized: the projection is global over the full page set, so edges never diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go index de2154d7c1..d7f1aa3084 100644 --- a/internal/ingestion/knowledge_compile/writer.go +++ b/internal/ingestion/knowledge_compile/writer.go @@ -34,6 +34,7 @@ import ( "ragflow/internal/entity" kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common" "ragflow/internal/service/file" + "ragflow/internal/tokenizer" "ragflow/internal/utility" ) @@ -1385,6 +1386,35 @@ func wikiGraphBareKey(slug string) string { return strings.TrimSpace(s) } +// tokenizeWikiGraphContent prepares the lexical fields used by wiki graph +// keyword search. Keep the raw text as a fallback so graph rows remain +// searchable when the tokenizer pool is unavailable during startup or tests. +func tokenizeWikiGraphContent(title, slug, description string) (string, string, string, string) { + title = strings.TrimSpace(title) + content := strings.TrimSpace(strings.Join([]string{slug, description}, " ")) + if title == "" && content == "" { + return "", "", "", "" + } + + titleLTKS, err := tokenizer.Tokenize(title) + if err != nil || titleLTKS == "" { + titleLTKS = title + } + titleSMLTKS, err := tokenizer.FineGrainedTokenize(titleLTKS) + if err != nil || titleSMLTKS == "" { + titleSMLTKS = titleLTKS + } + contentLTKS, err := tokenizer.Tokenize(content) + if err != nil || contentLTKS == "" { + contentLTKS = content + } + contentSMLTKS, err := tokenizer.FineGrainedTokenize(contentLTKS) + if err != nil || contentSMLTKS == "" { + contentSMLTKS = contentLTKS + } + return titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS +} + // wikiPageProjection is the subset of a merged wiki_page row that the graph // projection needs. It is reconstructed from the stored display columns (the // same fields GetWikiGraph reads back), not from the JSON payload. @@ -1799,6 +1829,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, if err != nil { return nil, err } + titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Title, p.Slug, p.Summary) rows = append(rows, map[string]interface{}{ "id": entityID, "doc_id": kb, @@ -1815,6 +1846,10 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, "weight_int": weight, "source_chunk_ids": p.SourceChunkIDs, "source_doc_ids": capSourceDocs(p.SourceDocIDs), + "title_tks": titleLTKS, + "title_sm_tks": titleSMLTKS, + "content_ltks": contentLTKS, + "content_sm_ltks": contentSMLTKS, "content_with_weight": string(content), }) diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go index 12cb35c739..ef20e5161b 100644 --- a/internal/service/dataset_artifact_service.go +++ b/internal/service/dataset_artifact_service.go @@ -42,6 +42,8 @@ const ( CompileKwdSkill = "skill" CompileKwdSkillAll = "skill_all" CompileKwdDatasetNav = "dataset_nav" + defaultWikiGraphTopN = 512 + maxWikiGraphTopN = 1024 ) // DatasetArtifactService reads knowledge-compilation artifacts (wiki pages, @@ -61,6 +63,10 @@ func wikiIndexName(tenantID string) string { // searchCompiled runs a filtered search over the tenant's document index for // the given dataset, returning the matching chunks. func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr) ([]map[string]interface{}, int64, error) { + return s.searchCompiledWithMatch(ctx, tenantID, datasetID, filter, selectFields, offset, limit, orderBy, nil) +} + +func (s *DatasetArtifactService) searchCompiledWithMatch(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr, matchExprs []interface{}) ([]map[string]interface{}, int64, error) { docEngine := engine.Get() if docEngine == nil { return nil, 0, fmt.Errorf("document engine is not initialized") @@ -80,6 +86,7 @@ func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, d SelectFields: selectFields, Filter: merged, OrderBy: orderBy, + MatchExprs: matchExprs, }) if err != nil { return nil, 0, err @@ -544,8 +551,12 @@ var wikiTopicCollator = collate.New(language.Chinese) // WikiGraph is the entity/relation graph for a dataset's wiki artifacts. type WikiGraph struct { - Entities []WikiGraphEntity `json:"entities"` - Relations []WikiGraphRelation `json:"relations"` + Entities []WikiGraphEntity `json:"entities"` + Relations []WikiGraphRelation `json:"relations"` + TotalEntities int `json:"total_entities"` + TotalRelations int `json:"total_relations"` + ReturnedEntities int `json:"returned_entities"` + ReturnedRelations int `json:"returned_relations"` } // WikiGraphEntity is a single wiki graph entity. @@ -565,12 +576,47 @@ type WikiGraphRelation struct { To string `json:"to"` } -// GetWikiGraph returns the wiki entity/relation graph for a dataset. -func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) { - entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, - map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}, - []string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"}, - 0, 1000, (&types.OrderByExpr{}).Desc("weight_int")) +// GetWikiGraph returns an incremental wiki graph slice for a dataset. keywords +// seeds the overview with BM25 entity matches; topN overrides the entity cap. +func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID, keywords string, topN *int) (*WikiGraph, error) { + entityFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1} + relationFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1} + _, totalEntities, err := s.searchCompiled(ctx, tenantID, datasetID, entityFilter, []string{"id"}, 0, 1, nil) + if err != nil { + return nil, err + } + _, totalRelations, err := s.searchCompiled(ctx, tenantID, datasetID, relationFilter, []string{"id"}, 0, 1, nil) + if err != nil { + return nil, err + } + + limit := defaultWikiGraphTopN + if topN != nil { + limit = *topN + if limit < 1 { + limit = 1 + } + if limit > maxWikiGraphTopN { + limit = maxWikiGraphTopN + } + } + + keywords = strings.TrimSpace(keywords) + var matchExprs []interface{} + var orderBy *types.OrderByExpr + if keywords != "" { + matchExprs = []interface{}{&types.MatchTextExpr{ + Fields: []string{"title_tks^10", "title_sm_tks^5", "content_ltks^2", "content_sm_ltks"}, + MatchingText: keywords, + TopN: limit, + ExtraOptions: map[string]interface{}{"original_query": keywords}, + }} + } else { + orderBy = (&types.OrderByExpr{}).Desc("weight_int") + } + entityChunks, _, err := s.searchCompiledWithMatch(ctx, tenantID, datasetID, entityFilter, + []string{"id", "slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"}, + 0, limit, orderBy, matchExprs) if err != nil { return nil, err } @@ -578,7 +624,12 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat for _, c := range entityChunks { fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"])) } - graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}} + graph := &WikiGraph{ + Entities: []WikiGraphEntity{}, + Relations: []WikiGraphRelation{}, + TotalEntities: int(totalEntities), + TotalRelations: int(totalRelations), + } for _, c := range entityChunks { w := intValue(c["weight_int"]) // Match the GetWikiPage / ListWikiPages contract: expose the bare slug @@ -605,8 +656,9 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat }) } if len(fromKwds) > 0 { + relationFilter["from_kwd"] = fromKwds relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, - map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1, "from_kwd": fromKwds}, + relationFilter, []string{"from_kwd", "to_kwd"}, 0, 10000, nil) if err != nil { return nil, err @@ -621,6 +673,8 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat }) } } + graph.ReturnedEntities = len(graph.Entities) + graph.ReturnedRelations = len(graph.Relations) return graph, nil } diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go index aa58a3639f..dd686cfb96 100644 --- a/internal/service/dataset_structure_graph.go +++ b/internal/service/dataset_structure_graph.go @@ -25,7 +25,7 @@ import ( const ( graphFullThreshold = 1024 // below this combined count, return all rows graphTopEntities = 256 // seed set A size for large buckets - graphKeywordCandidates = 16 // keyword/KNN candidate rows + graphKeywordCandidates = 16 // keyword candidate rows graphExpansionCap = 4096 // hub-node expansion cap ) @@ -684,65 +684,6 @@ func structureKindForBucket(kind string) string { return strings.ReplaceAll(k, "-", "_") } -// loadDocumentTemplateMeta resolves the doc's configured template group into -// configured_ids + template_meta maps, mirroring the Python resolution. -func (s *DatasetArtifactService) loadDocumentTemplateMeta(ctx context.Context, tenantID, documentID string) ([]string, map[string]map[string]interface{}, error) { - empty := map[string]map[string]interface{}{} - doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, documentID) - if err != nil || doc == nil { - return nil, empty, err - } - groupID := "" - if doc.ParserConfig != nil { - if gid, ok := doc.ParserConfig["compilation_template_group_id"].(string); ok && gid != "" { - groupID = gid - } else if ext, ok := doc.ParserConfig["ext"].(map[string]interface{}); ok { - if gid, ok := ext["compilation_template_group_id"].(string); ok && gid != "" { - groupID = gid - } - } - } - if groupID == "" { - return nil, empty, nil - } - group, err := NewCompilationTemplateGroupService().GetSaved(ctx, tenantID, groupID) - if err != nil { - return nil, empty, err - } - configuredIDs := []string{} - meta := map[string]map[string]interface{}{} - seen := map[string]bool{} - if group != nil { - for _, t := range group.Templates { - tid := t.ID - if tid == "" || seen[tid] { - continue - } - rawKind := "" - if t.Config != nil { - if k, ok := t.Config["kind"].(string); ok { - rawKind = k - } - } - if rawKind == "" { - rawKind = t.Kind - } - kindNorm := compilationTemplateKind(rawKind) - if kindNorm == "wiki" { - continue - } - seen[tid] = true - configuredIDs = append(configuredIDs, tid) - meta[tid] = map[string]interface{}{ - "template_id": tid, - "template_name": t.Name, - "kind": rawKind, - } - } - } - return configuredIDs, meta, nil -} - // DocumentStructureGraphInput is the parsed request for GetDocumentGraph. type DocumentStructureGraphInput struct { TenantID string @@ -755,10 +696,8 @@ type DocumentStructureGraphInput struct { // GetDocumentGraph returns the per-template structure graph of a document, // mirroring Python get_document_structure_graph (normal + keywords modes). func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumentStructureGraphInput) (*DocumentStructureGraphResponse, error) { - configuredIDs, templateMeta, err := s.loadDocumentTemplateMeta(ctx, in.TenantID, in.DocumentID) - if err != nil { - return nil, err - } + templateIDs := []string{} + templateMeta := map[string]map[string]interface{}{} resp := &DocumentStructureGraphResponse{Templates: []DocumentStructureGraphTemplate{}} entityCountFilter := map[string]interface{}{ @@ -771,13 +710,17 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume } resp.TotalEntities = int(entityTotal) - // keywords mode: name matching/KNN → matched entities' subgraph. + // keywords mode: name/BM25 matching → matched entities' subgraph. if in.Keywords != "" { bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, in.DocumentID, in.Keywords, templateMeta, nil) if err != nil { return nil, err } if bucketMeta == nil || (len(entities) == 0 && len(relations) == 0) { + if err := s.discoverDocumentGraphTemplateMeta(ctx, in.TenantID, in.DatasetID, in.DocumentID, &templateIDs, templateMeta); err != nil { + return nil, err + } + resp.Templates = buildDocumentGraphTemplateShells(templateIDs, templateMeta) return resp, nil } resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{ @@ -817,10 +760,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume bucketScopes := map[string]map[string]interface{}{} templateDAO := dao.NewCompilationTemplateDAO() for _, row := range metaRows { - // Pipeline-produced rows may contain a template id even when the - // document parser config does not contain the corresponding template - // group. Resolve that id directly so template_name is the user-visible - // name instead of the opaque id. + // Resolve template metadata from the persisted graph row. The template + // name is loaded only as display metadata; the row remains authoritative + // for whether the template exists in this document. templateID := rowTemplateID(row) if templateID != "" { if _, known := templateMeta[templateID]; !known { @@ -836,6 +778,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume } meta, scope := resolveGraphBucket(row, templateMeta, in.DocumentID) bid := graphStr(meta["template_id"]) + if bid != "" && !containsStr(templateIDs, bid) { + templateIDs = append(templateIDs, bid) + } if _, ok := bucketMetas[bid]; !ok { bucketMetas[bid] = meta bucketScopes[bid] = scope @@ -859,9 +804,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume } } - // Order: configured templates first, then discovered. + // Order: discovered templates first, then any buckets not seen in the scan. orderedIDs := []string{} - for _, tid := range configuredIDs { + for _, tid := range templateIDs { if _, ok := grouped[tid]; ok && !containsStr(orderedIDs, tid) { orderedIDs = append(orderedIDs, tid) } @@ -889,6 +834,93 @@ func containsStr(list []string, v string) bool { return false } +func buildDocumentGraphTemplateShells(templateIDs []string, templateMeta map[string]map[string]interface{}) []DocumentStructureGraphTemplate { + templates := make([]DocumentStructureGraphTemplate, 0, len(templateIDs)) + for _, templateID := range templateIDs { + meta := templateMeta[templateID] + if meta == nil { + continue + } + templateName := graphStr(meta["template_name"]) + if templateName == "" { + templateName = templateID + } + templates = append(templates, DocumentStructureGraphTemplate{ + TemplateID: templateID, + TemplateName: templateName, + Kind: graphStr(meta["kind"]), + Entities: []StructureGraphNode{}, + Relations: []StructureGraphRelation{}, + }) + } + return templates +} + +func (s *DatasetArtifactService) discoverDocumentGraphTemplateMeta(ctx context.Context, tenantID, datasetID, documentID string, templateIDs *[]string, templateMeta map[string]map[string]interface{}) error { + if templateIDs == nil || templateMeta == nil { + return nil + } + seen := make(map[string]bool, len(*templateIDs)) + for _, templateID := range *templateIDs { + seen[templateID] = true + } + templateDAO := dao.NewCompilationTemplateDAO() + fields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"} + filter := map[string]interface{}{ + "doc_id": []string{documentID}, + "knowledge_graph_kwd": []string{"entity", "relation"}, + } + for offset := 0; ; offset += 1000 { + rows, total, err := graphRowSearch(ctx, tenantID, datasetID, fields, filter, nil, offset, 1000, nil) + if err != nil { + return err + } + for _, row := range rows { + templateID := rowTemplateID(row) + kind := firstStringValue(row["compilation_template_kind_kwd"]) + if kind == "" { + kind = firstStringValue(row["compile_kwd"]) + } + if templateID == "" { + compileKind := firstStringValue(row["compile_kwd"]) + if compileKind == "" { + continue + } + templateID = "legacy:" + compileKind + } + if seen[templateID] { + continue + } + + templateName := templateID + if !strings.HasPrefix(templateID, "legacy:") { + if template, loadErr := templateDAO.GetTemplate(ctx, dao.DB, tenantID, templateID); loadErr == nil && template != nil { + templateName = template.Name + if kind == "" { + kind = template.Kind + } + } + } else { + templateName = "Legacy (" + strings.TrimPrefix(templateID, "legacy:") + ")" + } + if templateName == "" { + templateName = templateID + } + templateMeta[templateID] = map[string]interface{}{ + "template_id": templateID, + "template_name": templateName, + "kind": kind, + } + *templateIDs = append(*templateIDs, templateID) + seen[templateID] = true + } + if int64(offset+1000) >= total || len(rows) == 0 { + break + } + } + return nil +} + // DatasetStructureGraphInput is the parsed request for the dataset-scope // structure graph endpoint (GET /datasets/:id/artifacts/structure). // Kind is REQUIRED (mirrors Python dataset_api.py:715): missing or invalid → @@ -922,8 +954,7 @@ type DatasetStructureGraphResponse struct { // way). It collects distinct template ids, then reads each template's dataset // entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/ // delete-side only). When Keywords is set, it returns the keyword-matched -// entity subgraph using the same BM25/KNN and relation expansion path as the -// document-level endpoint. +// entity subgraph using the name/BM25 and relation expansion path. func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) { resolved := resolveDatasetStructureKind(in.Kind) if resolved == "" { @@ -1189,7 +1220,8 @@ func nameMatchesQuery(node StructureGraphNode, query string) bool { return true } -// keywordSubgraph mirrors sgc.keyword_subgraph (BM25 + KNN fallback + ancestor walk). +// keywordSubgraph mirrors sgc.keyword_subgraph (lexical lookup and ancestor +// walk). func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, datasetID, documentID, keywords string, templateMeta map[string]map[string]interface{}, baseEntityCond map[string]interface{}) (map[string]interface{}, []StructureGraphNode, []StructureGraphRelation, error) { if baseEntityCond == nil { baseEntityCond = map[string]interface{}{ @@ -1208,10 +1240,8 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, resolveBucket := func(row map[string]interface{}) (map[string]interface{}, map[string]interface{}) { templateID := rowTemplateID(row) if templateID != "" && !templateMetaLoaded[templateID] { - // Pipeline-produced rows can carry a template id even when the - // document parser config does not contain the corresponding group. - // Resolve it here as well as in the normal path so keyword searches - // expose the user-visible template name instead of the opaque id. + // Resolve the display metadata from the persisted template id so + // keyword searches expose the user-visible name instead of the id. templateMetaLoaded[templateID] = true if template, err := templateDAO.GetTemplate(ctx, dao.DB, tenantID, templateID); err == nil && template != nil { templateMeta[templateID] = map[string]interface{}{ @@ -1323,26 +1353,6 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, candidates = detailCandidates } - // Semantic fallback via KNN. - if len(candidates) == 0 { - vec, err := embedQuery(ctx, tenantID, datasetID, keywords) - if err != nil || len(vec) == 0 { - return nil, nil, nil, nil - } - denseExpr := &types.MatchDenseExpr{ - VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)), - EmbeddingData: vec, - EmbeddingDataType: "float", - DistanceType: "cosine", - TopN: graphKeywordCandidates, - ExtraOptions: map[string]interface{}{"similarity": 0.3}, - } - topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{denseExpr}) - if err != nil { - return nil, nil, nil, err - } - candidates = validTop(topMap) - } if len(candidates) == 0 { return nil, nil, nil, nil } @@ -1498,19 +1508,3 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, } return bucketMeta, entities, normalizeRelationEndpoints(entities, relations), nil } - -// embedQuery embeds a query via the nav embedder (best-effort). The embedder -// resolves the tenant's embedding model on demand; an empty result (or error) is -// treated as "no semantic candidates", matching Python's fallback behavior. -func embedQuery(ctx context.Context, tenantID, datasetID, query string) ([]float64, error) { - embedder := NewNavEmbedder(NewModelProviderService(), "") - vecs, err := embedder.Encode(ctx, tenantID, []string{query}) - if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 { - return nil, fmt.Errorf("embed query failed") - } - f := make([]float64, len(vecs[0])) - for i, v := range vecs[0] { - f[i] = float64(v) - } - return f, nil -} diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go index 0ed32ccbe3..07a3379e55 100644 --- a/internal/service/dataset_structure_graph_test.go +++ b/internal/service/dataset_structure_graph_test.go @@ -126,3 +126,18 @@ func TestResolveGraphBucket_UsesRawStructureMetadata(t *testing.T) { t.Fatalf("scope = %#v, want template-scoped raw-row filter", scope) } } + +func TestBuildDocumentGraphTemplateShells_PreservesTemplateMetadata(t *testing.T) { + templates := buildDocumentGraphTemplateShells([]string{"tree-1"}, map[string]map[string]interface{}{ + "tree-1": {"template_name": "Tree", "kind": "tree"}, + }) + if len(templates) != 1 { + t.Fatalf("got %d templates, want 1", len(templates)) + } + if templates[0].TemplateID != "tree-1" || templates[0].TemplateName != "Tree" || templates[0].Kind != "tree" { + t.Fatalf("template = %#v, want metadata for tree-1", templates[0]) + } + if templates[0].Entities == nil || templates[0].Relations == nil { + t.Fatal("empty template collections must be non-nil") + } +}