mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-11-10 21:32:43 +08:00
fix(memory): pin memory filter facet options to a canonical order (#19906)
This commit is contained in:
1 parent
25a7c14e4f
commit
18ebd2611d
6 files changed
+179
-12
No files matched your search
@@ -23,7 +23,7 @@ from api.db.services.canvas_service import UserCanvasService
|
||||
from api.db.services.task_service import TaskService
|
||||
from api.db.joint_services.memory_message_service import get_memory_size_cache, judge_system_prompt_is_default, queue_save_to_memory_task, query_message
|
||||
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_ids
|
||||
from api.utils.memory_utils import format_ret_data_from_memory, get_memory_type_human
|
||||
from api.utils.memory_utils import format_ret_data_from_memory, get_memory_type_human, memory_type_names, order_facet_options, order_owner_options
|
||||
from api.constants import MEMORY_NAME_LIMIT, MEMORY_SIZE_LIMIT
|
||||
from memory.services.messages import MessageService
|
||||
from memory.utils.prompt_util import PromptAssembler
|
||||
@@ -306,7 +306,14 @@ async def list_memory_filters():
|
||||
memory_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1
|
||||
item = memory["storage_type"]
|
||||
storage_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1
|
||||
return {"filter": {"owner": list(owner.values()), "memory_type": list(memory_type.values()), "storage_type": list(storage_type.values())}, "total": total}
|
||||
return {
|
||||
"filter": {
|
||||
"owner": order_owner_options(owner),
|
||||
"memory_type": order_facet_options(memory_type, memory_type_names()),
|
||||
"storage_type": order_facet_options(storage_type, ("table", "graph")),
|
||||
},
|
||||
"total": total,
|
||||
}
|
||||
|
||||
|
||||
async def get_memory_config(memory_id):
|
||||
|
||||
@@ -46,6 +46,20 @@ def get_memory_type_human(memory_type: int) -> List[str]:
|
||||
return [mem_type.name.lower() for mem_type in MemoryType if memory_type & mem_type.value]
|
||||
|
||||
|
||||
def memory_type_names() -> List[str]:
|
||||
return [mem_type.name.lower() for mem_type in MemoryType]
|
||||
|
||||
|
||||
def order_facet_options(counts: dict, canonical_order) -> List[dict]:
|
||||
ordered = [counts[name] for name in canonical_order if name in counts]
|
||||
ordered.extend(counts[name] for name in sorted(set(counts) - set(canonical_order)))
|
||||
return ordered
|
||||
|
||||
|
||||
def order_owner_options(counts: dict) -> List[dict]:
|
||||
return sorted(counts.values(), key=lambda option: (option["label"].lower(), option["id"]))
|
||||
|
||||
|
||||
def calculate_memory_type(memory_type_name_list: List[str]) -> int:
|
||||
memory_type = 0
|
||||
type_value_map = {mem_type.name.lower(): mem_type.value for mem_type in MemoryType}
|
||||
|
||||
@@ -95,6 +95,11 @@ func GetMemoryTypeHuman(memoryType int64) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
// MemoryTypeNames returns every memory type name in canonical bit-flag order.
|
||||
func MemoryTypeNames() []string {
|
||||
return GetMemoryTypeHuman(MemoryTypeRaw | MemoryTypeSemantic | MemoryTypeEpisodic | MemoryTypeProcedural)
|
||||
}
|
||||
|
||||
// MemoryDAO handles all Memory-related database operations
|
||||
type MemoryDAO struct{}
|
||||
|
||||
|
||||
+25
-10
@@ -27,6 +27,7 @@ import (
|
||||
models "ragflow/internal/entity/models"
|
||||
"ragflow/internal/utility"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -412,16 +413,8 @@ func (s *MemoryService) ListMemoryFilters(ctx context.Context, userID string) (*
|
||||
for id, count := range ownerCounts {
|
||||
resp.Filter.Owner = append(resp.Filter.Owner, MemoryFilterOption{ID: id, Label: ownerLabels[id], Count: count})
|
||||
}
|
||||
for _, id := range []string{"raw", "semantic", "episodic", "procedural"} {
|
||||
if count := typeCounts[id]; count > 0 {
|
||||
resp.Filter.MemoryType = append(resp.Filter.MemoryType, MemoryFilterOption{ID: id, Label: id, Count: count})
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"table", "graph"} {
|
||||
if count := storageCounts[id]; count > 0 {
|
||||
resp.Filter.StorageType = append(resp.Filter.StorageType, MemoryFilterOption{ID: id, Label: id, Count: count})
|
||||
}
|
||||
}
|
||||
resp.Filter.MemoryType = filterOptionsInOrder(typeCounts, dao.MemoryTypeNames())
|
||||
resp.Filter.StorageType = filterOptionsInOrder(storageCounts, []string{"table", "graph"})
|
||||
slices.SortFunc(resp.Filter.Owner, func(a, b MemoryFilterOption) int {
|
||||
return cmp.Or(cmp.Compare(strings.ToLower(a.Label), strings.ToLower(b.Label)), cmp.Compare(a.ID, b.ID))
|
||||
})
|
||||
@@ -429,6 +422,28 @@ func (s *MemoryService) ListMemoryFilters(ctx context.Context, userID string) (*
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func filterOptionsInOrder(counts map[string]int64, canonicalOrder []string) []MemoryFilterOption {
|
||||
options := make([]MemoryFilterOption, 0, len(counts))
|
||||
known := make(map[string]bool, len(canonicalOrder))
|
||||
for _, id := range canonicalOrder {
|
||||
if count, ok := counts[id]; ok {
|
||||
options = append(options, MemoryFilterOption{ID: id, Label: id, Count: count})
|
||||
known[id] = true
|
||||
}
|
||||
}
|
||||
rest := make([]string, 0, len(counts)-len(options))
|
||||
for id := range counts {
|
||||
if !known[id] {
|
||||
rest = append(rest, id)
|
||||
}
|
||||
}
|
||||
sort.Strings(rest)
|
||||
for _, id := range rest {
|
||||
options = append(options, MemoryFilterOption{ID: id, Label: id, Count: counts[id]})
|
||||
}
|
||||
return options
|
||||
}
|
||||
|
||||
// CreateMemory creates a new memory with the given parameters
|
||||
// It validates the request, generates a unique name if needed, and creates the memory record
|
||||
//
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -250,6 +251,70 @@ func TestListMemoryFiltersUsesAccessibleMemoryAggregates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMemoryFiltersKeepsCanonicalFacetOrder(t *testing.T) {
|
||||
setupMemoryMessageTestDB(t)
|
||||
if err := dao.DB.Create(&[]*entity.User{
|
||||
{ID: "user-1", Nickname: "Owner"},
|
||||
{ID: "user-2", Nickname: "Zeta"},
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed user: %v", err)
|
||||
}
|
||||
status := "1"
|
||||
if err := dao.DB.Create(&entity.UserTenant{
|
||||
ID: "ut-1", UserID: "user-1", TenantID: "user-2", Role: "normal", InvitedBy: "user-2", Status: &status,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed user tenant: %v", err)
|
||||
}
|
||||
oldest, middle, newest := int64(100), int64(200), int64(300)
|
||||
for _, memory := range []*entity.Memory{
|
||||
{ID: "mem-episodic", Name: "episodic oldest", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw | dao.MemoryTypeEpisodic, StorageType: "graph", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO), BaseModel: entity.BaseModel{CreateTime: &oldest}},
|
||||
{ID: "mem-procedural", Name: "procedural middle", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw | dao.MemoryTypeProcedural, StorageType: "alpha", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO), BaseModel: entity.BaseModel{CreateTime: &middle}},
|
||||
{ID: "mem-semantic", Name: "semantic newest", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw | dao.MemoryTypeSemantic, StorageType: "table", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO), BaseModel: entity.BaseModel{CreateTime: &newest}},
|
||||
{ID: "mem-shared", Name: "shared noncanonical storage", TenantID: "user-2", MemoryType: dao.MemoryTypeSemantic, StorageType: "zeta", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionTeam), ForgettingPolicy: string(ForgettingPolicyFIFO), BaseModel: entity.BaseModel{CreateTime: &newest}},
|
||||
} {
|
||||
if err := dao.DB.Create(memory).Error; err != nil {
|
||||
t.Fatalf("seed memory: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
filters, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ListMemoryFilters: %v", err)
|
||||
}
|
||||
if got, want := filters.Filter.Owner, []MemoryFilterOption{
|
||||
{ID: "user-1", Label: "Owner", Count: 3},
|
||||
{ID: "user-2", Label: "Zeta", Count: 1},
|
||||
}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("owner filter = %+v, want %+v", got, want)
|
||||
}
|
||||
gotTypes := make([]string, 0, len(filters.Filter.MemoryType))
|
||||
for _, option := range filters.Filter.MemoryType {
|
||||
gotTypes = append(gotTypes, option.ID)
|
||||
}
|
||||
if want := []string{"raw", "semantic", "episodic", "procedural"}; !slices.Equal(gotTypes, want) {
|
||||
t.Fatalf("memory type facet order = %v, want %v", gotTypes, want)
|
||||
}
|
||||
gotStorage := make([]string, 0, len(filters.Filter.StorageType))
|
||||
for _, option := range filters.Filter.StorageType {
|
||||
gotStorage = append(gotStorage, option.ID)
|
||||
}
|
||||
if want := []string{"table", "graph", "alpha", "zeta"}; !slices.Equal(gotStorage, want) {
|
||||
t.Fatalf("storage type facet order = %v, want %v", gotStorage, want)
|
||||
}
|
||||
|
||||
bumped := int64(400)
|
||||
if err := dao.DB.Model(&entity.Memory{ID: "mem-semantic"}).Update("update_time", bumped).Error; err != nil {
|
||||
t.Fatalf("bump update_time: %v", err)
|
||||
}
|
||||
refreshed, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
|
||||
if err != nil {
|
||||
t.Fatalf("ListMemoryFilters refresh: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(refreshed, filters) {
|
||||
t.Fatalf("refreshed filters = %+v, want %+v", refreshed, filters)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForgetMessageKeepsCompanionFieldForNonOceanBaseEngines(t *testing.T) {
|
||||
setupMemoryMessageTestDB(t)
|
||||
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#
|
||||
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import pytest
|
||||
|
||||
from api.utils.memory_utils import memory_type_names, order_facet_options, order_owner_options
|
||||
|
||||
|
||||
def _option_ids(options):
|
||||
return [option["id"] for option in options]
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
def test_memory_type_names_follow_canonical_bit_flag_order():
|
||||
assert memory_type_names() == ["raw", "semantic", "episodic", "procedural"]
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
def test_facet_options_keep_canonical_order_regardless_of_aggregation_order():
|
||||
counts = {
|
||||
"procedural": {"id": "procedural", "count": 5},
|
||||
"raw": {"id": "raw", "count": 13},
|
||||
"semantic": {"id": "semantic", "count": 7},
|
||||
"episodic": {"id": "episodic", "count": 7},
|
||||
}
|
||||
|
||||
assert _option_ids(order_facet_options(counts, memory_type_names())) == ["raw", "semantic", "episodic", "procedural"]
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
def test_facet_options_append_unknown_values_after_canonical_ones():
|
||||
counts = {
|
||||
"vector": {"id": "vector", "count": 2},
|
||||
"graph": {"id": "graph", "count": 1},
|
||||
"table": {"id": "table", "count": 3},
|
||||
}
|
||||
|
||||
assert _option_ids(order_facet_options(counts, ("table", "graph"))) == ["table", "graph", "vector"]
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
def test_owner_options_sort_by_label_case_insensitively_then_id():
|
||||
counts = {
|
||||
"tenant-2": {"id": "tenant-2", "label": "Zeta", "count": 1},
|
||||
"tenant-1": {"id": "tenant-1", "label": "owner", "count": 3},
|
||||
"tenant-3": {"id": "tenant-3", "label": "Owner", "count": 2},
|
||||
}
|
||||
|
||||
assert _option_ids(order_owner_options(counts)) == ["tenant-1", "tenant-3", "tenant-2"]
|
||||
Reference in new issue
Block a user