From b554c303193aa5d22af5f8a4b91430077113e6db Mon Sep 17 00:00:00 2001
From: ax_rd <ax_rd@aisim.cn>
Date: Thu, 03 Sep 2026 13:21:36 +0800
Subject: [PATCH] fix: search 长 CJK 词 bigram 展开统一下沉（ExpandCJKKeywords 共享 + textScore 按 Expanded 档计权 + graph query 边 provenance 填充）
---
internal/search/engine_test.go | 111 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 111 insertions(+), 0 deletions(-)
diff --git a/internal/search/engine_test.go b/internal/search/engine_test.go
index 2cd91d3..06b511f 100644
--- a/internal/search/engine_test.go
+++ b/internal/search/engine_test.go
@@ -78,6 +78,117 @@
}
}
+func TestCJKLikeChannel(t *testing.T) {
+ // CJK LIKE 通道：多字符中文词 FTS MATCH 匹配不到（unicode61 整串 token），
+ // 必须走 LIKE 才能命中 content/title
+ tmpDir := t.TempDir()
+ dbPath := filepath.Join(tmpDir, "test.db")
+ store, err := index.Open(dbPath)
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer store.Close()
+
+ nodes := []*graph.Node{
+ { // content 含"补气"，title 不含（靠 LIKE content 通道命中）
+ Path: "FAQ/001-a.md",
+ Title: "设备故障排查",
+ Section: "FAQ",
+ Content: "电子秤补气失败时的排查步骤",
+ },
+ { // title 含"补气"（title LIKE 命中，应排前）
+ Path: "FAQ/002-b.md",
+ Title: "电子秤补气失败",
+ Section: "FAQ",
+ Content: "补气失败的处理方法",
+ },
+ { // 不含"补气"（不应出现在结果中）
+ Path: "FAQ/003-c.md",
+ Title: "阀门漏气处理",
+ Section: "FAQ",
+ Content: "阀门漏气的原因和处理",
+ },
+ }
+ for _, n := range nodes {
+ if err := store.UpsertNode(n, 100, 1, "x"); err != nil {
+ t.Fatalf("UpsertNode failed: %v", err)
+ }
+ }
+ if err := store.CreateFTS(); err != nil {
+ t.Fatalf("CreateFTS failed: %v", err)
+ }
+ if err := store.PopulateFTS(); err != nil {
+ t.Fatalf("PopulateFTS failed: %v", err)
+ }
+
+ results, err := Search(store, []string{"补气"}, SearchOptions{})
+ if err != nil {
+ t.Fatalf("Search failed: %v", err)
+ }
+ if len(results) < 2 {
+ t.Fatalf("Expected 2 results (content 命中 + title 命中), got %d", len(results))
+ }
+ // 两个含"补气"的节点都应在结果中
+ if results[0].Path != "FAQ/002-b.md" && results[1].Path != "FAQ/002-b.md" {
+ t.Error("title 含'补气'的节点应出现在结果中")
+ }
+ if results[0].Path != "FAQ/001-a.md" && results[1].Path != "FAQ/001-a.md" {
+ t.Error("content 含'补气'的节点应出现在结果中")
+ }
+ // 不含"补气"的节点不应出现
+ for _, r := range results {
+ if r.Path == "FAQ/003-c.md" {
+ t.Error("不含关键词的节点不应出现在结果中")
+ }
+ }
+ // title 命中者应排前（title 计权高于 content）
+ if results[0].Path != "FAQ/002-b.md" {
+ t.Errorf("title 命中者应排第一, got %q", results[0].Path)
+ }
+}
+
+// TestSearchLongCJKBigram 回归（审查遗留）：单复合 CJK 词「电子秤补气失败」
+// 在内容中不连续出现时，整串 LIKE 匹配不到，search 必须对长 CJK 词做 bigram
+// 展开后检索才能命中（RED 证据：未下沉前该查询 0 结果）。
+func TestSearchLongCJKBigram(t *testing.T) {
+ tmpDir := t.TempDir()
+ dbPath := filepath.Join(tmpDir, "test.db")
+ store, err := index.Open(dbPath)
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer store.Close()
+
+ // content 含「电子秤补气」「补气失败」片段，但不含完整的连续「电子秤补气失败」
+ n := &graph.Node{
+ ID: 1,
+ Path: "FAQ/称重/001-test.md",
+ Title: "称重故障排查",
+ Section: "FAQ",
+ Content: "电子秤补气 时先检查阀门，补气失败后断电重启。",
+ }
+ if err := store.UpsertNode(n, 100, 1, "x"); err != nil {
+ t.Fatalf("UpsertNode failed: %v", err)
+ }
+ if err := store.CreateFTS(); err != nil {
+ t.Fatalf("CreateFTS failed: %v", err)
+ }
+ if err := store.PopulateFTS(); err != nil {
+ t.Fatalf("PopulateFTS failed: %v", err)
+ }
+
+ results, err := Search(store, []string{"电子秤补气失败"}, SearchOptions{})
+ if err != nil {
+ t.Fatalf("Search failed: %v", err)
+ }
+ if len(results) != 1 {
+ t.Fatalf("expected 1 result (bigram 展开命中), got %d", len(results))
+ }
+ if results[0].Path != "FAQ/称重/001-test.md" {
+ t.Errorf("expected hit FAQ/称重/001-test.md, got %q", results[0].Path)
+ }
+}
+
func TestIsGenericWord(t *testing.T) {
tests := []struct {
word string
--
Gitblit v1.10.0