From b554c303193aa5d22af5f8a4b91430077113e6db Mon Sep 17 00:00:00 2001
From: ax_rd <ax_rd@aisim.cn>
Date: Thu, 03 Sep 2026 13:21:36 +0800
Subject: [PATCH] fix: search 长 CJK 词 bigram 展开统一下沉（ExpandCJKKeywords 共享 + textScore 按 Expanded 档计权 + graph query 边 provenance 填充）
---
internal/search/engine_test.go | 42 ++++++++++++++
internal/search/expand.go | 39 +++++++++++++
internal/search/explore_test.go | 2
internal/search/engine.go | 22 +++++++
internal/search/explore.go | 40 -------------
internal/index/sqlite.go | 4
6 files changed, 106 insertions(+), 43 deletions(-)
diff --git a/internal/index/sqlite.go b/internal/index/sqlite.go
index 77cbb56..9f8d4c5 100644
--- a/internal/index/sqlite.go
+++ b/internal/index/sqlite.go
@@ -482,7 +482,7 @@
visited[id] = true
query := `
- SELECT id, from_node, to_node, relation, label
+ SELECT id, from_node, to_node, relation, label, provenance
FROM edges
WHERE from_node = ?
`
@@ -501,7 +501,7 @@
for rows.Next() {
var e graph.Edge
var edgeID int64
- if err := rows.Scan(&edgeID, &e.FromNode, &e.ToNode, &e.Relation, &e.Label); err != nil {
+ if err := rows.Scan(&edgeID, &e.FromNode, &e.ToNode, &e.Relation, &e.Label, &e.Provenance); err != nil {
rows.Close()
return nil, fmt.Errorf("扫描边失败: %w", err)
}
diff --git a/internal/search/engine.go b/internal/search/engine.go
index a1dd52a..13e2ff1 100644
--- a/internal/search/engine.go
+++ b/internal/search/engine.go
@@ -36,8 +36,24 @@
// 合并所有关键词
allKeywords := append(append(keywords, opts.Expanded...), opts.Symptom...)
+ // 长 CJK 词 bigram 展开（统一下沉）：FTS5 unicode61 把连续中文当整串单 token，
+ // 长复合词整串 LIKE 匹配不到；ExpandCJKKeywords 拆成 bigram 滑动窗口
+ // （原词保留、去重保序）。bigram 参与检索，textScore 按 ScoreExpanded 档计权
+ // （与现有 Expanded 机制一致），原词仍按原词档。
+ searchKeywords := ExpandCJKKeywords(allKeywords)
+ originalSet := make(map[string]bool, len(allKeywords))
+ for _, kw := range allKeywords {
+ originalSet[kw] = true
+ }
+ var bigrams []string
+ for _, kw := range searchKeywords {
+ if !originalSet[kw] {
+ bigrams = append(bigrams, kw)
+ }
+ }
+
// 1. 双通道检索（ASCII 走 FTS，CJK 走 LIKE）
- candidates, err := store.KeywordSearch(allKeywords, 100)
+ candidates, err := store.KeywordSearch(searchKeywords, 100)
if err != nil {
return nil, err
}
@@ -55,6 +71,10 @@
for _, kw := range opts.Expanded {
score += scoreResult(r, kw, ScoreExpanded)
}
+ // bigram 展开词按 Expanded 档计权（与用户显式扩展词同档）
+ for _, kw := range bigrams {
+ score += scoreResult(r, kw, ScoreExpanded)
+ }
for _, kw := range opts.Symptom {
score += scoreResult(r, kw, ScoreSymptom)
}
diff --git a/internal/search/engine_test.go b/internal/search/engine_test.go
index 6aeeb1e..06b511f 100644
--- a/internal/search/engine_test.go
+++ b/internal/search/engine_test.go
@@ -147,6 +147,48 @@
}
}
+// TestSearchLongCJKBigram 回归（审查遗留）：单复合 CJK 词「电子秤补气失败」
+// 在内容中不连续出现时，整串 LIKE 匹配不到，search 必须对长 CJK 词做 bigram
+// 展开后检索才能命中（RED 证据：未下沉前该查询 0 结果）。
+func TestSearchLongCJKBigram(t *testing.T) {
+ tmpDir := t.TempDir()
+ dbPath := filepath.Join(tmpDir, "test.db")
+ store, err := index.Open(dbPath)
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ defer store.Close()
+
+ // content 含「电子秤补气」「补气失败」片段，但不含完整的连续「电子秤补气失败」
+ n := &graph.Node{
+ ID: 1,
+ Path: "FAQ/称重/001-test.md",
+ Title: "称重故障排查",
+ Section: "FAQ",
+ Content: "电子秤补气 时先检查阀门，补气失败后断电重启。",
+ }
+ if err := store.UpsertNode(n, 100, 1, "x"); err != nil {
+ t.Fatalf("UpsertNode failed: %v", err)
+ }
+ if err := store.CreateFTS(); err != nil {
+ t.Fatalf("CreateFTS failed: %v", err)
+ }
+ if err := store.PopulateFTS(); err != nil {
+ t.Fatalf("PopulateFTS failed: %v", err)
+ }
+
+ results, err := Search(store, []string{"电子秤补气失败"}, SearchOptions{})
+ if err != nil {
+ t.Fatalf("Search failed: %v", err)
+ }
+ if len(results) != 1 {
+ t.Fatalf("expected 1 result (bigram 展开命中), got %d", len(results))
+ }
+ if results[0].Path != "FAQ/称重/001-test.md" {
+ t.Errorf("expected hit FAQ/称重/001-test.md, got %q", results[0].Path)
+ }
+}
+
func TestIsGenericWord(t *testing.T) {
tests := []struct {
word string
diff --git a/internal/search/expand.go b/internal/search/expand.go
new file mode 100644
index 0000000..1c1d835
--- /dev/null
+++ b/internal/search/expand.go
@@ -0,0 +1,39 @@
+package search
+
+// containsCJK 是否含 CJK 统一表意文字
+func containsCJK(s string) bool {
+ for _, r := range s {
+ if r >= 0x4E00 && r <= 0x9FFF {
+ return true
+ }
+ }
+ return false
+}
+
+// ExpandCJKKeywords 关键词展开（search 与 explore 共享）：CJK 长词（>3 字符）
+// 拆成 bigram 滑动窗口（FTS5 unicode61 把连续中文当整串单 token，长复合词整串
+// LIKE 匹配不到；文档里 "电子秤补气失败" 通常不是连续子串，bigram 才能命中）。
+// 原词保留（段落命中判断时原词更精确）。末尾去重保序（原词在前、展开词在后）。
+func ExpandCJKKeywords(keywords []string) []string {
+ var out []string
+ for _, kw := range keywords {
+ out = append(out, kw)
+ if containsCJK(kw) && len([]rune(kw)) > 3 {
+ runes := []rune(kw)
+ for i := 0; i+1 < len(runes); i++ {
+ bg := string(runes[i : i+2])
+ out = append(out, bg)
+ }
+ }
+ }
+ // 去重保序（原词在前、展开词在后）：避免 bigram 与原词重复时 text 分重复计权
+ var dedup []string
+ seen := make(map[string]bool, len(out))
+ for _, kw := range out {
+ if !seen[kw] {
+ seen[kw] = true
+ dedup = append(dedup, kw)
+ }
+ }
+ return dedup
+}
diff --git a/internal/search/explore.go b/internal/search/explore.go
index 9645fe8..c0cb05d 100644
--- a/internal/search/explore.go
+++ b/internal/search/explore.go
@@ -68,44 +68,6 @@
return extractRelevantParagraphs(content, keywords, budget)
}
-// containsCJK 是否含 CJK 统一表意文字
-func containsCJK(s string) bool {
- for _, r := range s {
- if r >= 0x4E00 && r <= 0x9FFF {
- return true
- }
- }
- return false
-}
-
-// expandKeywords 关键词展开：CJK 长词（>3 字符）拆成 bigram 滑动窗口
-// （FTS5 unicode61 把连续中文当整串单 token，长复合词整串 LIKE 匹配不到；
-// 文档里 "电子秤补气失败" 通常不是连续子串，bigram 才能命中）。
-// 原词保留（段落命中判断时原词更精确）。
-func expandKeywords(keywords []string) []string {
- var out []string
- for _, kw := range keywords {
- out = append(out, kw)
- if containsCJK(kw) && len([]rune(kw)) > 3 {
- runes := []rune(kw)
- for i := 0; i+1 < len(runes); i++ {
- bg := string(runes[i : i+2])
- out = append(out, bg)
- }
- }
- }
- // 去重保序（原词在前、展开词在后）：避免 bigram 与原词重复时 text 分重复计权
- var dedup []string
- seen := make(map[string]bool, len(out))
- for _, kw := range out {
- if !seen[kw] {
- seen[kw] = true
- dedup = append(dedup, kw)
- }
- }
- return dedup
-}
-
// ExploreOptions explore 参数
type ExploreOptions struct {
Budget int // 字节预算（0 = 用配置默认）
@@ -135,7 +97,7 @@
return nil, nil
}
// 段落命中用原词（精确），FTS/LIKE 检索用展开后的 bigram
- searchKeywords := expandKeywords(keywords)
+ searchKeywords := ExpandCJKKeywords(keywords)
budget := cfg.Budget
if budget <= 0 {
budget = 16000 // 代码缺省（config 读取在 cmd 层完成）
diff --git a/internal/search/explore_test.go b/internal/search/explore_test.go
index 1bba277..bfe2677 100644
--- a/internal/search/explore_test.go
+++ b/internal/search/explore_test.go
@@ -29,7 +29,7 @@
// TestExploreKeywordExpansion 长 CJK 词拆 bigram（原词保留，ASCII/短词不拆；末尾去重保序）
func TestExploreKeywordExpansion(t *testing.T) {
- got := expandKeywords([]string{"电子秤补气失败", "补气", "abc"})
+ got := ExpandCJKKeywords([]string{"电子秤补气失败", "补气", "abc"})
want := []string{"电子秤补气失败", "电子", "子秤", "秤补", "补气", "气失", "失败", "abc"}
if len(got) != len(want) {
t.Fatalf("展开数量: got=%d want=%d (%v)", len(got), len(want), got)
--
Gitblit v1.10.0