From 5a2be87f658d01bc5b3b7ecdba92ac69bf09dcf7 Mon Sep 17 00:00:00 2001
From: ai_xiaopei <xiaopei@aisim.cn>
Date: Thu, 03 Sep 2026 15:15:37 +0800
Subject: [PATCH] fix(search): 搜索准确性四连修——FTS5 MATCH 引号包裹(连字符不再被解析成 MINUS)；KeywordSearch 截断按命中关键词数排序(高频 bigram 不再挤出强相关文档)；原词奖励档(完整短语压过泛化 bigram)；tags 字段真实参与评分(此前误用 Section)
---
internal/index/reconcile.go | 372 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 372 insertions(+), 0 deletions(-)
diff --git a/internal/index/reconcile.go b/internal/index/reconcile.go
new file mode 100644
index 0000000..18c25c3
--- /dev/null
+++ b/internal/index/reconcile.go
@@ -0,0 +1,372 @@
+package index
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/aisim/kb-cli/internal/graph"
+ "github.com/aisim/kb-cli/internal/vault"
+)
+
+// ReconcileResult 对账结果统计
+type ReconcileResult struct {
+ Added int
+ Modified int
+ Deleted int
+ Unchanged int
+ Resolved int // 悬空链接自动补全成功的条数
+}
+
+// Reconcile 增量对账：stat 比对 → 只对变更文件解析和写库
+func Reconcile(store *Store, vaultPath string) (*ReconcileResult, error) {
+ res := &ReconcileResult{}
+
+ // 1. vault 侧指纹
+ vaultStats, err := vault.ScanVaultStat(vaultPath)
+ if err != nil {
+ return nil, fmt.Errorf("扫描失败: %w", err)
+ }
+ dbStats, err := store.GetFileStats()
+ if err != nil {
+ return nil, fmt.Errorf("读取索引指纹失败: %w", err)
+ }
+
+ // 2. 分类
+ type change struct {
+ path string
+ stat vault.FileStat
+ known FileStat
+ }
+ var added, modified []change
+ for _, v := range vaultStats {
+ known, ok := dbStats[v.Path]
+ if !ok {
+ added = append(added, change{v.Path, v, FileStat{}})
+ continue
+ }
+ if v.Size == known.Size && v.Mtime == known.Mtime {
+ res.Unchanged++
+ continue
+ }
+ // size/mtime 变化 → sha256 二次确认
+ hash, err := fileHash(filepath.Join(vaultPath, v.Path))
+ if err != nil {
+ return nil, fmt.Errorf("哈希 %s 失败: %w", v.Path, err)
+ }
+ if known.ContentHash != "" && hash == known.ContentHash {
+ res.Unchanged++ // 内容没变（如 touch），只更新指纹
+ store.db.Exec(`UPDATE nodes SET size=?, mtime=? WHERE path=?`, v.Size, v.Mtime, v.Path)
+ continue
+ }
+ modified = append(modified, change{v.Path, v, known})
+ }
+ // 3. 删除
+ for path := range dbStats {
+ found := false
+ for _, v := range vaultStats {
+ if v.Path == path {
+ found = true
+ break
+ }
+ }
+ if !found {
+ if err := store.DeleteNode(path); err != nil {
+ return nil, err
+ }
+ res.Deleted++
+ }
+ }
+
+ // 4. 新增 + 修改：解析 → 写节点 → 重建该节点边
+ for _, c := range append(added, modified...) {
+ isNew := c.known.Path == ""
+ meta, err := vault.ParseFile(filepath.Join(vaultPath, c.path), c.path)
+ if err != nil {
+ return nil, fmt.Errorf("解析 %s 失败: %w", c.path, err)
+ }
+ hash, err := fileHash(filepath.Join(vaultPath, c.path))
+ if err != nil {
+ return nil, err
+ }
+ if isNew {
+ res.Added++
+ } else {
+ res.Modified++
+ }
+ if err := applyNode(store, vaultPath, meta, c.stat.Size, c.stat.Mtime, hash); err != nil {
+ return nil, err
+ }
+ }
+
+ // 5. 悬空链接重试：新节点入库后，历史悬空链接可能变可解析
+ resolved, err := store.RetryUnresolved()
+ if err != nil {
+ return nil, fmt.Errorf("悬空链接重试失败: %w", err)
+ }
+ res.Resolved = resolved
+ return res, nil
+}
+
+func fileHash(path string) (string, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return "", err
+ }
+ sum := sha256.Sum256(data)
+ return hex.EncodeToString(sum[:]), nil
+}
+
+// applyNode 写节点 + 重建该节点的出边（tag/entity/wikilink）+ 悬空入表
+func applyNode(store *Store, vaultPath string, meta *vault.FileMeta, size, mtime int64, hash string) error {
+ n := &graph.Node{
+ Path: meta.Path, Title: meta.Title, Section: meta.Section,
+ Tags: meta.Tags, Entities: meta.Entities, Wikilinks: meta.Wikilinks,
+ Aliases: meta.Aliases, Status: meta.Status, Content: meta.Content,
+ }
+ if err := store.UpsertNode(n, size, mtime, hash); err != nil {
+ return err
+ }
+ var nodeID int64
+ if err := store.db.QueryRow(`SELECT id FROM nodes WHERE path=?`, meta.Path).Scan(&nodeID); err != nil {
+ return err
+ }
+ // 删旧边后重建出边
+ if err := store.DeleteNodeEdges(nodeID); err != nil {
+ return err
+ }
+ return store.buildNodeEdges(nodeID, meta)
+}
+
+// buildNodeEdges 为单个节点建出边；wikilink 解析失败入 unresolved_links
+func (s *Store) buildNodeEdges(nodeID int64, meta *vault.FileMeta) error {
+ // tag / entity 边（虚拟节点 ID 沿用 1000000+ 规则，按 label 查现有行避免重复）
+ for _, tag := range meta.Tags {
+ if err := s.insertTagEntityEdge(nodeID, "tag:"+tag, "tag", tag); err != nil {
+ return err
+ }
+ }
+ for _, entity := range meta.Entities {
+ if err := s.insertTagEntityEdge(nodeID, "entity:"+entity, "entity", entity); err != nil {
+ return err
+ }
+ }
+ // wikilink 边 + 悬空
+ for _, link := range meta.Wikilinks {
+ targetID, prov, ok := s.resolveWikilink(link)
+ if !ok {
+ tail := nameTail(link)
+ // 幂等守卫：同 from_node+link_text 不重复插（unresolved_links 无 UNIQUE 约束，用 WHERE NOT EXISTS）
+ s.db.Exec(`INSERT INTO unresolved_links (from_node, link_text, name_tail, status)
+ SELECT ?, ?, ?, 'pending'
+ WHERE NOT EXISTS (
+ SELECT 1 FROM unresolved_links WHERE from_node = ? AND link_text = ?)`,
+ nodeID, link, tail, nodeID, link)
+ continue
+ }
+ if err := s.InsertEdge(&graph.Edge{
+ FromNode: nodeID, ToNode: targetID,
+ Relation: "wikilink", Label: link, Provenance: prov,
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// insertTagEntityEdge tag/entity 边（虚拟节点按 label 复用 ID）
+func (s *Store) insertTagEntityEdge(fromNode int64, key, relation, label string) error {
+ var virtualID int64
+ err := s.db.QueryRow(`SELECT to_node FROM edges WHERE relation=? AND label=? LIMIT 1`, relation, label).Scan(&virtualID)
+ if err != nil {
+ // 新虚拟节点：分配 ID = 1000000 + 行号（稳定：按 label 排序后的行号）
+ var maxID int64
+ s.db.QueryRow(`SELECT COALESCE(MAX(to_node), 1000000) FROM edges WHERE to_node >= 1000000 AND relation=?`, relation).Scan(&maxID)
+ virtualID = maxID + 1
+ }
+ return s.InsertEdge(&graph.Edge{FromNode: fromNode, ToNode: virtualID, Relation: relation, Label: label, Provenance: relation})
+}
+
+// resolveWikilink 解析 wikilink 目标，返回 (nodeID, provenance, ok)
+// provenance: exact = 标题或文件名精确匹配；fuzzy = 标题包含匹配
+func (s *Store) resolveWikilink(link string) (int64, string, bool) {
+ // 去锚点：[[标题|别名]] 取标题部分
+ if idx := strings.Index(link, "|"); idx >= 0 {
+ link = link[:idx]
+ }
+ var id int64
+ var title, path string
+ // 1. 标题精确
+ err := s.db.QueryRow(`SELECT id, title, path FROM nodes WHERE title = ? LIMIT 1`, link).Scan(&id, &title, &path)
+ if err == nil {
+ return id, "exact", true
+ }
+ // 2. 文件名精确（去 .md 和编号前缀）
+ rows, err := s.db.Query(`SELECT id, title, path FROM nodes`)
+ if err != nil {
+ return 0, "", false
+ }
+ defer rows.Close()
+ var fuzzyID int64
+ for rows.Next() {
+ var nid int64
+ var nTitle, nPath string
+ if err := rows.Scan(&nid, &nTitle, &nPath); err != nil {
+ return 0, "", false
+ }
+ base := filepath.Base(nPath)
+ base = strings.TrimSuffix(base, ".md")
+ if dash := strings.Index(base, "-"); dash >= 0 {
+ base = base[dash+1:]
+ }
+ if base == link || nTitle == link {
+ return nid, "exact", true
+ }
+ // 空 link 跳过 fuzzy（Contains 对空串恒真会误建边）
+ if fuzzyID == 0 && link != "" && strings.Contains(nTitle, link) {
+ fuzzyID = nid
+ }
+ }
+ if fuzzyID != 0 {
+ return fuzzyID, "fuzzy", true
+ }
+ return 0, "", false
+}
+
+// nameTail 取 link 尾部用于重试匹配（去锚点修饰）
+func nameTail(link string) string {
+ if idx := strings.Index(link, "|"); idx >= 0 {
+ link = link[:idx]
+ }
+ return link
+}
+
+// RetryUnresolved 重试解析悬空链接：用当前全部节点的标题/文件名去匹配 unresolved_links 的 name_tail。
+// 命中则建边（provenance 按匹配严格度）、删行。返回成功解析条数。
+func (s *Store) RetryUnresolved() (int, error) {
+ rows, err := s.db.Query(`SELECT id, from_node, link_text, name_tail FROM unresolved_links`)
+ if err != nil {
+ return 0, err
+ }
+ type pending struct {
+ id int64
+ fromNode int64
+ linkText string
+ tail string
+ }
+ var pendings []pending
+ for rows.Next() {
+ var p pending
+ if err := rows.Scan(&p.id, &p.fromNode, &p.linkText, &p.tail); err != nil {
+ rows.Close()
+ return 0, err
+ }
+ pendings = append(pendings, p)
+ }
+ rows.Close()
+ if len(pendings) == 0 {
+ return 0, nil
+ }
+
+ // 建匹配索引：标题/文件名(去编号) → nodeID，精确匹配优先
+ type matchInfo struct {
+ id int64
+ prov string
+ }
+ exactMap := make(map[string]matchInfo)
+ var fuzzyRows []struct {
+ id int64
+ title string
+ }
+ nrows, err := s.db.Query(`SELECT id, title, path FROM nodes`)
+ if err != nil {
+ return 0, err
+ }
+ for nrows.Next() {
+ var id int64
+ var title, path string
+ if err := nrows.Scan(&id, &title, &path); err != nil {
+ nrows.Close()
+ return 0, err
+ }
+ if _, ok := exactMap[title]; !ok {
+ exactMap[title] = matchInfo{id, "exact"}
+ }
+ base := filepath.Base(path)
+ base = strings.TrimSuffix(base, ".md")
+ if dash := strings.Index(base, "-"); dash >= 0 {
+ base = base[dash+1:]
+ }
+ if _, ok := exactMap[base]; !ok {
+ exactMap[base] = matchInfo{id, "exact"}
+ }
+ fuzzyRows = append(fuzzyRows, struct {
+ id int64
+ title string
+ }{id, title})
+ }
+ nrows.Close()
+
+ resolved := 0
+ for _, p := range pendings {
+ if info, ok := exactMap[p.tail]; ok {
+ // 精确命中：建边 + 删悬空行
+ if err := s.InsertEdge(&graph.Edge{
+ FromNode: p.fromNode, ToNode: info.id,
+ Relation: "wikilink", Label: p.linkText, Provenance: info.prov,
+ }); err != nil {
+ return resolved, err
+ }
+ if _, err := s.db.Exec(`DELETE FROM unresolved_links WHERE id=?`, p.id); err != nil {
+ return resolved, err
+ }
+ resolved++
+ continue
+ }
+ // fuzzy：标题包含（空 tail 跳过，避免 Contains 恒真）
+ if p.tail == "" {
+ continue
+ }
+ for _, fr := range fuzzyRows {
+ if strings.Contains(fr.title, p.tail) {
+ if err := s.InsertEdge(&graph.Edge{
+ FromNode: p.fromNode, ToNode: fr.id,
+ Relation: "wikilink", Label: p.linkText, Provenance: "fuzzy",
+ }); err != nil {
+ return resolved, err
+ }
+ if _, err := s.db.Exec(`DELETE FROM unresolved_links WHERE id=?`, p.id); err != nil {
+ return resolved, err
+ }
+ resolved++
+ break
+ }
+ }
+ }
+ return resolved, nil
+}
+
+// QuickCheck 只 stat 比对（不读内容不哈希），返回是否有差异
+func QuickCheck(store *Store, vaultPath string) (bool, error) {
+ vaultStats, err := vault.ScanVaultStat(vaultPath)
+ if err != nil {
+ return false, err
+ }
+ dbStats, err := store.GetFileStats()
+ if err != nil {
+ return false, err
+ }
+ if len(vaultStats) != len(dbStats) {
+ return true, nil
+ }
+ for _, v := range vaultStats {
+ known, ok := dbStats[v.Path]
+ if !ok || v.Size != known.Size || v.Mtime != known.Mtime {
+ return true, nil
+ }
+ }
+ return false, nil
+}
--
Gitblit v1.10.0