package search import ( "regexp" "strings" "github.com/aisim/kb-cli/internal/index" ) // headingRe 1-4 级 Markdown 标题 var headingRe = regexp.MustCompile(`^#{1,4} .+$`) // splitParagraphs 按 1-4 级标题切段。无标题的文档整体为一段。 func splitParagraphs(content string) []string { lines := strings.Split(content, "\n") var paras []string var cur []string flush := func() { if len(cur) > 0 { paras = append(paras, strings.TrimRight(strings.Join(cur, "\n"), "\n")+"\n") cur = nil } } for _, l := range lines { if headingRe.MatchString(l) { flush() } cur = append(cur, l) } flush() return paras } // extractRelevantParagraphs 返回命中关键词的段落(整段不截半句)。 // 文档总长 <= budget 时整篇输出;无命中段落时输出空串; // 预算约束:累计超预算的段落丢弃(不截半段)。 func extractRelevantParagraphs(content string, keywords []string, budget int) string { if budget <= 0 || len(content) <= budget { return content } var out []string for _, p := range splitParagraphs(content) { for _, kw := range keywords { if strings.Contains(p, kw) { out = append(out, p) break } } } // 预算约束:累计超预算的段落丢弃(不截半段) var total int kept := []string{} for _, p := range out { if total+len(p) > budget { break } total += len(p) kept = append(kept, p) } return strings.Join(kept, "") } // extractWithBudget extractRelevantParagraphs 的预算版(budget<=0 视为无预算) func extractWithBudget(content string, keywords []string, budget int) string { if budget <= 0 { budget = 100000 } return extractRelevantParagraphs(content, keywords, budget) } // containsCJK 是否含 CJK 统一表意文字 func containsCJK(s string) bool { for _, r := range s { if r >= 0x4E00 && r <= 0x9FFF { return true } } return false } // expandKeywords 关键词展开:CJK 长词(>3 字符)拆成 bigram 滑动窗口 // (FTS5 unicode61 把连续中文当整串单 token,长复合词整串 LIKE 匹配不到; // 文档里 "电子秤补气失败" 通常不是连续子串,bigram 才能命中)。 // 原词保留(段落命中判断时原词更精确)。 func expandKeywords(keywords []string) []string { var out []string for _, kw := range keywords { out = append(out, kw) if containsCJK(kw) && len([]rune(kw)) > 3 { runes := []rune(kw) for i := 0; i+1 < len(runes); i++ { bg := string(runes[i : i+2]) out = append(out, bg) } } } // 去重保序(原词在前、展开词在后):避免 bigram 与原词重复时 text 分重复计权 var dedup []string seen := make(map[string]bool, len(out)) for _, kw := range out { if !seen[kw] { seen[kw] = true dedup = append(dedup, kw) } } return dedup } // ExploreOptions explore 参数 type ExploreOptions struct { Budget int // 字节预算(0 = 用配置默认) TopN int // 0 = 用配置默认 HardBudget int // 字节硬上限(0 = 用代码缺省 32000) } // ExploredDoc 入选文档及其输出正文 type ExploredDoc struct { Path string `json:"path"` Title string `json:"title"` Section string `json:"section"` Score int `json:"score"` Body string `json:"body"` } // ExploreResult explore 结果 type ExploreResult struct { Docs []ExploredDoc `json:"docs"` Related map[string][]string `json:"related"` UnresolvedLinks []string `json:"unresolved_links"` } // Explore 一次调用返回相关文档原文 + 关联清单 + 悬空链接 func Explore(store *index.Store, keywords []string, cfg ExploreOptions) (*ExploreResult, error) { if len(keywords) == 0 { return nil, nil } // 段落命中用原词(精确),FTS/LIKE 检索用展开后的 bigram searchKeywords := expandKeywords(keywords) budget := cfg.Budget if budget <= 0 { budget = 16000 // 代码缺省(config 读取在 cmd 层完成) } if budget > 32000 { budget = 32000 // 硬上限缺省 } // 配置显式给出的硬上限优先(cmd 层读 config.yaml 的 explore.hard_budget) if cfg.HardBudget > 0 && budget > cfg.HardBudget { budget = cfg.HardBudget } topN := cfg.TopN if topN <= 0 { topN = 5 } opts := SearchOptions{TopN: topN} results, err := Search(store, searchKeywords, opts) if err != nil { return nil, err } if len(results) == 0 { return &ExploreResult{Related: map[string][]string{}}, nil } // 按分数降序分配预算:每文档至少 800 字节 res := &ExploreResult{Related: map[string][]string{}} perDoc := budget / len(results) if perDoc < 800 { perDoc = 800 } for _, r := range results { content, _, _, err := store.GetNodeContent(r.ID) if err != nil || content == "" { continue } // 两级回退:先用原词提取(精确);原词在任何段落都不连续出现时 //(复合 CJK 词常见:bigram 召回了文档,但原词不连续)回退到展开词提取, // 避免大文档被 continue 静默丢弃、最终输出「未找到相关文档」。 body := extractRelevantParagraphs(content, keywords, perDoc) if body == "" { body = extractRelevantParagraphs(content, searchKeywords, perDoc) } if body == "" { // 无命中段落但文档入选 → 整篇(若放得下),否则跳过 if len(content) <= perDoc { body = content } else { continue } } res.Docs = append(res.Docs, ExploredDoc{ Path: r.Path, Title: r.Title, Section: r.Section, Score: r.Score, Body: body, }) // 关联清单 if links, err := store.GetNodeLinks(r.ID); err == nil { res.Related[r.Path] = links } } // 悬空链接提示:入选文档的 wikilinks 中未解析的 for _, d := range res.Docs { links, err := store.GetUnresolvedLinks(d.Path) if err == nil { res.UnresolvedLinks = append(res.UnresolvedLinks, links...) } } return res, nil }