diff --git a/dice/builtin_commands.go b/dice/builtin_commands.go index fda32d9d..f7f96eac 100644 --- a/dice/builtin_commands.go +++ b/dice/builtin_commands.go @@ -13,11 +13,12 @@ import ( "time" "github.com/golang-module/carbon" - "github.com/samber/lo" - "github.com/juliangruber/go-intersect" cp "github.com/otiai10/copy" + "github.com/samber/lo" ds "github.com/sealdice/dicescript" + + "sealdice-core/dice/docengine" ) /** 这几条指令不能移除 */ @@ -250,6 +251,7 @@ func (d *Dice) registerCoreCommands() { var id string if cmdArgs.GetKwarg("rand") != nil || cmdArgs.GetKwarg("随机") != nil { + // FIXME: byd WHAT IS THAT _id := rand.Uint64()%d.Parent.Help.CurID + 1 id = strconv.FormatUint(_id, 10) } @@ -270,8 +272,8 @@ func (d *Dice) registerCoreCommands() { } if id != "" { - text, exists := d.Parent.Help.TextMap.Load(id) - if exists { + text, err := d.Parent.Help.searchEngine.GetItemByID(id) + if err == nil { content := d.Parent.Help.GetContent(text, 0) ReplyToSender(ctx, msg, fmt.Sprintf("词条: %s:%s\n%s", text.PackageName, text.Title, content)) } else { @@ -321,6 +323,7 @@ func (d *Dice) registerCoreCommands() { // 未指定搜索分组时,取当前群指定的分组 group = ctx.Group.DefaultHelpGroup } + // 进行结果搜索 search, total, pgStart, pgEnd, err := d.Parent.Help.Search(ctx, text, false, numLimit, page, group) if err != nil { ReplyToSender(ctx, msg, groupStr+"搜索故障: "+err.Error()) @@ -336,20 +339,25 @@ func (d *Dice) registerCoreCommands() { } hasSecond := len(search.Hits) >= 2 - best, ok := d.Parent.Help.TextMap.Load(search.Hits[0].ID) - if !ok { - d.Logger.Errorf("加载d.Parent.Help.TextMap.Load(search.Hits[0].ID)->(%s)的数据出现错误!", search.Hits[0].ID) - ReplyToSender(ctx, msg, "未找到搜索结果,出现数据加载错误!") - return CmdExecuteResult{Matched: true, Solved: true} + // 准备接下来读取这里面的Fields + bestRaw := search.Hits[0].Fields + best := &docengine.HelpTextItem{ + Group: fmt.Sprintf("%v", bestRaw["group"]), + From: fmt.Sprintf("%v", bestRaw["from"]), + Title: fmt.Sprintf("%v", bestRaw["title"]), + Content: fmt.Sprintf("%v", bestRaw["content"]), + PackageName: fmt.Sprintf("%v", bestRaw["package"]), + // 这俩是什么东西?! + // KeyWords: "", + // RelatedExt: nil, } others := "" for _, i := range search.Hits { - t, ok := d.Parent.Help.TextMap.Load(i.ID) - if !ok { - d.Logger.Errorf("加载d.Parent.Help.TextMap.Load(search.Hits[0].ID)->(%s)的数据出现错误!", search.Hits[0].ID) - ReplyToSender(ctx, msg, "未找到搜索结果,出现数据加载错误!") - return CmdExecuteResult{Matched: true, Solved: true} + t := &docengine.HelpTextItem{ + Group: fmt.Sprintf("%v", i.Fields["group"]), + Title: fmt.Sprintf("%v", i.Fields["title"]), + PackageName: fmt.Sprintf("%v", i.Fields["package"]), } if t.Group != "" && t.Group != HelpBuiltinGroup { others += fmt.Sprintf("[%s][%s]【%s:%s】 匹配度%.2f\n", i.ID, t.Group, t.PackageName, t.Title, i.Score) @@ -468,11 +476,15 @@ func (d *Dice) registerCoreCommands() { search, _, _, _, err := d.Parent.Help.Search(ctx, cmdArgs.CleanArgs, true, 1, 1, "") if err == nil { if len(search.Hits) > 0 { - a, ok := d.Parent.Help.TextMap.Load(search.Hits[0].ID) - if !ok { - d.Logger.Error("HELPDOC:读取ID对应的信息出现问题") - ReplyToSender(ctx, msg, "HELPDOC:读取ID对应的信息出现问题") - return CmdExecuteResult{Matched: true, Solved: true} + a := &docengine.HelpTextItem{ + Group: fmt.Sprintf("%v", search.Hits[0].Fields["group"]), + From: fmt.Sprintf("%v", search.Hits[0].Fields["from"]), + Title: fmt.Sprintf("%v", search.Hits[0].Fields["title"]), + Content: fmt.Sprintf("%v", search.Hits[0].Fields["content"]), + PackageName: fmt.Sprintf("%v", search.Hits[0].Fields["package"]), + // 这俩是什么东西?! + KeyWords: "", + RelatedExt: nil, } content := d.Parent.Help.GetContent(a, 0) ReplyToSender(ctx, msg, fmt.Sprintf("%s:%s\n%s", a.PackageName, a.Title, content)) diff --git a/dice/dice_attrs_manager.go b/dice/dice_attrs_manager.go index 40960c0a..85938d43 100644 --- a/dice/dice_attrs_manager.go +++ b/dice/dice_attrs_manager.go @@ -1,6 +1,7 @@ package dice import ( + "context" "errors" "fmt" "time" @@ -15,9 +16,15 @@ import ( type AttrsManager struct { db *gorm.DB logger *log.Helper + cancel context.CancelFunc m SyncMap[string, *AttributesItem] } +func (am *AttrsManager) Stop() { + log.Info("结束数据库保存程序...") + am.cancel() +} + // LoadByCtx 获取当前角色,如有绑定,则获取绑定的角色,若无绑定,获取群内默认卡 func (am *AttrsManager) LoadByCtx(ctx *MsgContext) (*AttributesItem, error) { return am.Load(ctx.Group.GroupID, ctx.Player.UserID) @@ -148,14 +155,25 @@ func (am *AttrsManager) LoadById(id string) (*AttributesItem, error) { func (am *AttrsManager) Init(d *Dice) { am.db = d.DBData am.logger = d.Logger + // 创建一个 context 用于取消 goroutine + ctx, cancel := context.WithCancel(context.Background()) + // 确保程序退出时取消上下文 go func() { // NOTE(Xiangze Li): 这种不退出的goroutine不利于平稳结束程序 for { - am.CheckForSave() - am.CheckAndFreeUnused() - time.Sleep(15 * time.Second) + select { + case <-ctx.Done(): + // 检测到取消信号后退出循环 + return + default: + // 正常工作 + am.CheckForSave() + am.CheckAndFreeUnused() + time.Sleep(15 * time.Second) + } } }() + am.cancel = cancel } func (am *AttrsManager) CheckForSave() (int, int) { @@ -200,15 +218,22 @@ func (am *AttrsManager) CheckAndFreeUnused() { prepareToFree := map[string]int{} currentTime := time.Now().Unix() + tx := db.Begin() am.m.Range(func(key string, value *AttributesItem) bool { if value.LastUsedTime-currentTime > 60*10 { prepareToFree[key] = 1 // 直接保存 - value.SaveToDB(am.db) + value.SaveToDB(tx) } return true }) - + err := tx.Commit().Error + if err != nil { + if am.logger != nil { + am.logger.Errorf("定期清理无用用户数据出错(提交事务): %v", err) + } + _ = tx.Rollback() + } for key := range prepareToFree { am.m.Delete(key) } diff --git a/dice/dice_help.go b/dice/dice_help.go index 1c650467..a26378b0 100644 --- a/dice/dice_help.go +++ b/dice/dice_help.go @@ -10,26 +10,20 @@ import ( "path/filepath" "regexp" "runtime" - "sort" "strconv" "strings" + "time" + "sealdice-core/dice/docengine" log "sealdice-core/utils/kratos" "gopkg.in/yaml.v3" nanoid "github.com/matoous/go-nanoid/v2" - "github.com/blevesearch/bleve/v2" - "github.com/blevesearch/bleve/v2/search" - "github.com/blevesearch/bleve/v2/search/query" - "github.com/sahilm/fuzzy" "github.com/xuri/excelize/v2" ) -// 分词器封存了,看起来不太需要 -// _ "github.com/leopku/bleve-gse-tokenizer/v2" - const HelpBuiltinGroup = "builtin" const ( @@ -51,17 +45,7 @@ type HelpDoc struct { Children []*HelpDoc `json:"children"` } -type HelpTextItem struct { - Group string - From string - Title string - Content string - PackageName string - KeyWords string - RelatedExt []string -} - -type HelpTextItems []*HelpTextItem +type HelpTextItems []*docengine.HelpTextItem func (e HelpTextItems) String(i int) string { return e[i].Title @@ -73,30 +57,31 @@ func (e HelpTextItems) Len() int { type HelpManager struct { CurID uint64 - Index bleve.Index - TextMap SyncMap[string, *HelpTextItem] // map[string]*HelpTextItem Parent *DiceManager - EngineType int - batch *bleve.Batch - batchNum int + EngineType EngineType LoadingFn string HelpDocTree []*HelpDoc GroupAliases map[string]string + // SearchEngine + searchEngine docengine.SearchEngine Config *HelpConfig } +type EngineType int + +const ( + BleveSearch EngineType = iota // 0 + Clover // 1 + MeiliSearch // 2 +) + const HelpConfigFilename = "help_config.yaml" type HelpConfig struct { Aliases map[string][]string `yaml:"aliases" json:"aliases"` } -func (m *HelpManager) GetNextID() string { - m.CurID++ - return strconv.FormatUint(m.CurID, 10) -} - type HelpDocFormat struct { Mod string `json:"mod"` Author string `json:"author"` @@ -107,70 +92,40 @@ type HelpDocFormat struct { func (m *HelpManager) loadSearchEngine() { if runtime.GOARCH == "arm64" { - m.EngineType = 1 // 默认0,bleve - } - - // not bleve - if m.EngineType != 0 { - return + // 等木落测试,测试之前先不实现这个Clover模式,如果直接就能用,那也不必再实现他了 + m.EngineType = BleveSearch } - - // 删除旧版本的 + // 删除旧版本数据,这里先不改,先集中精力测试BleveSearch indexDir := "./data/_index" _ = os.RemoveAll(indexDir) - - mapping := bleve.NewIndexMapping() indexDir = "./_help_cache" _ = os.RemoveAll(indexDir) - - // if m.Parent.UseDictForTokenizer { - // 这些代码封存,看起来不怎么需要 - // if err := mapping.AddCustomTokenizer("gse", map[string]interface{}{ - // "type": "gse", - // "user_dicts": "./data/dict/zh/dict.txt", // <-- MUST specified, otherwise panic would occurred. - // }); err != nil { - // panic(err) - // } - // if err := mapping.AddCustomAnalyzer("gse", map[string]interface{}{ - // "type": "gse", - // "tokenizer": "gse", - // }); err != nil { - // panic(err) - // } - // mapping.DefaultAnalyzer = "gse" - // } - - docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("title", bleve.NewTextFieldMapping()) - docMapping.AddFieldMappingsAt("content", bleve.NewTextFieldMapping()) - docMapping.AddFieldMappingsAt("package", bleve.NewTextFieldMapping()) - - mapping.AddDocumentMapping("helpdoc", docMapping) - mapping.TypeField = "_type" // 此为默认值,可修改 - - index, err := bleve.New(indexDir, mapping) - if err != nil { - panic(err) + switch m.EngineType { + case Clover: + case BleveSearch: + engine, err := docengine.NewBleveSearchEngine() + if err != nil { + log.Errorf("初始化帮助文档失败,帮助文档不可用!") + return + } + m.searchEngine = engine + default: + // 如果BleveSearch兼容性差,到时候全部回退到Clover查询 + panic("unhandled default case") } - - m.Index = index } func (m *HelpManager) Close() { - if m.EngineType == 0 { - if m.Index != nil { - _ = m.Index.Close() - m.Index = nil - - _ = os.RemoveAll("./_help_cache") - } - } + // 关闭Bucket,并删除所有数据 + // TODO:暂时先不动删除逻辑 + m.searchEngine.Close() + _ = os.RemoveAll("./_help_cache") } func (m *HelpManager) Load() { m.loadSearchEngine() - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: HelpBuiltinGroup, Title: "骰点", Content: `.help 骰点: @@ -183,7 +138,7 @@ func (m *HelpManager) Load() { PackageName: "帮助", }) - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: HelpBuiltinGroup, Title: "扩展", Content: `.help 扩展: @@ -201,7 +156,7 @@ func (m *HelpManager) Load() { PackageName: "帮助", }) - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: HelpBuiltinGroup, Title: "跑团", Content: `.help 跑团: @@ -225,7 +180,11 @@ func (m *HelpManager) Load() { if err != nil { log.Errorf("unable to read helpdoc folder: %v", err) } - for _, entry := range entries { + start := time.Now() // 获取当前时间 + totalEntries := len(entries) + for i, entry := range entries { + progress := float64(i+1) / float64(totalEntries) * 100 + log.Infof("处理帮助文档组[文件夹]: 当前帮助文档加载总进度: %s %.2f%% (%d/%d)", entry.Name(), progress, i+1, totalEntries) if strings.HasPrefix(entry.Name(), ".") { continue } @@ -249,7 +208,9 @@ func (m *HelpManager) Load() { buildHelpDocTree(&child, func(d *HelpDoc) { if !d.IsDir { ok := m.loadHelpDoc(d.Group, d.Path) - if ok { + // TODO: Batch过大好像不会释放…… + err = m.AddItemApply(false) + if ok && err == nil { d.LoadStatus = Loaded } else { d.LoadStatus = LoadError @@ -258,7 +219,10 @@ func (m *HelpManager) Load() { }) m.HelpDocTree = append(m.HelpDocTree, &child) } - _ = m.AddItemApply() + _ = m.AddItemApply(true) + m.CurID = m.searchEngine.GetTotalID() + elapsed := time.Since(start) // 计算执行时间 + log.Infof("帮助文档加载完毕,共耗费时间: %s 共计加载条目:%d\n", elapsed, m.CurID) } func (m *HelpManager) loadHelpConfig() { @@ -314,7 +278,7 @@ func (m *HelpManager) loadHelpDoc(group string, path string) bool { err = json.Unmarshal(pack, &data) if err == nil { for k, v := range data.Helpdoc { - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: group, From: path, Title: k, @@ -360,7 +324,7 @@ func (m *HelpManager) loadHelpDoc(group string, path string) bool { } content := row[synonymCount+1] - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: group, From: path, Title: key, @@ -454,7 +418,7 @@ func (dm *DiceManager) AddHelpWithDice(dice *Dice) { if content == "" { content = v.ShortHelp } - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: HelpBuiltinGroup, Title: k, Content: content, @@ -465,7 +429,7 @@ func (dm *DiceManager) AddHelpWithDice(dice *Dice) { addCmdMap("核心指令", dice.CmdMap) for _, i := range dice.ExtList { - _ = m.AddItem(HelpTextItem{ + _ = m.AddItem(docengine.HelpTextItem{ Group: HelpBuiltinGroup, Title: i.Name, Content: i.GetDescText(i), @@ -473,177 +437,39 @@ func (dm *DiceManager) AddHelpWithDice(dice *Dice) { }) addCmdMap(i.Name, i.CmdMap) } - _ = m.AddItemApply() + _ = m.AddItemApply(false) } -func (m *HelpManager) AddItem(item HelpTextItem) error { - data := map[string]string{ - "group": item.Group, - "from": item.From, - "title": item.Title, - "content": item.Content, - "package": item.PackageName, - "_type": "helpdoc", - } - - id := m.GetNextID() - m.TextMap.Store(id, &item) - - if m.EngineType == 0 { - if m.batch == nil { - m.batch = m.Index.NewBatch() - } - if m.batchNum >= 50 { - err := m.Index.Batch(m.batch) - if err != nil { - return err - } - m.batch.Reset() - m.batchNum = 0 - } - - m.batchNum++ - return m.batch.Index(id, data) - } - return nil +func (m *HelpManager) AddItem(item docengine.HelpTextItem) error { + _, err := m.searchEngine.AddItem(item) + return err } -func (m *HelpManager) AddItemApply() error { - if m.batch != nil { - err := m.Index.Batch(m.batch) - m.batch.Reset() - m.batch = nil +func (m *HelpManager) AddItemApply(end bool) error { + err := m.searchEngine.AddItemApply(end) + if err != nil { return err } return nil } -func (m *HelpManager) searchBleve(ctx *MsgContext, text string, titleOnly bool, pageSize, pageNum int, group string) (*bleve.SearchResult, int, int, int, error) { - // 在标题中查找 - queryTitle := query.NewMatchPhraseQuery(text) - queryTitle.SetField("title") - - titleOrContent := bleve.NewDisjunctionQuery(queryTitle) - - // 在正文中查找 - if !titleOnly { - for _, i := range reSpace.Split(text, -1) { - queryContent := query.NewMatchPhraseQuery(i) - queryContent.SetField("content") - titleOrContent.AddQuery(queryContent) - } - } - - andQuery := bleve.NewConjunctionQuery(titleOrContent) - - // 限制查询组 - for _, i := range ctx.Group.HelpPackages { - queryPack := query.NewMatchPhraseQuery(i) - queryPack.SetField("package") - andQuery.AddQuery(queryPack) - } - - // 查询指定文档组 - if group != "" { - queryPack := query.NewMatchPhraseQuery(group) - queryPack.SetField("group") - andQuery.AddQuery(queryPack) - } - - req := bleve.NewSearchRequestOptions(andQuery, pageSize, (pageNum-1)*pageSize, false) - - index := m.Index - res, err := index.Search(req) - if err != nil { - return res, 0, 0, 0, err - } - - total := int(res.Total) - pageStart := (pageNum - 1) * pageSize - pageEnd := pageStart + len(res.Hits) - return res, total, pageStart, pageEnd, nil -} - -func (m *HelpManager) Search(ctx *MsgContext, text string, titleOnly bool, pageSize, pageNum int, group string) (res *bleve.SearchResult, total, pageStart, pageEnd int, err error) { - if pageSize <= 0 || pageNum <= 0 { - // 为了使Search的结果完全忠实于分页参数, 而不产生有结果但与分页不相符的情况 - return nil, 0, 0, 0, errors.New("分页参数错误") - } - - if m.EngineType == 0 { - return m.searchBleve(ctx, text, titleOnly, pageSize, pageNum, group) - } - - // 不是很好的做法,待优化 - items := HelpTextItems{} - var idLst []string - - m.TextMap.Range(func(id string, v *HelpTextItem) bool { - items = append(items, v) - idLst = append(idLst, id) - return true - }) - - hits := search.DocumentMatchCollection{} - matches := fuzzy.FindFrom(text, items) - - total = len(matches) - pageStart = (pageNum - 1) * pageSize - pageEnd = pageNum * pageSize - - if pageStart < total { - if pageEnd > total { - pageEnd = total - } - - for _, i := range matches[pageStart:pageEnd] { - hits = append(hits, &search.DocumentMatch{ - ID: idLst[i.Index], - Score: float64(i.Score), - }) - } - } else { - // 分页超出范围, 返回空结果 - pageStart = -1 - pageEnd = -1 - } - - return &bleve.SearchResult{ - Status: nil, - Request: nil, - Hits: hits, - Total: uint64(total), - }, total, pageStart, pageEnd, nil +func (m *HelpManager) Search(ctx *MsgContext, text string, titleOnly bool, pageSize, pageNum int, group string) (res *docengine.GeneralSearchResult, total, pageStart, pageEnd int, err error) { + return m.searchEngine.Search(ctx.Group.HelpPackages, text, titleOnly, pageSize, pageNum, group) } func (m *HelpManager) GetSuffixText() string { - switch m.EngineType { - case 0: - return "(本次搜索由全文搜索完成)" - default: - return "(本次搜索由快速文档查找完成)" - } + return m.searchEngine.GetSuffixText() } func (m *HelpManager) GetPrefixText() string { - switch m.EngineType { - case 0: - return "[全文搜索]" - default: - return "[快速文档查找]" - } + return m.searchEngine.GetPrefixText() } func (m *HelpManager) GetShowBestOffset() int { - switch m.EngineType { - case 0: - return 1 - default: - return 15 - } + return m.searchEngine.GetShowBestOffset() } -func (m *HelpManager) GetContent(item *HelpTextItem, depth int) string { +func (m *HelpManager) GetContent(item *docengine.HelpTextItem, depth int) string { if depth > 7 { return "{递归层数过多,不予显示}" } @@ -675,20 +501,15 @@ func (m *HelpManager) GetContent(item *HelpTextItem, depth int) string { result.WriteString(txt[formattedIdx:left]) formattedIdx = right name := txt[left+1 : right-1] - matched := false - // 注意: 效率更加不高 - m.TextMap.Range(func(key string, v *HelpTextItem) bool { - if v.Title == name { - result.WriteString(m.GetContent(v, depth+1)) - matched = true - return false - } - return true - }) - if !matched { + // 搜索TitleOnly,严格匹配Title的情形 + // 如果查询到对应数据,那么就调用m.GetContent + valueResult, err := m.searchEngine.GetHelpTextItemByTermTitle(name) + if err != nil { result.WriteByte('{') result.WriteString(name) result.WriteString(" - 未能找到}") + } else { + result.WriteString(m.GetContent(valueResult, depth+1)) } } result.WriteString(txt[formattedIdx:]) @@ -700,45 +521,56 @@ func generateHelpDocKey() string { return key } +// 修改 buildHelpDocTree 函数签名,添加进度参数 func buildHelpDocTree(node *HelpDoc, fn func(d *HelpDoc)) { - p, err := os.Stat(node.Path) - if err != nil { - return - } + // 收集所有节点 + allNodes := []*HelpDoc{node} - fn(node) + for i := 0; i < len(allNodes); i++ { + current := allNodes[i] - if !p.IsDir() { - return - } - - subs, err := os.ReadDir(node.Path) - if err != nil { - return - } + p, err := os.Stat(current.Path) + if err != nil { + continue + } - for _, sub := range subs { - if strings.HasPrefix(sub.Name(), ".") { + if !p.IsDir() { continue } - var child HelpDoc - child.Key = generateHelpDocKey() - child.Name = sub.Name() - child.Path = path.Join(node.Path, sub.Name()) - child.Group = node.Group - child.IsDir = sub.IsDir() - if sub.IsDir() { - child.Type = "dir" - child.Children = make([]*HelpDoc, 0) - } else { - child.Type = filepath.Ext(sub.Name()) + + subs, err := os.ReadDir(current.Path) + if err != nil { + continue } - fn(&child) - if sub.IsDir() { - buildHelpDocTree(&child, fn) + current.Children = make([]*HelpDoc, 0) + + for _, sub := range subs { + if strings.HasPrefix(sub.Name(), ".") { + continue + } + + var child HelpDoc + child.Key = generateHelpDocKey() + child.Name = sub.Name() + child.Path = path.Join(current.Path, sub.Name()) + child.Group = current.Group + child.IsDir = sub.IsDir() + + if sub.IsDir() { + child.Type = "dir" + child.Children = make([]*HelpDoc, 0) + } else { + child.Type = filepath.Ext(sub.Name()) + } + + allNodes = append(allNodes, &child) + current.Children = append(current.Children, &child) } - node.Children = append(node.Children, &child) + } + for _, current := range allNodes { + // 调用处理函数 + fn(current) } } @@ -925,12 +757,13 @@ func (m *HelpManager) GetHelpItemPage(pageNum, pageSize int, id, group, from, ti return 0, HelpTextVos{} } + // 如果ID不为空 if id != "" { - item, ok := m.TextMap.Load(id) - if ok && - strings.Contains(item.Group, group) && - strings.Contains(item.From, from) && - strings.Contains(item.Title, title) { + // 加载对应ID的数据 + item, err := m.searchEngine.GetItemByID(id) + // 若成功 + if err == nil { + // 返回这条数据 vo := HelpTextVo{ Group: item.Group, From: item.From, @@ -944,36 +777,25 @@ func (m *HelpManager) GetHelpItemPage(pageNum, pageSize int, id, group, from, ti } return 0, HelpTextVos{} } - temp := make(HelpTextVos, 0, m.TextMap.Len()) - m.TextMap.Range(func(i string, item *HelpTextItem) bool { - if strings.Contains(item.Group, group) && - strings.Contains(item.From, from) && - strings.Contains(item.Title, title) { - vo := HelpTextVo{ - Group: item.Group, - From: item.From, - Title: item.Title, - Content: item.Content, - PackageName: item.PackageName, - KeyWords: item.KeyWords, - } - vo.ID, _ = strconv.Atoi(i) - temp = append(temp, vo) + // ID为空的情形,分页查询数据 + total, result, err := m.searchEngine.PaginateDocuments(pageSize, pageNum, group, from, title) + if err != nil { + return 0, nil + } + var items = make(HelpTextVos, 0) + for _, item := range result { + vo := HelpTextVo{ + Group: item.Group, + From: item.From, + Title: item.Title, + Content: item.Content, + PackageName: item.PackageName, + KeyWords: item.KeyWords, } - return true - }) - - sort.Sort(temp) - - start := (pageNum - 1) * pageSize - end := start + pageSize - total := len(temp) - if start >= total { - return total, HelpTextVos{} - } else if end < total { - return total, temp[start:end] + vo.ID, _ = strconv.Atoi(id) + items = append(items, vo) } - return total, temp[start:] + return int(total), items } // SetDefaultHelpGroup 设置群默认搜索分组 diff --git a/dice/dice_manager.go b/dice/dice_manager.go index 14b58586..3d0704de 100644 --- a/dice/dice_manager.go +++ b/dice/dice_manager.go @@ -113,7 +113,7 @@ func (dm *DiceManager) InitHelp() { _ = os.MkdirAll("./data/helpdoc", 0755) dm.Help = new(HelpManager) dm.Help.Parent = dm - dm.Help.EngineType = dm.HelpDocEngineType + dm.Help.EngineType = EngineType(dm.HelpDocEngineType) dm.Help.Load() dm.IsHelpReloading = false } diff --git a/dice/docengine/bleve.go b/dice/docengine/bleve.go new file mode 100644 index 00000000..4692989d --- /dev/null +++ b/dice/docengine/bleve.go @@ -0,0 +1,323 @@ +package docengine + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "sync/atomic" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + index "github.com/blevesearch/bleve_index_api" + + log "sealdice-core/utils/kratos" +) + +type BleveSearchEngine struct { + Index bleve.Index + batch *bleve.Batch + batchSize int + CurID uint64 +} + +var indexDir = "./data/_index" +var reSpace = regexp.MustCompile(`\s+`) + +// getNextID 使用原子操作,避免并发问题 +func (d *BleveSearchEngine) getNextID() string { + // 使用原子操作安全递增 CurID + nextID := atomic.AddUint64(&d.CurID, 1) + return strconv.FormatUint(nextID, 10) +} + +// NewEngine 创建并初始化 BleveSearchEngine +func NewBleveSearchEngine() (*BleveSearchEngine, error) { + engine := &BleveSearchEngine{} + err := engine.Init() + if err != nil { + return nil, err + } + return engine, nil +} + +func (d *BleveSearchEngine) GetSuffixText() string { + return "(本次搜索由全文搜索完成)" +} + +func (d *BleveSearchEngine) GetPrefixText() string { + return "[全文搜索]" +} + +func (d *BleveSearchEngine) GetShowBestOffset() int { + return 1 +} + +func (d *BleveSearchEngine) Init() error { + mapping := bleve.NewIndexMapping() + docMapping := bleve.NewDocumentMapping() + contentFieldMapping := bleve.NewTextFieldMapping() + keywordMapping := bleve.NewKeywordFieldMapping() + // 注意: 这里group,from,title,package都是keywordMapping,这样就能进行精确搜索。 + docMapping.AddFieldMappingsAt("group", keywordMapping) + docMapping.AddFieldMappingsAt("from", keywordMapping) + docMapping.AddFieldMappingsAt("title", contentFieldMapping) + // Content才是真正的文档 + docMapping.AddFieldMappingsAt("content", contentFieldMapping) + docMapping.AddFieldMappingsAt("package", keywordMapping) + mapping.AddDocumentMapping("helpdoc", docMapping) + mapping.TypeField = "_type" + i, err := bleve.New(indexDir, mapping) + if err != nil { + return err + } + d.Index = i + // 初始化ID列表 + d.CurID = 0 + // 初始化新的batch + d.batch = d.Index.NewBatch() + return nil +} + +func (d *BleveSearchEngine) Close() { + if d.Index != nil { + _ = d.Index.Close() + d.Index = nil + } +} + +func (d *BleveSearchEngine) GetTotalID() uint64 { + return d.CurID +} + +// AddItem 这里引用了dice,其实不妥,应该将它单独拆出来的。 +func (d *BleveSearchEngine) AddItem(item HelpTextItem) (string, error) { + // 如果batch为空,初始化一个batch + if d.batch == nil { + return "", errors.New("已通过end参数执行AddItemApply,不允许新增文档。请检查代码逻辑") + } + id := d.getNextID() + data := map[string]string{ + "group": item.Group, + "from": item.From, + "title": item.Title, + "content": item.Content, + "package": item.PackageName, + "_type": "helpdoc", + } + d.batchSize++ + // 五十一次执行 + if d.batchSize >= 50 { + err := d.AddItemApply(false) + d.batchSize = 0 + if err != nil { + return "", err + } + } + return id, d.batch.Index(id, data) +} + +// AddItemApply 这里认为是真正执行插入文档的逻辑 +// 由于现在已经将执行函数改为了可按文件执行,所以可以按文件进行Apply,这应当不会有太大的量级。 +// end代表是否是最后一次执行,一般用在所有的数据都处理完之后,关闭逻辑的时候使用,如bleve batch重复利用后最后销毁 +// TODO: 似乎很奇怪,这家伙貌似不会回收内存的吗? +func (d *BleveSearchEngine) AddItemApply(end bool) error { + if d.batch != nil { + // 执行batch + err := d.Index.Batch(d.batch) + if err != nil { + return err + } + // 如果是最后一批 + if end { + // 销毁batch + d.batch.Reset() + d.batch = nil + } else { + // 否则重置batch + d.batch.Reset() + } + return err + } + return nil +} + +func (d *BleveSearchEngine) Search(helpPackages []string, text string, titleOnly bool, pageSize, pageNum int, group string) (*GeneralSearchResult, int, int, int, error) { + // 在标题中查找 + queryTitle := query.NewMatchPhraseQuery(text) + queryTitle.SetField("title") + + titleOrContent := bleve.NewDisjunctionQuery(queryTitle) + + // 在正文中查找 + if !titleOnly { + for _, i := range reSpace.Split(text, -1) { + queryContent := query.NewMatchPhraseQuery(i) + queryContent.SetField("content") + titleOrContent.AddQuery(queryContent) + } + } + + andQuery := bleve.NewConjunctionQuery(titleOrContent) + + // 限制查询组 + for _, i := range helpPackages { + queryPack := query.NewMatchPhraseQuery(i) + queryPack.SetField("package") + andQuery.AddQuery(queryPack) + } + + // 查询指定文档组 + if group != "" { + queryPack := query.NewMatchPhraseQuery(group) + queryPack.SetField("group") + andQuery.AddQuery(queryPack) + } + + req := bleve.NewSearchRequestOptions(andQuery, pageSize, (pageNum-1)*pageSize, false) + // 设置要被返回的数据 + req.Fields = []string{"*"} + res, err := d.Index.Search(req) + if err != nil { + return nil, 0, 0, 0, err + } + var resultList = make(MatchCollection, 0) + for _, hit := range res.Hits { + result := MatchResult{ + ID: hit.ID, + Fields: hit.Fields, + Score: hit.Score, + } + resultList = append(resultList, &result) + } + // 转换搜索格式 + responseResult := GeneralSearchResult{ + Hits: resultList, + Total: res.Total, + } + total := int(res.Total) + pageStart := (pageNum - 1) * pageSize + pageEnd := pageStart + len(res.Hits) + return &responseResult, total, pageStart, pageEnd, nil +} + +// 下面的代码都应该重构,因为它们返回的不是我们想要的结果 +// PaginateAllDocuments 分页查询所有文档 +// TODO:这里坏了,没有办法用,本来应该是精确匹配NewMatchQuery +func (d *BleveSearchEngine) PaginateDocuments(pageSize, pageNum int, group, from, title string) (uint64, []*HelpTextItem, error) { + var items []*HelpTextItem + // 只有Keyword才支持NewTermQuery + conjunctionQuery := bleve.NewConjunctionQuery() + if group != "" { + groupQuery := bleve.NewTermQuery(group) + groupQuery.SetField("group") + conjunctionQuery.AddQuery(groupQuery) + } + if from != "" { + fromQuery := bleve.NewTermQuery(from) + fromQuery.SetField("from") + conjunctionQuery.AddQuery(fromQuery) + } + if title != "" { + titleQuery := bleve.NewTermQuery(title) + titleQuery.SetField("title") + conjunctionQuery.AddQuery(titleQuery) + } + + // 计算分页参数 + fromInt := (pageNum - 1) * pageSize // 起始位置 + if fromInt < 0 { + fromInt = 0 + } + var searchRequest *bleve.SearchRequest + // 创建查询请求,设置分页参数 + if group == "" && from == "" && title == "" { + searchRequest = bleve.NewSearchRequestOptions(bleve.NewMatchAllQuery(), pageSize, fromInt, false) + } else { + searchRequest = bleve.NewSearchRequestOptions(conjunctionQuery, pageSize, fromInt, true) + } + searchRequest.Fields = []string{"*"} // 设置需要返回的字段 + + // 执行查询 + searchResult, err := d.Index.Search(searchRequest) + if err != nil { + return 0, nil, err + } + + // 处理结果 + for _, hit := range searchResult.Hits { + fields := hit.Fields + item := &HelpTextItem{ + Group: fmt.Sprintf("%v", fields["group"]), + From: fmt.Sprintf("%v", fields["from"]), + Title: fmt.Sprintf("%v", fields["title"]), + Content: fmt.Sprintf("%v", fields["content"]), + PackageName: fmt.Sprintf("%v", fields["package"]), + KeyWords: "", // 暂时空值 + RelatedExt: nil, // 暂时空值 + } + items = append(items, item) + } + return searchResult.Total, items, nil +} + +func (d *BleveSearchEngine) GetItemByID(id string) (*HelpTextItem, error) { + document, err := d.Index.Document(id) + if err != nil { + return nil, err + } + // 检查是否找到文档 + if document == nil { + return nil, errors.New("未找到匹配的文档") + } + item := HelpTextItem{} + // 看了看源码,意思就是这样访问文档内的所有fields + document.VisitFields(func(field index.Field) { + name := field.Name() + value := string(field.Value()) + // 这里的代码有点抽象…… + switch name { + case "group": + item.Group = value + case "from": + item.From = value + case "title": + item.Title = value + case "content": + item.Content = value + case "package": + item.PackageName = value + // 好像会碰到Type的参数? + default: + log.Debugf("这是个什么参数 %s", name) + } + }) + return &item, nil +} + +// 精确查询title +func (d *BleveSearchEngine) GetHelpTextItemByTermTitle(title string) (*HelpTextItem, error) { + newTermQuery := query.NewTermQuery(title) + newTermQuery.SetField("title") // 精确匹配title + req := bleve.NewSearchRequest(newTermQuery) + req.Fields = []string{"*"} + res, err := d.Index.Search(req) + if err != nil { + return nil, err + } + // 取出结果 + if len(res.Hits) > 0 { + fields := res.Hits[0].Fields + return &HelpTextItem{ + Group: fmt.Sprintf("%v", fields["group"]), + From: fmt.Sprintf("%v", fields["from"]), + Title: fmt.Sprintf("%v", fields["title"]), + Content: fmt.Sprintf("%v", fields["content"]), + PackageName: fmt.Sprintf("%v", fields["package"]), + // 这俩是什么东西?! + KeyWords: "", + RelatedExt: nil, + }, nil + } + return nil, errors.New("查询失败,未查询到数据") +} diff --git a/dice/docengine/enter.go b/dice/docengine/enter.go new file mode 100644 index 00000000..faa5795a --- /dev/null +++ b/dice/docengine/enter.go @@ -0,0 +1,54 @@ +package docengine + +type MatchResult struct { + ID string `json:"id"` + Fields map[string]interface{} `json:"fields"` + Score float64 `json:"score"` +} + +type Fields struct { +} + +type MatchCollection []*MatchResult + +// GeneralSearchResult Copied from bleve +type GeneralSearchResult struct { + Hits MatchCollection + Total uint64 +} + +type HelpTextItem struct { + Group string + From string + Title string + Content string + PackageName string + // 这俩玩意有用? + KeyWords string + RelatedExt []string +} + +// SearchEngine TODO: 进一步优化结构,封装成通用的搜索 +type SearchEngine interface { + GetSuffixText() string + GetPrefixText() string + GetShowBestOffset() int + // Init 初始化搜索引擎 + Init() error + // Close 关闭搜索引擎 + Close() + // AddItem 添加文档条目,返回添加文档的ID + AddItem(item HelpTextItem) (string, error) + // AddItemApply 提交文档条目 + AddItemApply(end bool) error + // Search 搜索文档条目 + Search(helpPackages []string, text string, titleOnly bool, pageSize, pageNum int, group string) (*GeneralSearchResult, int, int, int, error) + // GetHelpTextItemByTermTitle 精确查询title,用于嵌套获取数据的情形 + GetHelpTextItemByTermTitle(title string) (*HelpTextItem, error) + // GetItemByID 通过ID获取Item数据的方案 + GetItemByID(id string) (*HelpTextItem, error) + // PaginateDocuments 分页获取数据 + PaginateDocuments(pageSize, pageNum int, group, from, title string) (uint64, []*HelpTextItem, error) + // GetTotalID 获取当前ID总数,注意,ID必须是顺序排列的 + GetTotalID() uint64 +} diff --git a/go.mod b/go.mod index 99dc5161..23b453cb 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0 github.com/antlabs/strsim v0.0.3 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/blevesearch/bleve/v2 v2.3.10 + github.com/blevesearch/bleve/v2 v2.4.3 github.com/bwmarrin/discordgo v0.28.1 github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd github.com/dop251/goja_nodejs v0.0.0-20240728170619-29b559befffc @@ -67,10 +67,10 @@ require ( github.com/yuin/goldmark v1.7.4 go.etcd.io/bbolt v1.3.11 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.28.0 + golang.org/x/crypto v0.29.0 golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 - golang.org/x/sys v0.26.0 - golang.org/x/text v0.19.0 + golang.org/x/sys v0.27.0 + golang.org/x/text v0.20.0 golang.org/x/time v0.5.0 gopkg.in/elazarl/goproxy.v1 v1.0.0-20180725130230-947c36da3153 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df @@ -80,6 +80,7 @@ require ( ) require ( + github.com/blevesearch/bleve_index_api v1.1.13 github.com/go-gorm/caches/v4 v4.0.5 github.com/gofrs/flock v0.12.1 github.com/spaolacci/murmur3 v1.1.0 @@ -90,24 +91,25 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect - github.com/RoaringBitmap/roaring v1.2.3 // indirect - github.com/bits-and-blooms/bitset v1.2.2 // indirect + github.com/RoaringBitmap/roaring v1.9.3 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.2.0 // indirect - github.com/blevesearch/bleve_index_api v1.0.6 // indirect - github.com/blevesearch/geo v0.1.18 // indirect + github.com/blevesearch/geo v0.1.20 // indirect + github.com/blevesearch/go-faiss v1.0.23 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.16 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect - github.com/blevesearch/vellum v1.0.10 // indirect + github.com/blevesearch/vellum v1.0.11 // indirect github.com/blevesearch/zapx/v11 v11.3.10 // indirect github.com/blevesearch/zapx/v12 v12.3.10 // indirect github.com/blevesearch/zapx/v13 v13.3.10 // indirect github.com/blevesearch/zapx/v14 v14.3.10 // indirect - github.com/blevesearch/zapx/v15 v15.3.13 // indirect + github.com/blevesearch/zapx/v15 v15.3.16 // indirect + github.com/blevesearch/zapx/v16 v16.1.8 // indirect github.com/disintegration/imaging v1.6.2 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -130,7 +132,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/geo v0.0.0-20230404232722-c4acd7a044dc // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/hhrutter/lzw v1.0.0 // indirect @@ -158,6 +160,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/sacOO7/go-logger v0.0.0-20180719173527-9ac9add5a50d // indirect + github.com/stretchr/testify v1.10.0 // indirect github.com/sunshineplan/pdf v1.0.7 // indirect github.com/sunshineplan/tiff v0.0.0-20220128141034-29b9d69bd906 // indirect github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect @@ -175,9 +178,9 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/image v0.19.0 // indirect golang.org/x/mod v0.18.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/term v0.25.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/term v0.26.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect @@ -191,6 +194,9 @@ require ( replace ( github.com/Szzrain/dodo-open-go v0.2.7 => github.com/sealdice/dodo-open-go v0.2.8 + // Try to fix arm64 bug with better snappy. + github.com/blevesearch/zapx/v16 v16.1.8 => github.com/PaienNate/zapx/v16 v16.1.9 + // Try to fix sqlite in cgofree github.com/glebarez/sqlite v1.11.0 => github.com/PaienNate/sqlite v0.0.0-20241102151933-067d82f14685 github.com/lonelyevil/kook v0.0.31 => github.com/sealdice/kook v0.0.3 github.com/sacOO7/gowebsocket v0.0.0-20221109081133-70ac927be105 => github.com/fy0/GoWebsocket v0.0.0-20231128163937-aa5c110b25c6 diff --git a/go.sum b/go.sum index b6c25c7d..1f12c53d 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,10 @@ github.com/Milly/go-base2048 v0.1.0 h1:7ZgpCR3cjcAAVqIo+B8Q3P1+VFHRS8zilzAq062rU github.com/Milly/go-base2048 v0.1.0/go.mod h1:kl6eYBwGnoIjv8k9UmgS+bekm6870ojptcVnT11e3jE= github.com/PaienNate/sqlite v0.0.0-20241102151933-067d82f14685 h1:O6OPpCufcEJD+eWENAwuwVZHONKmP77X2wlzMTQZ5gg= github.com/PaienNate/sqlite v0.0.0-20241102151933-067d82f14685/go.mod h1:GajiCpqLxU0a1gP13oAEiJAx9r87kVSdfEQy4O69ZTo= -github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= -github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/PaienNate/zapx/v16 v16.1.9 h1:GA4jIOx9OPFqTGym7ucqKNNSw01BaIyOIjDPxR3w47A= +github.com/PaienNate/zapx/v16 v16.1.9/go.mod h1:zuxVgVaLZ0g4lZvrv06xDc24N6nLCOzXYHVkXI7LMHM= +github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= +github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/ShiraazMoollatjie/goluhn v0.0.0-20211017190329-0d86158c056a h1:NPnGVqpua4c1iEFVdxnBJA9viP5bo2Zp2jfflbcjdto= github.com/ShiraazMoollatjie/goluhn v0.0.0-20211017190329-0d86158c056a/go.mod h1:5LI6VqIHoGmWsR0EJLbct5bBrtM/0pTonaAyGKmFk9U= github.com/Szzrain/DingTalk-go v0.0.8-alpha h1:mSR/ORDDjtndoR12WrEdd3hdxxXXm9VMQ/r75NJkkkE= @@ -28,32 +30,35 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.2.0 h1:N+g3GTQ0TVbghahYyzwkQbMZR+IwIwFFC8dpIChtN0U= github.com/bits-and-blooms/bloom/v3 v3.2.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= -github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= -github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= -github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= -github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= -github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/bleve/v2 v2.4.3 h1:XDYj+1prgX84L2Cf+V3ojrOPqXxy0qxyd2uLMmeuD+4= +github.com/blevesearch/bleve/v2 v2.4.3/go.mod h1:hEPDPrbYw3vyrm5VOa36GyS4bHWuIf4Fflp7460QQXY= +github.com/blevesearch/bleve_index_api v1.1.13 h1:+nrA6oRJr85aCPyqaeZtsruObwKojutfonHJin/BP48= +github.com/blevesearch/bleve_index_api v1.1.13/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.23 h1:Wmc5AFwDLKGl2L6mjLX1Da3vCL0EKa2uHHSorcIS1Uc= +github.com/blevesearch/go-faiss v1.0.23/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/scorch_segment_api/v2 v2.2.16 h1:uGvKVvG7zvSxCwcm4/ehBa9cCEuZVE+/zvrSl57QUVY= +github.com/blevesearch/scorch_segment_api/v2 v2.2.16/go.mod h1:VF5oHVbIFTu+znY1v30GjSpT5+9YFs9dV2hjvuh34F0= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= -github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= -github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/blevesearch/vellum v1.0.11 h1:SJI97toEFTtA9WsDZxkyGTaBWFdWl1n2LEDCXLCq/AU= +github.com/blevesearch/vellum v1.0.11/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= @@ -62,8 +67,8 @@ github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIq github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= -github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v15 v15.3.16 h1:Ct3rv7FUJPfPk99TI/OofdC+Kpb4IdyfdMH48sb+FmE= +github.com/blevesearch/zapx/v15 v15.3.16/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4= @@ -176,8 +181,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -376,8 +381,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/sunshineplan/imgconv v1.1.4 h1:lViOZUbDIgW8o74naySXJqZOFgXSW1AdU/cdzZRnVTo= github.com/sunshineplan/imgconv v1.1.4/go.mod h1:Bc4qh4Z+nslcq+Csck01QZgzWvirKUdltRI7vnEAKd8= github.com/sunshineplan/pdf v1.0.7 h1:62xlc079jh4tGLDjiihyyhwVFkn0IsxLyDpHplbG9Ew= @@ -463,8 +469,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -492,16 +498,16 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -532,16 +538,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -550,8 +556,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= diff --git a/main.go b/main.go index f50fa45f..37190dc6 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,6 @@ package main +// _ "net/http/pprof" import ( "errors" "fmt" @@ -16,8 +17,6 @@ import ( "syscall" "time" - // _ "net/http/pprof" - "github.com/gofrs/flock" "github.com/jessevdk/go-flags" "github.com/labstack/echo/v4" @@ -72,6 +71,7 @@ func cleanupCreate(diceManager *dice.DiceManager) func() { if i.IsAlreadyLoadConfig { i.Config.BanList.SaveChanged(i) i.Save(true) + i.AttrsManager.Stop() for _, j := range i.ExtList { if j.Storage != nil { // 关闭