Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-12-24 02:20:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

♻️ Improve database loading performance https://github.com/siyuan-note/siyuan/issues/12818

Browse source
This commit is contained in:
Daniel 2024-10-18 00:03:43 +08:00
parent d675496e52
commit 9cea8ca069
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
2 changed files with 41 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

40
kernel/filesys/tree.go
View file

@ -25,11 +25,13 @@ import (
"path/filepath"
"strings"
"sync"
"time"
"github.com/88250/lute"
"github.com/88250/lute/parse"
"github.com/88250/lute/render"
jsoniter "github.com/json-iterator/go"
"github.com/panjf2000/ants/v2"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/cache"
@ -53,7 +55,7 @@ func LoadTrees(ids []string) (ret map[string]*parse.Tree) {
blockIDs[bt.RootID] = append(blockIDs[bt.RootID], bt.ID)
}
trees, errs := BatchLoadTrees(boxIDs, paths, luteEngine)
trees, errs := batchLoadTrees(boxIDs, paths, luteEngine)
for i := range trees {
tree := trees[i]
err := errs[i]
@ -82,31 +84,37 @@ func LoadTree(boxID, p string, luteEngine *lute.Lute) (ret *parse.Tree, err erro
return
}
func BatchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
var wg sync.WaitGroup
func batchLoadTrees(boxIDs, paths []string, luteEngine *lute.Lute) (ret []*parse.Tree, errs []error) {
waitGroup := sync.WaitGroup{}
lock := sync.Mutex{}
loaded := map[string]bool{}
start := time.Now()
p, _ := ants.NewPoolWithFunc(8, func(arg interface{}) {
defer waitGroup.Done()
i := arg.(int)
boxID := boxIDs[i]
path := paths[i]
tree, err := LoadTree(boxID, path, luteEngine)
lock.Lock()
ret = append(ret, tree)
errs = append(errs, err)
lock.Unlock()
})
for i := range paths {
if loaded[boxIDs[i]+paths[i]] {
continue
}
loaded[boxIDs[i]+paths[i]] = true
wg.Add(1)
go func(i int) {
defer wg.Done()
boxID := boxIDs[i]
path := paths[i]
tree, err := LoadTree(boxID, path, luteEngine)
lock.Lock()
ret = append(ret, tree)
errs = append(errs, err)
lock.Unlock()
}(i)
waitGroup.Add(1)
p.Invoke(i)
}
wg.Wait()
waitGroup.Wait()
p.Release()
logging.LogInfof("batch load trees [%d] cost [%s]", len(paths), time.Since(start))
return
}