From 804eeb5e1956a0e6102977d8b607f3dd9313cbae Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 20:21:11 +0800 Subject: [PATCH 1/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=9D=97?= =?UTF-8?q?=E6=A0=91=E6=95=B0=E6=8D=AE=E5=AD=98=E5=8F=96=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7168?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 4cddf9217..5c435c65c 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -488,10 +488,15 @@ func SaveBlockTree(force bool) { os.MkdirAll(util.BlockTreePath, 0755) size := uint64(0) - blockTrees.Range(func(key, value interface{}) bool { - slice := value.(*btSlice) + poolSize := runtime.NumCPU() + waitGroup := &sync.WaitGroup{} + p, _ := ants.NewPoolWithFunc(poolSize, func(arg interface{}) { + defer waitGroup.Done() + + key := arg.(map[string]interface{})["key"].(string) + slice := arg.(map[string]interface{})["value"].(*btSlice) if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) { - return true + return } slice.m.Lock() @@ -499,24 +504,32 @@ func SaveBlockTree(force bool) { if nil != err { logging.LogErrorf("marshal block tree failed: %s", err) os.Exit(util.ExitCodeBlockTreeErr) - return false + return } slice.m.Unlock() - p := filepath.Join(util.BlockTreePath, key.(string)) + ".msgpack" + p := filepath.Join(util.BlockTreePath, key) + ".msgpack" if err = gulu.File.WriteFileSafer(p, data, 0644); nil != err { logging.LogErrorf("write block tree failed: %s", err) os.Exit(util.ExitCodeBlockTreeErr) - return false + return } slice.changed = time.Time{} size += uint64(len(data)) + }) + + blockTrees.Range(func(key, value interface{}) bool { + waitGroup.Add(1) + p.Invoke(map[string]interface{}{"key": key, "value": value}) return true }) - runtime.GC() + waitGroup.Wait() + p.Release() - if elapsed := time.Since(start).Seconds(); 2 < elapsed { + runtime.GC() + elapsed := time.Since(start).Seconds() + if 2 < elapsed { logging.LogWarnf("save block tree [size=%s] to [%s], elapsed [%.2fs]", humanize.Bytes(size), util.BlockTreePath, elapsed) } } From 13e6ff8557841885049a224d8380fd149fd15a71 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 20:45:39 +0800 Subject: [PATCH 2/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/queue.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index df9cfe9f2..c8528f850 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -128,11 +128,15 @@ func FlushQueue() { return } - if 16 < i && 0 == i%256 { + if 16 < i && 0 == i%128 { runtime.GC() } } + if 128 < len(ops) { + runtime.GC() + } + elapsed := time.Now().Sub(start).Milliseconds() if 5000 < elapsed { logging.LogInfof("op tx [%dms]", elapsed) From f6d290bd61508e239f790ccf9541a532df497883 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Thu, 26 Jan 2023 20:57:52 +0800 Subject: [PATCH 3/3] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/blocktree.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index 5c435c65c..ff18cf3e4 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -519,6 +519,11 @@ func SaveBlockTree(force bool) { }) blockTrees.Range(func(key, value interface{}) bool { + slice := value.(*btSlice) + if !force && (slice.changed.IsZero() || slice.changed.After(start.Add(-7*time.Second))) { + return true + } + waitGroup.Add(1) p.Invoke(map[string]interface{}{"key": key, "value": value}) return true