From 98f682e507997255141455e7d925ea43ea739c5a Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 21:08:08 +0800 Subject: [PATCH 1/7] =?UTF-8?q?:art:=20=E5=B5=8C=E5=85=A5=E5=9D=97?= =?UTF-8?q?=E7=BA=B3=E5=85=A5=E5=BC=95=E7=94=A8=E8=AE=A1=E6=95=B0=E5=92=8C?= =?UTF-8?q?=E5=8F=8D=E9=93=BE=20https://github.com/siyuan-note/siyuan/issu?= =?UTF-8?q?es/7096?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/sql/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sql/database.go b/kernel/sql/database.go index b11a12003..3c8ca4230 100644 --- a/kernel/sql/database.go +++ b/kernel/sql/database.go @@ -361,7 +361,7 @@ func buildEmbedRef(tree *parse.Tree, embedNode *ast.Node) *Ref { RootID: tree.ID, Box: tree.Box, Path: tree.Path, - Content: "", + Content: "", // 通过嵌入块构建引用时定义块可能还没有入库，所以这里统一不填充内容 Markdown: "", Type: treenode.TypeAbbr(embedNode.Type.String()), } From 492389470f8b47936f3e84bbb2c202eab2a3d5e6 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:00:30 +0800 Subject: [PATCH 2/7] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/repo.go | 6 +- kernel/main.go | 5 +- kernel/mobile/kernel.go | 5 +- kernel/model/box.go | 9 ++- kernel/model/sync.go | 32 ++------ kernel/task/queue.go | 162 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 181 insertions(+), 38 deletions(-) create mode 100644 kernel/task/queue.go diff --git a/kernel/api/repo.go b/kernel/api/repo.go index 6fe7f08d2..26d18ac84 100644 --- a/kernel/api/repo.go +++ b/kernel/api/repo.go @@ -113,11 +113,7 @@ func checkoutRepo(c *gin.Context) { } id := arg["id"].(string) - if err := model.CheckoutRepo(id); nil != err { - ret.Code = -1 - ret.Msg = model.Conf.Language(141) - return - } + model.CheckoutRepo(id) } func downloadCloudSnapshot(c *gin.Context) { diff --git a/kernel/main.go b/kernel/main.go index ff84ace50..426622bd8 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -23,6 +23,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/model" "github.com/siyuan-note/siyuan/kernel/server" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -42,6 +43,8 @@ func main() { model.LoadFlashcards() model.LoadAssetsTexts() + go task.Loop() + go model.AutoGenerateDocHistory() go model.AutoSync() go model.AutoStat() @@ -49,7 +52,7 @@ func main() { util.PushClearAllMsg() go model.AutoRefreshCheck() go model.AutoFlushTx() - go sql.AutoFlushTreeQueue() + go sql.AutoFlushQueue() go treenode.AutoFlushBlockTree() go cache.LoadAssets() go model.AutoFixIndex() diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index a5feb8438..67b339a1d 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -27,6 +27,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/model" "github.com/siyuan-note/siyuan/kernel/server" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" _ "golang.org/x/mobile/bind" @@ -56,6 +57,8 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang model.LoadFlashcards() model.LoadAssetsTexts() + go task.Loop() + go model.AutoGenerateDocHistory() go model.AutoSync() go model.AutoStat() @@ -63,7 +66,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang util.PushClearAllMsg() go model.AutoRefreshCheck() go model.AutoFlushTx() - go sql.AutoFlushTreeQueue() + go sql.AutoFlushQueue() go treenode.AutoFlushBlockTree() go cache.LoadAssets() go model.AutoFixIndex() diff --git a/kernel/model/box.go b/kernel/model/box.go index a90fb5677..02ad30924 100644 --- a/kernel/model/box.go +++ b/kernel/model/box.go @@ -39,6 +39,7 @@ import ( "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -504,10 +505,11 @@ func genTreeID(tree *parse.Tree) { return } -var isFullReindexing = false - func FullReindex() { - isFullReindexing = true + task.PrependTask(task.DatabaseIndexFull, fullReindex) +} + +func fullReindex() { util.PushEndlessProgress(Conf.Language(35)) WaitForWritingFiles() @@ -526,7 +528,6 @@ func FullReindex() { LoadFlashcards() util.PushEndlessProgress(Conf.Language(58)) - isFullReindexing = false go func() { time.Sleep(1 * time.Second) util.ReloadUI() diff --git a/kernel/model/sync.go b/kernel/model/sync.go index 67fd8e43d..aa67f5966 100644 --- a/kernel/model/sync.go +++ b/kernel/model/sync.go @@ -32,6 +32,7 @@ import ( "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -106,6 +107,10 @@ func BootSyncData() { } func SyncData(boot, exit, byHand bool) { + task.PrependTask(task.CloudSync, syncData, boot, exit, byHand) +} + +func syncData(boot, exit, byHand bool) { defer logging.Recover() if !boot && !exit && 2 == Conf.Sync.Mode && !byHand { @@ -267,53 +272,35 @@ func SetCloudSyncDir(name string) { return } - syncLock.Lock() - defer syncLock.Unlock() - Conf.Sync.CloudName = name Conf.Save() } func SetSyncGenerateConflictDoc(b bool) { - syncLock.Lock() - defer syncLock.Unlock() - Conf.Sync.GenerateConflictDoc = b Conf.Save() return } func SetSyncEnable(b bool) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - Conf.Sync.Enabled = b Conf.Save() return } func SetSyncMode(mode int) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - Conf.Sync.Mode = mode Conf.Save() return } func SetSyncProvider(provider int) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - Conf.Sync.Provider = provider Conf.Save() return } func SetSyncProviderS3(s3 *conf.S3) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - s3.Endpoint = strings.TrimSpace(s3.Endpoint) s3.Endpoint = util.NormalizeEndpoint(s3.Endpoint) s3.AccessKey = strings.TrimSpace(s3.AccessKey) @@ -328,9 +315,6 @@ func SetSyncProviderS3(s3 *conf.S3) (err error) { } func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) { - syncLock.Lock() - defer syncLock.Unlock() - webdav.Endpoint = strings.TrimSpace(webdav.Endpoint) webdav.Endpoint = util.NormalizeEndpoint(webdav.Endpoint) webdav.Username = strings.TrimSpace(webdav.Username) @@ -350,9 +334,6 @@ func CreateCloudSyncDir(name string) (err error) { return } - syncLock.Lock() - defer syncLock.Unlock() - name = strings.TrimSpace(name) name = gulu.Str.RemoveInvisible(name) if !cloud.IsValidCloudDirName(name) { @@ -380,9 +361,6 @@ func RemoveCloudSyncDir(name string) (err error) { msgId := util.PushMsg(Conf.Language(116), 15000) - syncLock.Lock() - defer syncLock.Unlock() - if "" == name { return } diff --git a/kernel/task/queue.go b/kernel/task/queue.go new file mode 100644 index 000000000..e5649ebd3 --- /dev/null +++ b/kernel/task/queue.go @@ -0,0 +1,162 @@ +// SiYuan - Build Your Eternal Digital Garden +// Copyright (c) 2020-present, b3log.org +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package task + +import ( + "reflect" + "sync" + "time" + + "github.com/siyuan-note/logging" +) + +var ( + taskQueue []*Task + taskQueueStatus int + queueLock = sync.Mutex{} + taskLock = sync.Mutex{} +) + +const ( + QueueStatusRunning = iota + QueueStatusClosing +) + +type Task struct { + Action string + Handler reflect.Value + Args []interface{} + Created time.Time +} + +func PrependTask(action string, handler interface{}, args ...interface{}) { + queueLock.Lock() + defer queueLock.Unlock() + + if QueueStatusRunning != taskQueueStatus { + logging.LogWarnf("task queue is paused, action [%s] will be ignored", action) + return + } + + cancelTask(action) + taskQueue = append([]*Task{newTask(action, handler, args...)}, taskQueue...) +} + +func AppendTask(action string, handler interface{}, args ...interface{}) { + queueLock.Lock() + defer queueLock.Unlock() + + if QueueStatusRunning != taskQueueStatus { + logging.LogWarnf("task queue is paused, action [%s] will be ignored", action) + return + } + + cancelTask(action) + taskQueue = append(taskQueue, newTask(action, handler, args...)) +} + +func CancelTask(actions ...string) { + queueLock.Lock() + defer queueLock.Unlock() + + cancelTask(actions...) +} + +func cancelTask(actions ...string) { + for i := len(taskQueue) - 1; i >= 0; i-- { + task := taskQueue[i] + for _, action := range actions { + if action == task.Action { + taskQueue = append(taskQueue[:i], taskQueue[i+1:]...) + break + } + } + } +} + +func newTask(action string, handler interface{}, args ...interface{}) *Task { + return &Task{ + Action: action, + Handler: reflect.ValueOf(handler), + Args: args, + Created: time.Now(), + } +} + +const ( + CloudSync = "task.cloud.sync" // 数据同步 + RepoCheckout = "task.repo.checkout" // 从快照中检出 + DatabaseIndexFull = "task.database.index.full" // 重建索引 + DatabaseIndex = "task.database.index" // 数据库所以队列 + DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 + OCRImage = "task.ocr.image" // 图片 OCR 提取文本 + DatabaseIndexEmbedBlock = "task.database.index.embedblock" // 数据库索引嵌入块 +) + +func Loop() { + for { + time.Sleep(10 * time.Millisecond) + task := popTask() + if nil == task { + continue + } + + execTask(task) + } +} + +func CloseWait() { + queueLock.Lock() + defer queueLock.Unlock() + + taskQueueStatus = QueueStatusClosing + for { + time.Sleep(10 * time.Millisecond) + if 1 > len(taskQueue) { + break + } + } +} + +func popTask() (ret *Task) { + queueLock.Lock() + defer queueLock.Unlock() + + if 0 == len(taskQueue) { + return + } + + ret = taskQueue[0] + taskQueue = taskQueue[1:] + return +} + +func execTask(task *Task) { + taskLock.Lock() + defer taskLock.Unlock() + defer logging.Recover() + + args := make([]reflect.Value, len(task.Args)) + for i, v := range task.Args { + if nil == v { + args[i] = reflect.New(task.Handler.Type().In(i)).Elem() + } else { + args[i] = reflect.ValueOf(v) + } + } + task.Handler.Call(args) +} From c40355e47ffb308d64f93aa1f68b4efdf624fa19 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:03:07 +0800 Subject: [PATCH 3/7] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/conf.go | 2 ++ kernel/model/repository.go | 25 +++++++++--------- kernel/model/transaction.go | 52 ++----------------------------------- 3 files changed, 16 insertions(+), 63 deletions(-) diff --git a/kernel/model/conf.go b/kernel/model/conf.go index f96e50907..fa8ea985c 100644 --- a/kernel/model/conf.go +++ b/kernel/model/conf.go @@ -37,6 +37,7 @@ import ( "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" "golang.org/x/text/language" @@ -425,6 +426,7 @@ func Close(force bool, execInstallPkg int) (exitCode int) { } } + task.CloseWait() Conf.Close() sql.CloseDatabase() treenode.SaveBlockTree(false) diff --git a/kernel/model/repository.go b/kernel/model/repository.go index 767fb2489..c65660a62 100644 --- a/kernel/model/repository.go +++ b/kernel/model/repository.go @@ -49,6 +49,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" "github.com/studio-b12/gowebdav" @@ -501,16 +502,21 @@ func InitRepoKey() (err error) { return } -var isCheckoutRepo bool +func CheckoutRepo(id string) { + task.PrependTask(task.RepoCheckout, checkoutRepo, id) +} -func CheckoutRepo(id string) (err error) { +func checkoutRepo(id string) { + var err error if 1 > len(Conf.Repo.Key) { - err = errors.New(Conf.Language(26)) + util.PushErrMsg(Conf.Language(26), 7000) return } repo, err := newRepository() if nil != err { + logging.LogErrorf("new repository failed: %s", err) + util.PushErrMsg(Conf.Language(141), 7000) return } @@ -525,23 +531,16 @@ func CheckoutRepo(id string) (err error) { Conf.Sync.Enabled = false Conf.Save() - if util.IsMutexLocked(&syncLock) { - err = errors.New("sync is running, please try again later") - return - } - - isCheckoutRepo = true - defer func() { - isCheckoutRepo = false - }() - _, _, err = repo.Checkout(id, map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress}) if nil != err { + logging.LogErrorf("checkout repository failed: %s", err) util.PushClearProgress() + util.PushErrMsg(Conf.Language(141), 7000) return } FullReindex() + if syncEnabled { func() { time.Sleep(5 * time.Second) diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go index 17098e9a0..cda7666c9 100644 --- a/kernel/model/transaction.go +++ b/kernel/model/transaction.go @@ -20,7 +20,6 @@ import ( "bytes" "errors" "fmt" - "io" "os" "path" "path/filepath" @@ -38,6 +37,7 @@ import ( "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -106,12 +106,7 @@ func AutoFlushTx() { } } -var txLock = sync.Mutex{} - func flushTx() { - txLock.Lock() - defer txLock.Unlock() - defer logging.Recover() currentTx = mergeTx() @@ -1224,7 +1219,7 @@ func updateRefText(refNode *ast.Node, changedDefNodes map[string]*ast.Node) (cha // AutoFixIndex 自动校验数据库索引 https://github.com/siyuan-note/siyuan/issues/7016 func AutoFixIndex() { for { - autoFixIndex() + task.AppendTask(task.DatabaseIndexFix, autoFixIndex) time.Sleep(10 * time.Minute) } } @@ -1234,21 +1229,6 @@ var autoFixLock = sync.Mutex{} func autoFixIndex() { defer logging.Recover() - if isFullReindexing { - logging.LogInfof("skip check index caused by full reindexing") - return - } - - if util.IsMutexLocked(&syncLock) { - logging.LogInfof("skip check index caused by sync lock") - return - } - - if isCheckoutRepo { - logging.LogInfof("skip check index caused by checkout repo") - return - } - if util.IsMutexLocked(&autoFixLock) { return } @@ -1262,10 +1242,6 @@ func autoFixIndex() { boxPath := filepath.Join(util.DataDir, box.ID) var paths []string filepath.Walk(boxPath, func(path string, info os.FileInfo, err error) error { - if isFullReindexing { - return io.EOF - } - if !info.IsDir() && filepath.Ext(path) == ".sy" { p := path[len(boxPath):] p = filepath.ToSlash(p) @@ -1278,19 +1254,11 @@ func autoFixIndex() { redundantPaths := treenode.GetRedundantPaths(box.ID, paths) for _, p := range redundantPaths { - if isFullReindexing { - break - } - treenode.RemoveBlockTreesByPath(p) } missingPaths := treenode.GetNotExistPaths(box.ID, paths) for i, p := range missingPaths { - if isFullReindexing { - break - } - id := path.Base(p) id = strings.TrimSuffix(id, ".sy") if !ast.IsNodeIDPattern(id) { @@ -1308,10 +1276,6 @@ func autoFixIndex() { i := -1 size := len(rootUpdatedMap) for rootID, updated := range rootUpdatedMap { - if isFullReindexing { - break - } - i++ rootUpdated := dbRootUpdatedMap[rootID] @@ -1341,10 +1305,6 @@ func autoFixIndex() { duplicatedRootIDs := sql.GetDuplicatedRootIDs() size := len(duplicatedRootIDs) for i, rootID := range duplicatedRootIDs { - if isFullReindexing { - break - } - root := sql.GetBlock(rootID) if nil == root { continue @@ -1359,10 +1319,6 @@ func autoFixIndex() { } func reindexTreeByPath(box, p string, i, size int) { - if isFullReindexing { - return - } - tree, err := LoadTree(box, p) if nil != err { return @@ -1372,10 +1328,6 @@ func reindexTreeByPath(box, p string, i, size int) { } func reindexTree(rootID string, i, size int) { - if isFullReindexing { - return - } - root := treenode.GetBlockTree(rootID) if nil == root { logging.LogWarnf("root block not found", rootID) From 2d94fd2b76050f720ed626897df5069a206d3d36 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:05:13 +0800 Subject: [PATCH 4/7] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/main.go | 2 +- kernel/mobile/kernel.go | 2 +- kernel/model/ocr.go | 5 +++-- kernel/sql/queue.go | 7 ++++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/kernel/main.go b/kernel/main.go index 426622bd8..caf2dd1c7 100644 --- a/kernel/main.go +++ b/kernel/main.go @@ -52,7 +52,7 @@ func main() { util.PushClearAllMsg() go model.AutoRefreshCheck() go model.AutoFlushTx() - go sql.AutoFlushQueue() + go sql.AutoFlushTx() go treenode.AutoFlushBlockTree() go cache.LoadAssets() go model.AutoFixIndex() diff --git a/kernel/mobile/kernel.go b/kernel/mobile/kernel.go index 67b339a1d..c043809dc 100644 --- a/kernel/mobile/kernel.go +++ b/kernel/mobile/kernel.go @@ -66,7 +66,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang util.PushClearAllMsg() go model.AutoRefreshCheck() go model.AutoFlushTx() - go sql.AutoFlushQueue() + go sql.AutoFlushTx() go treenode.AutoFlushBlockTree() go cache.LoadAssets() go model.AutoFixIndex() diff --git a/kernel/model/ocr.go b/kernel/model/ocr.go index 19cc77dae..740145e4d 100644 --- a/kernel/model/ocr.go +++ b/kernel/model/ocr.go @@ -1,7 +1,6 @@ package model import ( - "github.com/dustin/go-humanize" "io" "os" "path/filepath" @@ -12,9 +11,11 @@ import ( "time" "github.com/88250/gulu" + "github.com/dustin/go-humanize" "github.com/panjf2000/ants/v2" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/cache" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -24,7 +25,7 @@ func AutoOCRAssets() { } for { - autoOCRAssets() + task.AppendTask(task.OCRImage, autoOCRAssets) time.Sleep(7 * time.Second) } } diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go index c9c71cfa2..2b076f97c 100644 --- a/kernel/sql/queue.go +++ b/kernel/sql/queue.go @@ -29,6 +29,7 @@ import ( "github.com/emirpasic/gods/sets/hashset" "github.com/siyuan-note/eventbus" "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -50,10 +51,10 @@ type treeQueueOperation struct { renameTreeOldHPath string // rename } -func AutoFlushTreeQueue() { +func AutoFlushTx() { for { - flushTreeQueue() time.Sleep(util.SQLFlushInterval) + task.PrependTask(task.DatabaseIndex, FlushQueue) } } @@ -91,7 +92,7 @@ func ClearQueue() { operationQueue = nil } -func flushTreeQueue() { +func FlushQueue() { ops := mergeUpsertTrees() if 1 > len(ops) { return From 0da0fb671269dfcfd6b9b819aa6c3bf6afab6100 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:15:00 +0800 Subject: [PATCH 5/7] =?UTF-8?q?:art:=20=E6=94=B9=E8=BF=9B=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E4=BB=BB=E5=8A=A1=E8=B0=83=E5=BA=A6=E6=9C=BA=E5=88=B6?= =?UTF-8?q?=E6=8F=90=E5=8D=87=E7=A8=B3=E5=AE=9A=E6=80=A7=20https://github.?= =?UTF-8?q?com/siyuan-note/siyuan/issues/7113?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/history.go | 3 ++- kernel/task/queue.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/model/history.go b/kernel/model/history.go index 7372df72e..44cfe23cd 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -38,6 +38,7 @@ import ( "github.com/siyuan-note/siyuan/kernel/conf" "github.com/siyuan-note/siyuan/kernel/search" "github.com/siyuan-note/siyuan/kernel/sql" + "github.com/siyuan-note/siyuan/kernel/task" "github.com/siyuan-note/siyuan/kernel/treenode" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -48,7 +49,7 @@ func AutoGenerateDocHistory() { ChangeHistoryTick(Conf.Editor.GenerateHistoryInterval) for { <-historyTicker.C - generateDocHistory() + task.PrependTask(task.HistoryGenerateDoc, generateDocHistory) } } diff --git a/kernel/task/queue.go b/kernel/task/queue.go index e5649ebd3..37bd4d092 100644 --- a/kernel/task/queue.go +++ b/kernel/task/queue.go @@ -104,7 +104,8 @@ const ( DatabaseIndex = "task.database.index" // 数据库所以队列 DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正 OCRImage = "task.ocr.image" // 图片 OCR 提取文本 - DatabaseIndexEmbedBlock = "task.database.index.embedblock" // 数据库索引嵌入块 + HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史 + DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块 ) func Loop() { From 2476776c83fe945fb0d128eb5267344e0862e2d0 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:42:26 +0800 Subject: [PATCH 6/7] =?UTF-8?q?:art:=20=E6=AF=8F=E6=AC=A1=20OCR=20?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=9C=80=E5=A4=9A=E5=A4=84=E7=90=86=2064=20?= =?UTF-8?q?=E5=BC=A0=E5=9B=BE=E7=89=87=E4=BB=A5=E9=98=B2=E6=AD=A2=E5=8D=A1?= =?UTF-8?q?=E9=A1=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/model/ocr.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/model/ocr.go b/kernel/model/ocr.go index 740145e4d..96c493c8b 100644 --- a/kernel/model/ocr.go +++ b/kernel/model/ocr.go @@ -53,17 +53,21 @@ func autoOCRAssets() { util.AssetsTextsLock.Unlock() util.AssetsTextsChanged = true }) - for _, assetAbsPath := range assets { + for i, assetAbsPath := range assets { waitGroup.Add(1) p.Invoke(assetAbsPath) + + if 63 <= i { // 一次最多处理 64 张图片，防止卡顿 + break + } } waitGroup.Wait() p.Release() - cleanNotFoundAssetsTexts() + cleanNotExistAssetsTexts() } -func cleanNotFoundAssetsTexts() { +func cleanNotExistAssetsTexts() { tmp := util.AssetsTexts assetsPath := util.GetDataAssetsAbsPath() From e706f26e6815b93938f668753c1d5597e4efddef Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Wed, 18 Jan 2023 23:49:42 +0800 Subject: [PATCH 7/7] =?UTF-8?q?:bug:=20=E5=9B=BE=E7=89=87=20OCR=20?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E7=BB=93=E6=9E=9C=E5=90=8E=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E5=A4=9A=E4=BD=99=E7=9A=84=20`span`=20=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2=20Fix=20https://github.com/siyuan-note/siyuan/issues/?= =?UTF-8?q?7119?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/treenode/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/treenode/node.go b/kernel/treenode/node.go index be06e58a9..52e416ab7 100644 --- a/kernel/treenode/node.go +++ b/kernel/treenode/node.go @@ -186,7 +186,7 @@ func NodeStaticContent(node *ast.Node, excludeTypes []string) string { buf.WriteByte(' ') } if nil != linkDest { - buf.Write(n.Tokens) + buf.Write(linkDest.Tokens) buf.WriteByte(' ') }