🎨 Improve kernel stability by eliminating some data races https://github.com/siyuan-note/siyuan/issues/9842

kernel/model/conf.go

@@ -492,7 +492,7 @@ var exitLock = sync.Mutex{}
 func Close(force bool, execInstallPkg int) (exitCode int) {
 	exitLock.Lock()
 	defer exitLock.Unlock()
-	util.IsExiting = true
+	util.IsExiting.Store(true)
 
 	logging.LogInfof("exiting kernel [force=%v, execInstallPkg=%d]", force, execInstallPkg)
 	util.PushMsg(Conf.Language(95), 10000*60)

kernel/model/import.go

@@ -84,8 +84,8 @@ func ImportSY(zipPath, boxID, toPath string) (err error) {
 	util.PushEndlessProgress(Conf.Language(73))
 	defer util.ClearPushProgress(100)
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	baseName := filepath.Base(zipPath)
 	ext := filepath.Ext(baseName)
@@ -550,8 +550,8 @@ func ImportData(zipPath string) (err error) {
 	util.PushEndlessProgress(Conf.Language(73))
 	defer util.ClearPushProgress(100)
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	baseName := filepath.Base(zipPath)
 	ext := filepath.Ext(baseName)
@@ -605,6 +605,9 @@ func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
 		}
 	}()
 
+	lockSync()
+	defer unlockSync()
+
 	WaitForWritingFiles()
 
 	var baseHPath, baseTargetPath, boxLocalPath string

kernel/model/index.go

@@ -24,6 +24,7 @@ import (
 	"runtime/debug"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/88250/gulu"
@@ -70,13 +71,12 @@ func index(boxID string) {
 	if 1 > boxLen {
 		boxLen = 1
 	}
-	bootProgressPart := 30.0 / float64(boxLen) / float64(len(files))
+	bootProgressPart := int32(30.0 / float64(boxLen) / float64(len(files)))
 
 	start := time.Now()
 	luteEngine := util.NewLute()
-	var treeCount int
-	var treeSize int64
-	i := 0
+	treeCount := atomic.Int32{}
+	treeSize := atomic.Int64{}
 	util.PushStatusBar(fmt.Sprintf("["+html.EscapeString(box.Name)+"] "+Conf.Language(64), len(files)))
 
 	poolSize := runtime.NumCPU()
@@ -109,12 +109,12 @@ func index(boxID string) {
 		sql.IndexTreeQueue(box.ID, file.path)
 
 		util.IncBootProgress(bootProgressPart, fmt.Sprintf(Conf.Language(92), util.ShortPathForBootingDisplay(tree.Path)))
-		treeSize += file.size
-		treeCount++
+		treeSize.Add(file.size)
+		treeCount.Add(1)
+		i := treeCount.Load()
 		if 1 < i && 0 == i%64 {
-			util.PushStatusBar(fmt.Sprintf(Conf.Language(88), i, len(files)-i))
+			util.PushStatusBar(fmt.Sprintf(Conf.Language(88), i, int32(len(files))-i))
 		}
-		i++
 	})
 	for _, file := range files {
 		if file.isdir || !strings.HasSuffix(file.name, ".sy") {
@@ -130,7 +130,7 @@ func index(boxID string) {
 	box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
 	end := time.Now()
 	elapsed := end.Sub(start).Seconds()
-	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount, humanize.Bytes(uint64(treeSize)))
+	logging.LogInfof("rebuilt database for notebook [%s] in [%.2fs], tree [count=%d, size=%s]", box.ID, elapsed, treeCount.Load(), humanize.Bytes(uint64(treeSize.Load())))
 	debug.FreeOSMemory()
 	return
 }
@@ -187,7 +187,7 @@ func IndexRefs() {
 	i := 0
 	size := len(defBlockIDs)
 	if 0 < size {
-		bootProgressPart := 10.0 / float64(size)
+		bootProgressPart := int32(10.0 / float64(size))
 
 		for _, defBlockID := range defBlockIDs {
 			defTree, loadErr := LoadTreeByID(defBlockID)

kernel/model/index_fix.go

@@ -116,7 +116,7 @@ func removeDuplicateDatabaseIndex() {
 		}
 		deletes++
 		toRemoveRootIDs = append(toRemoveRootIDs, rootID)
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 	}
@@ -335,12 +335,12 @@ func fixBlockTreeByFileSys() {
 			}
 
 			reindexTreeByPath(box.ID, p, i, size, luteEngine)
-			if util.IsExiting {
+			if util.IsExiting.Load() {
 				break
 			}
 		}
 
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 	}
@@ -371,7 +371,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 	for rootID, updated := range rootUpdatedMap {
 		i++
 
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 
@@ -396,7 +396,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 			continue
 		}
 
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 	}
@@ -407,7 +407,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 			rootIDs = append(rootIDs, rootID)
 		}
 
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 	}
@@ -424,7 +424,7 @@ func reindexTreeByUpdated(rootUpdatedMap, dbRootUpdatedMap map[string]string) {
 		}
 
 		toRemoveRootIDs = append(toRemoveRootIDs, id)
-		if util.IsExiting {
+		if util.IsExiting.Load() {
 			break
 		}
 	}

kernel/model/repository.go

@@ -1294,12 +1294,12 @@ func processSyncMergeResult(exit, byHand bool, mergeResult *dejavu.MergeResult,
 	}
 
 	if 1 > len(mergeResult.Upserts) && 1 > len(mergeResult.Removes) && 1 > len(mergeResult.Conflicts) { // 没有数据变更
-		syncSameCount++
-		if 10 < syncSameCount {
-			syncSameCount = 5
+		syncSameCount.Add(1)
+		if 10 < syncSameCount.Load() {
+			syncSameCount.Store(5)
 		}
 		if !byHand {
-			delay := time.Minute * time.Duration(int(math.Pow(2, float64(syncSameCount))))
+			delay := time.Minute * time.Duration(int(math.Pow(2, float64(syncSameCount.Load()))))
 			if fixSyncInterval.Minutes() > delay.Minutes() {
 				delay = time.Minute * 8
 			}
@@ -1618,11 +1618,11 @@ func subscribeRepoEvents() {
 		}
 		coWalkDataCount++
 	})
-	var bootProgressPart float64
+	var bootProgressPart int32
 	eventbus.Subscribe(eventbus.EvtCheckoutUpsertFiles, func(context map[string]interface{}, total int) {
 		msg := fmt.Sprintf(Conf.Language(162), 0, total)
 		util.SetBootDetails(msg)
-		bootProgressPart = 10 / float64(total)
+		bootProgressPart = int32(10 / float64(total))
 		util.ContextPushMsg(context, msg)
 	})
 	coUpsertFileCount := 0
@@ -1637,7 +1637,7 @@ func subscribeRepoEvents() {
 	eventbus.Subscribe(eventbus.EvtCheckoutRemoveFiles, func(context map[string]interface{}, total int) {
 		msg := fmt.Sprintf(Conf.Language(163), 0, total)
 		util.SetBootDetails(msg)
-		bootProgressPart = 10 / float64(total)
+		bootProgressPart = int32(10 / float64(total))
 		util.ContextPushMsg(context, msg)
 	})
 
@@ -1658,7 +1658,7 @@ func subscribeRepoEvents() {
 	eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadFiles, func(context map[string]interface{}, total int) {
 		msg := fmt.Sprintf(Conf.Language(165), 0, total)
 		util.SetBootDetails(msg)
-		bootProgressPart = 10 / float64(total)
+		bootProgressPart = int32(10 / float64(total))
 		util.ContextPushMsg(context, msg)
 	})
 
@@ -1672,7 +1672,7 @@ func subscribeRepoEvents() {
 	eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadChunks, func(context map[string]interface{}, total int) {
 		msg := fmt.Sprintf(Conf.Language(166), 0, total)
 		util.SetBootDetails(msg)
-		bootProgressPart = 10 / float64(total)
+		bootProgressPart = int32(10 / float64(total))
 		util.ContextPushMsg(context, msg)
 	})
 	eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadChunk, func(context map[string]interface{}, count, total int) {

kernel/model/sync.go

@@ -26,6 +26,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/88250/gulu"
@@ -55,8 +56,8 @@ func SyncDataDownload() {
 		return
 	}
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	now := util.CurrentTimeMillis()
 	Conf.Sync.Synced = now
@@ -82,8 +83,8 @@ func SyncDataUpload() {
 		return
 	}
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	now := util.CurrentTimeMillis()
 	Conf.Sync.Synced = now
@@ -98,9 +99,11 @@ func SyncDataUpload() {
 }
 
 var (
-	syncSameCount    = 0
+	syncSameCount    = atomic.Int32{}
 	autoSyncErrCount = 0
 	fixSyncInterval  = 5 * time.Minute
+
+	syncPlanTimeLock = sync.Mutex{}
 	syncPlanTime     = time.Now().Add(fixSyncInterval)
 
 	BootSyncSucc = -1 // -1：未执行，0：执行成功，1：执行失败
@@ -108,9 +111,12 @@ var (
 )
 
 func SyncDataJob() {
+	syncPlanTimeLock.Lock()
 	if time.Now().Before(syncPlanTime) {
+		syncPlanTimeLock.Unlock()
 		return
 	}
+	syncPlanTimeLock.Unlock()
 
 	SyncData(false)
 }
@@ -132,8 +138,8 @@ func BootSyncData() {
 		return
 	}
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	util.IncBootProgress(3, "Syncing data from the cloud...")
 	BootSyncSucc = 0
@@ -155,6 +161,16 @@ func SyncData(byHand bool) {
 	syncData(false, byHand)
 }
 
+func lockSync() {
+	syncLock.Lock()
+	isSyncing = true
+}
+
+func unlockSync() {
+	isSyncing = false
+	syncLock.Unlock()
+}
+
 func syncData(exit, byHand bool) {
 	defer logging.Recover()
 
@@ -168,8 +184,8 @@ func syncData(exit, byHand bool) {
 		return
 	}
 
-	syncLock.Lock()
-	defer syncLock.Unlock()
+	lockSync()
+	defer unlockSync()
 
 	if exit {
 		ExitSyncSucc = 0
@@ -240,7 +256,7 @@ func checkSync(boot, exit, byHand bool) bool {
 		}
 	}
 
-	if gulu.IsMutexLocked(&syncLock) {
+	if isSyncing {
 		logging.LogWarnf("sync is in progress")
 		planSyncAfter(fixSyncInterval)
 		return false
@@ -266,7 +282,7 @@ func incReindex(upserts, removes []string) (upsertRootIDs, removeRootIDs []strin
 
 	luteEngine := util.NewLute()
 	// 先执行 remove，否则移动文档时 upsert 会被忽略，导致未被索引
-	bootProgressPart := 10 / float64(len(removes))
+	bootProgressPart := int32(10 / float64(len(removes)))
 	for _, removeFile := range removes {
 		if !strings.HasSuffix(removeFile, ".sy") {
 			continue
@@ -288,7 +304,7 @@ func incReindex(upserts, removes []string) (upsertRootIDs, removeRootIDs []strin
 	msg = fmt.Sprintf(Conf.Language(35))
 	util.PushStatusBar(msg)
 
-	bootProgressPart = 10 / float64(len(upserts))
+	bootProgressPart = int32(10 / float64(len(upserts)))
 	for _, upsertFile := range upserts {
 		if !strings.HasSuffix(upsertFile, ".sy") {
 			continue
@@ -413,7 +429,10 @@ func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
 	return
 }
 
-var syncLock = sync.Mutex{}
+var (
+	syncLock  = sync.Mutex{}
+	isSyncing bool
+)
 
 func CreateCloudSyncDir(name string) (err error) {
 	if conf.ProviderSiYuan != Conf.Sync.Provider {
@@ -587,12 +606,14 @@ func getIgnoreLines() (ret []string) {
 }
 
 func IncSync() {
-	syncSameCount = 0
+	syncSameCount.Store(0)
 	planSyncAfter(30 * time.Second)
 }
 
 func planSyncAfter(d time.Duration) {
+	syncPlanTimeLock.Lock()
 	syncPlanTime = time.Now().Add(d)
+	syncPlanTimeLock.Unlock()
 }
 
 func isProviderOnline(byHand bool) (ret bool) {

kernel/model/transaction.go

@@ -87,7 +87,7 @@ var (
 
 func isWritingFiles() bool {
 	time.Sleep(time.Duration(20) * time.Millisecond)
-	return 0 < len(txQueue) || gulu.IsMutexLocked(&flushLock)
+	return 0 < len(txQueue)
 }
 
 func init() {

kernel/model/updater.go

@@ -82,11 +82,9 @@ func checkDownloadInstallPkg() {
 		return
 	}
 
-	if gulu.IsMutexLocked(&checkDownloadInstallPkgLock) {
+	if !checkDownloadInstallPkgLock.TryLock() {
 		return
 	}
-
-	checkDownloadInstallPkgLock.Lock()
 	defer checkDownloadInstallPkgLock.Unlock()
 
 	downloadPkgURLs, checksum, err := getUpdatePkg()