From b886556cbb6b06e2bb81ba92a6dfddda861533d7 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 22 Dec 2023 08:24:37 +0800 Subject: [PATCH 1/5] :art: Improve data sync to prevent an old snapshot to overwrite the new one https://github.com/siyuan-note/siyuan/issues/9949 Abandon snapshot if file does not exist when creating snapshot https://github.com/siyuan-note/siyuan/issues/9948 --- kernel/go.mod | 2 +- kernel/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/go.mod b/kernel/go.mod index 963dbe90e..64ea492b9 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -50,7 +50,7 @@ require ( github.com/rqlite/sql v0.0.0-20221103124402-8f9ff0ceb8f0 github.com/sashabaranov/go-openai v1.17.9 github.com/shirou/gopsutil/v3 v3.23.11 - github.com/siyuan-note/dejavu v0.0.0-20231221144323-86aba7fc891e + github.com/siyuan-note/dejavu v0.0.0-20231222001002-f9ee1c8562bf github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 github.com/siyuan-note/eventbus v0.0.0-20231210025112-82fdedd8e374 github.com/siyuan-note/filelock v0.0.0-20231211015131-3b3dfabdce9c diff --git a/kernel/go.sum b/kernel/go.sum index 1c47feeb6..3ff1b934b 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -354,8 +354,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20231221144323-86aba7fc891e h1:vto8rhxvaM5umJlM7kIikiDJ3y1HsX0EMRf+CsM2BP4= -github.com/siyuan-note/dejavu v0.0.0-20231221144323-86aba7fc891e/go.mod h1:Zpo0p/Z5dSebzbxMiGJKp0X4OeJ4kBrLOJO6a0txDQs= +github.com/siyuan-note/dejavu v0.0.0-20231222001002-f9ee1c8562bf h1:ou1qYDJkVuDGPf7x6rD/7Z5KE1Seaqf884A+POh1WwA= +github.com/siyuan-note/dejavu v0.0.0-20231222001002-f9ee1c8562bf/go.mod h1:Zpo0p/Z5dSebzbxMiGJKp0X4OeJ4kBrLOJO6a0txDQs= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20231210025112-82fdedd8e374 h1:haPIOZBOAEQaXsUPUpyZeOeMgw6g/9O7RBbS3d39Ta8= From 3c52ad491e6a04a707564ffc83b887581e1ddf7a Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 22 Dec 2023 08:25:04 +0800 Subject: [PATCH 2/5] :recycle: Upgrade Electron v27.2.0 https://github.com/siyuan-note/siyuan/issues/9953 --- .github/CONTRIBUTING.md | 6 +++--- .github/CONTRIBUTING_zh_CN.md | 6 +++--- app/package.json | 2 +- app/pnpm-lock.yaml | 14 +++++++------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 71be8319b..b0fd7f512 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -16,10 +16,10 @@ Install pnpm: `npm install -g pnpm` Set the Electron mirror environment variable and install Electron: -* macOS/Linux: `ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/ pnpm install electron@27.1.3 -D` +* macOS/Linux: `ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/ pnpm install electron@27.2.0 -D` * Windows: * `SET ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/` - * `pnpm install electron@27.1.3 -D` + * `pnpm install electron@27.2.0 -D` NPM mirror: @@ -30,7 +30,7 @@ NPM mirror: On the desktop, go to the app folder to run: -* `pnpm install electron@27.1.3 -D` +* `pnpm install electron@27.2.0 -D` * `pnpm run dev` * `pnpm run start` diff --git a/.github/CONTRIBUTING_zh_CN.md b/.github/CONTRIBUTING_zh_CN.md index 55ff5d524..eeba2f832 100644 --- a/.github/CONTRIBUTING_zh_CN.md +++ b/.github/CONTRIBUTING_zh_CN.md @@ -18,11 +18,11 @@ * macOS/Linux: ``` - ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/ pnpm install electron@27.1.3 -D + ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/ pnpm install electron@27.2.0 -D ``` * Windows: * `SET ELECTRON_MIRROR=https://cnpmjs.org/mirrors/electron/` - * `pnpm install electron@27.1.3 -D` + * `pnpm install electron@27.2.0 -D` NPM 镜像： @@ -32,7 +32,7 @@ NPM 镜像： 桌面端进入 app 文件夹运行： -* `pnpm install electron@27.1.3 -D` +* `pnpm install electron@27.2.0 -D` * `pnpm run dev` * `pnpm run start` diff --git a/app/package.json b/app/package.json index 3d3837ea5..ef0b8c753 100644 --- a/app/package.json +++ b/app/package.json @@ -53,7 +53,7 @@ "clean-webpack-plugin": "^4.0.0", "css-loader": "^6.7.1", "dayjs": "^1.11.5", - "electron": "27.1.3", + "electron": "27.2.0", "electron-builder": "^24.9.1", "encoding": "^0.1.13", "esbuild-loader": "^3.0.1", diff --git a/app/pnpm-lock.yaml b/app/pnpm-lock.yaml index fe95442ad..059ae97bc 100644 --- a/app/pnpm-lock.yaml +++ b/app/pnpm-lock.yaml @@ -7,7 +7,7 @@ settings: dependencies: '@electron/remote': specifier: ^2.0.12 - version: 2.0.12(electron@27.1.3) + version: 2.0.12(electron@27.2.0) devDependencies: '@types/node': @@ -32,8 +32,8 @@ devDependencies: specifier: ^1.11.5 version: 1.11.5 electron: - specifier: 27.1.3 - version: 27.1.3 + specifier: 27.2.0 + version: 27.2.0 electron-builder: specifier: ^24.9.1 version: 24.9.1 @@ -161,12 +161,12 @@ packages: - supports-color dev: true - /@electron/remote@2.0.12(electron@27.1.3): + /@electron/remote@2.0.12(electron@27.2.0): resolution: {integrity: sha512-IJN6xLAxptq5MCvXNCU6+pdQyz0DjpPtX6g2TPJftu3Z9pU6BTdnos9ZMN8nK471LkASqiA6C+Hzjv5SS8PAQw==} peerDependencies: electron: '>= 13.0.0' dependencies: - electron: 27.1.3 + electron: 27.2.0 dev: false /@electron/universal@1.4.1: @@ -1670,8 +1670,8 @@ packages: resolution: {integrity: sha512-YoVeFrGd/7ROjz4R9uPoND1K/hSRC/xADy9639ZmIZeJSaBnKdYx3I6LMPsY7CXLpK7JFgKQVzeZ/dk2br6Eaw==} dev: true - /electron@27.1.3: - resolution: {integrity: sha512-7eD8VMhhlL5J531OOawn00eMthUkX1e3qN5Nqd7eMK8bg5HxQBrn8bdPlvUEnCano9KhrVwaDnGeuzWoDOGpjQ==} + /electron@27.2.0: + resolution: {integrity: sha512-no/iMICVLI/5G0IqgKFbB89HDN88DWwKeRO+dPfJPkpJISdEX8Cx/sMEOFuuRa4VNInNe5CKCqRWExK5z3AdcQ==} engines: {node: '>= 12.20.55'} hasBin: true requiresBuild: true From b5494e9af70a504bedd9d9d45af35092d2102260 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 22 Dec 2023 10:00:40 +0800 Subject: [PATCH 3/5] :art: Concurrency control when requesting the kernel API https://github.com/siyuan-note/siyuan/issues/9939 --- kernel/model/session.go | 20 ++++++++++++++++++++ kernel/server/serve.go | 1 + 2 files changed, 21 insertions(+) diff --git a/kernel/model/session.go b/kernel/model/session.go index e14a4452a..75ca732d6 100644 --- a/kernel/model/session.go +++ b/kernel/model/session.go @@ -23,6 +23,7 @@ import ( "os" "strconv" "strings" + "sync" "time" "github.com/88250/gulu" @@ -317,3 +318,22 @@ func Recover(c *gin.Context) { c.Next() } + +var ( + requestingLock = sync.Mutex{} + requesting = map[string]*sync.Mutex{} +) + +func ControlConcurrency(c *gin.Context) { + requestingLock.Lock() + mutex := requesting[c.Request.URL.Path] + if nil == mutex { + mutex = &sync.Mutex{} + requesting[c.Request.URL.Path] = mutex + } + requestingLock.Unlock() + + mutex.Lock() + c.Next() + mutex.Unlock() +} diff --git a/kernel/server/serve.go b/kernel/server/serve.go index 41c7ab4fd..3a3698201 100644 --- a/kernel/server/serve.go +++ b/kernel/server/serve.go @@ -52,6 +52,7 @@ func Serve(fastMode bool) { ginServer := gin.New() ginServer.MaxMultipartMemory = 1024 * 1024 * 32 // 插入较大的资源文件时内存占用较大 https://github.com/siyuan-note/siyuan/issues/5023 ginServer.Use( + model.ControlConcurrency, // 请求串行化 Concurrency control when requesting the kernel API https://github.com/siyuan-note/siyuan/issues/9939 model.Timing, model.Recover, corsMiddleware(), // 后端服务支持 CORS 预检请求验证 https://github.com/siyuan-note/siyuan/pull/5593 From c32cad7c3e61765a0b4b787505bcdde6af7ec1dd Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 22 Dec 2023 10:14:59 +0800 Subject: [PATCH 4/5] :art: Distinguish between new cards and review cards during spaced repetition https://github.com/siyuan-note/siyuan/issues/9377 --- kernel/model/flashcard.go | 52 +++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/kernel/model/flashcard.go b/kernel/model/flashcard.go index 0b660855e..c8f60bc3d 100644 --- a/kernel/model/flashcard.go +++ b/kernel/model/flashcard.go @@ -626,8 +626,8 @@ func getAllDueFlashcards(reviewedCardIDs []string) (ret []*Flashcard, unreviewed for _, deck := range Decks { cards, unreviewedCnt, unreviewedNewCardCnt, unreviewedOldCardCnt := getDeckDueCards(deck, reviewedCardIDs, nil, Conf.Flashcard.NewCardLimit, Conf.Flashcard.ReviewCardLimit) unreviewedCount += unreviewedCnt - unreviewedNewCardCnt += unreviewedNewCardCnt - unreviewedOldCardCnt += unreviewedOldCardCnt + unreviewedNewCardCount += unreviewedNewCardCnt + unreviewedOldCardCount += unreviewedOldCardCnt for _, card := range cards { ret = append(ret, newFlashcard(card, card.BlockID(), deck.ID, now)) } @@ -994,38 +994,16 @@ func getDeckDueCards(deck *riff.Deck, reviewedCardIDs, blockIDs []string, newCar dues := deck.Dues() var tmp []riff.Card - for _, c := range dues { - if nil == treenode.GetBlockTree(c.BlockID()) { - continue - } - tmp = append(tmp, c) - } - dues = tmp - tmp = nil - for _, c := range dues { if 0 < len(blockIDs) && !gulu.Str.Contains(c.BlockID(), blockIDs) { continue } - tmp = append(tmp, c) - if 0 < len(reviewedCardIDs) { - if !gulu.Str.Contains(c.ID(), reviewedCardIDs) { - unreviewedCount++ - if riff.New == c.GetState() { - unreviewedNewCardCount++ - } else { - unreviewedOldCardCount++ - } - } - } else { - unreviewedCount++ - if riff.New == c.GetState() { - unreviewedNewCardCount++ - } else { - unreviewedOldCardCount++ - } + if nil == treenode.GetBlockTree(c.BlockID()) { + continue } + + tmp = append(tmp, c) } dues = tmp @@ -1055,8 +1033,22 @@ func getDeckDueCards(deck *riff.Deck, reviewedCardIDs, blockIDs []string, newCar } } - if 0 < len(reviewedCardIDs) && !gulu.Str.Contains(c.ID(), reviewedCardIDs) { - continue + if 0 < len(reviewedCardIDs) { + if !gulu.Str.Contains(c.ID(), reviewedCardIDs) { + unreviewedCount++ + if riff.New == c.GetState() { + unreviewedNewCardCount++ + } else { + unreviewedOldCardCount++ + } + } + } else { + unreviewedCount++ + if riff.New == c.GetState() { + unreviewedNewCardCount++ + } else { + unreviewedOldCardCount++ + } } ret = append(ret, c) From a190066523f82a776b8fefb76182d07f813c6f11 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 22 Dec 2023 10:21:15 +0800 Subject: [PATCH 5/5] :art: Concurrency control when requesting the kernel API https://github.com/siyuan-note/siyuan/issues/9939 --- kernel/model/session.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/model/session.go b/kernel/model/session.go index 75ca732d6..d25950deb 100644 --- a/kernel/model/session.go +++ b/kernel/model/session.go @@ -28,6 +28,7 @@ import ( "github.com/88250/gulu" "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" "github.com/siyuan-note/logging" "github.com/siyuan-note/siyuan/kernel/util" "github.com/steambap/captcha" @@ -325,6 +326,11 @@ var ( ) func ControlConcurrency(c *gin.Context) { + if websocket.IsWebSocketUpgrade(c.Request) { + c.Next() + return + } + requestingLock.Lock() mutex := requesting[c.Request.URL.Path] if nil == mutex {