Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2026-01-25 01:36:09 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/dev' into dev

Browse source
This commit is contained in:
Vanessa 2023-02-17 15:34:54 +08:00
commit 9e948196ed
26 changed files with 298 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app/appearance/langs/en_US.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Execute database index fix",
"task.ocr.image": "Execute image OCR to extract text",
"task.history.generateDoc": "Execute GenerateDoc History",
"task.history.database.index.commit": "Execute history database index commit",
"task.database.index.embedBlock": "Execute database index embed block",
"task.reload.ui": "Execute reload UI"
},
@ -1089,6 +1090,7 @@
"187": "Unlocking cloud sync directory",
"188": "Failed to lock the cloud sync directory, please try again later",
"189": "Cloud sync directory is still locked by other devices, please try again later",
"190": "A problem was found while validating the index, which has been automatically fixed"
"190": "A problem was found while validating the index, which has been automatically fixed",
"191": "[%d/%d] Created historical data index"
}
}

4
app/appearance/langs/es_ES.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Ejecutar corrección del índice de la base de datos",
"task.ocr.image": "Ejecutar OCR de imagen para extraer texto",
"task.history.generateDoc": "Ejecutar Historial GenerateDoc",
"task.history.database.index.commit": "Ejecutar la confirmación del índice de la base de datos del historial",
"task.database.index.embedBlock": "Ejecutar bloque de incrustación de índice de base de datos",
"task.reload.ui": "IU de recarga de tareas"
},
@ -1089,6 +1090,7 @@
"187": "Desbloqueando el directorio de sincronización en la nube",
"188": "Error al bloquear el directorio de sincronizaci\u00f3n en la nube, int\u00e1ntelo de nuevo m\u00e1s tarde",
"189": "El directorio de sincronización en la nube todavía está bloqueado por otros dispositivos, inténtalo de nuevo más tarde",
"190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente"
"190": "Se encontro un problema al validar el indice, el cual se soluciono automaticamente",
"191": "[%d/%d] Índice de datos históricos creado"
}
}

4
app/appearance/langs/fr_FR.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "Effectuer la correction de l'index de la base de données",
"task.ocr.image": "Exécute l'OCR d'image pour extraire le texte",
"task.history.generateDoc": "Exécuter l'historique de GenerateDoc",
"task.history.database.index.commit": "Effectuer la validation de l'index de la base de données d'historique",
"task.database.index.embedBlock": "Exécuter le bloc d'intégration d'index de base de données",
"task.reload.ui": "Interface utilisateur de rechargement de tâche"
},
@ -1089,6 +1090,7 @@
"187": "Déverrouillage du répertoire de synchronisation cloud",
"188": "Échec du verrouillage du répertoire de synchronisation cloud, veuillez réessayer plus tard",
"189": "Le répertoire de synchronisation cloud est toujours verrouillé par d'autres appareils, veuillez réessayer plus tard",
"190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé"
"190": "Un problème a été trouvé lors de la validation de l'index, qui a été automatiquement corrigé",
"191": "[%d/%d] Création d'un index de données historiques"
}
}

4
app/appearance/langs/zh_CHT.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "執行數據庫索引訂正",
"task.ocr.image": "執行圖片 OCR 提取文本",
"task.history.generateDoc": "執行生成文件歷史",
"task.history.database.index.commit": "執行歷史數據庫索引提交",
"task.database.index.embedBlock": "執行數據庫索引嵌入塊",
"task.reload.ui": "執行重載界面"
},
@ -1089,6 +1090,7 @@
"187": "正在解鎖雲端同步目錄",
"188": "鎖定雲端同步目錄失敗，請稍後再試",
"189": "雲端同步目錄還在被其他設備鎖定，請稍後再試",
"190": "校驗索引時發現一個問題，已經自動修復"
"190": "校驗索引時發現一個問題，已經自動修復",
"191": "[%d/%d] 已经建立条历史数据索引"
}
}

4
app/appearance/langs/zh_CN.json
View file

@ -885,6 +885,7 @@
"task.database.index.fix": "执行数据库索引订正",
"task.ocr.image": "执行图片 OCR 提取文本",
"task.history.generateDoc": "执行生成文件历史",
"task.history.database.index.commit": "执行历史数据库索引提交",
"task.database.index.embedBlock": "执行数据库索引嵌入块",
"task.reload.ui": "执行重载界面"
},
@ -1089,6 +1090,7 @@
"187": "正在解锁云端同步目录",
"188": "锁定云端同步目录失败，请稍后再试",
"189": "云端同步目录还在被其他设备锁定，请稍后再试",
"190": "校验索引时发现一个问题，已经自动修复"
"190": "校验索引时发现一个问题，已经自动修复",
"191": "[%d/%d] 已经建立条历史数据索引"
}
}

2
app/src/protyle/breadcrumb/index.ts
View file

@ -286,7 +286,7 @@ export class Breadcrumb {
}
}
}).element);
if (!needSubscribe("")) {
if (window.siyuan.user) { // 登录链滴账号后即可使用 `分享到链滴` https://github.com/siyuan-note/siyuan/issues/7392
window.siyuan.menus.menu.append(new MenuItem({
label: window.siyuan.languages.share2Liandi,
icon: "iconLiandi",

23
kernel/api/setting.go
View file

@ -91,6 +91,10 @@ func setEditor(c *gin.Context) {
editor.KaTexMacros = "{}"
}
oldVirtualBlockRef := model.Conf.Editor.VirtualBlockRef
oldVirtualBlockRefInclude := model.Conf.Editor.VirtualBlockRefInclude
oldVirtualBlockRefExclude := model.Conf.Editor.VirtualBlockRefExclude
model.Conf.Editor = editor
model.Conf.Save()
@ -98,6 +102,12 @@ func setEditor(c *gin.Context) {
model.ChangeHistoryTick(editor.GenerateHistoryInterval)
}
if oldVirtualBlockRef != model.Conf.Editor.VirtualBlockRef ||
oldVirtualBlockRefInclude != model.Conf.Editor.VirtualBlockRefInclude ||
oldVirtualBlockRefExclude != model.Conf.Editor.VirtualBlockRefExclude {
model.ResetVirtualBlockRefCache()
}
ret.Data = model.Conf.Editor
}
@ -214,6 +224,11 @@ func setSearch(c *gin.Context) {
}
oldCaseSensitive := model.Conf.Search.CaseSensitive
oldVirtualRefName := model.Conf.Search.VirtualRefName
oldVirtualRefAlias := model.Conf.Search.VirtualRefAlias
oldVirtualRefAnchor := model.Conf.Search.VirtualRefAnchor
oldVirtualRefDoc := model.Conf.Search.VirtualRefDoc
oldVirtualRefKeywordsLimit := model.Conf.Search.VirtualRefKeywordsLimit
model.Conf.Search = s
model.Conf.Save()
@ -221,6 +236,14 @@ func setSearch(c *gin.Context) {
if s.CaseSensitive != oldCaseSensitive {
model.FullReindex()
}
if oldVirtualRefName != s.VirtualRefName ||
oldVirtualRefAlias != s.VirtualRefAlias ||
oldVirtualRefAnchor != s.VirtualRefAnchor ||
oldVirtualRefDoc != s.VirtualRefDoc ||
oldVirtualRefKeywordsLimit != s.VirtualRefKeywordsLimit {
model.ResetVirtualBlockRefCache()
}
ret.Data = s
}

7
kernel/go.mod
View file

@ -42,7 +42,7 @@ require (
github.com/shirou/gopsutil/v3 v3.23.1
github.com/siyuan-note/dejavu v0.0.0-20230212031819-32964d704bd2
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e
github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea
github.com/siyuan-note/logging v0.0.0-20221031125421-9b7234d79d8a
@ -58,13 +58,18 @@ require (
require (
dmitri.shuralyov.com/font/woff2 v0.0.0-20180220214647-957792cbbdab // indirect
github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 // indirect
github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6 // indirect
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect
github.com/aws/aws-sdk-go v1.44.199 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 // indirect
github.com/dlclark/regexp2 v1.8.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect

14
kernel/go.sum
View file

@ -16,7 +16,11 @@ github.com/88250/pdfcpu v0.3.13 h1:touMWMZkCGalMIbEg9bxYp7rETM+zwb9hXjwhqi4I7Q=
github.com/88250/pdfcpu v0.3.13/go.mod h1:S5YT38L/GCjVjmB4PB84PymA1qfopjEhfhTNQilLpv4=
github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1 h1:48T899JQDwyyRu9yXHePYlPdHtpJfrJEUGBMH3SMBWY=
github.com/88250/vitess-sqlparser v0.0.0-20210205111146-56a2ded2aba1/go.mod h1:U3pckKQIgxxkmZjV5yXQjHdGxQK0o/vEZeZ6cQsxfHw=
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3 h1:JsUUbdK1JdhsJt2ebcQFFfeO3jpzh6Vv7pmCph6qeik=
github.com/ClarkThan/ahocorasick v0.0.0-20230216061320-bccdb98581a3/go.mod h1:a3CzWIqeRxiODAscAIfZ4wbFRXxywBrdCwTENVAWB2g=
github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732 h1:0EDePskeF4vNFCk70ATaFHQzjmwXsk+VImnMJttecNU=
github.com/ConradIrwin/font v0.0.0-20210318200717-ce8d41cc0732/go.mod h1:krTLO7JWu6g8RMxG8sl+T1Hf8W93XQacBKJmqFZ2MFY=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@ -33,6 +37,10 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0 h1:onfun1RA+KcxaMk1lfrRnwCd1UUuOjJM/lri5eM1qMs=
github.com/anknown/ahocorasick v0.0.0-20190904063843-d75dbd5169c0/go.mod h1:4yg+jNTYlDEzBjhGS96v+zjyA3lfXlFd5CiTLIkPBLI=
github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6 h1:HblK3eJHq54yET63qPCTJnks3loDse5xRmmqHgHzwoI=
github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6/go.mod h1:pbiaLIeYLUbgMY1kwEAdwO6UKD5ZNwdPGQlwokS9fe8=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM=
@ -42,6 +50,8 @@ github.com/aws/aws-sdk-go v1.44.199/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k=
github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0=
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@ -272,6 +282,10 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03 h1:W7nGGluE6sBrFSVkmucGsh2NruleOPsQle7fcAW115o=
github.com/siyuan-note/eventbus v0.0.0-20230203085647-fb624740be03/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12 h1:DjZa4jP3J+cZK9BuCXXzY4kr37QXXaCX+IHt8JP8UXQ=
github.com/siyuan-note/eventbus v0.0.0-20230216101534-15f8e2f2fb12/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b h1:828lTUW2C0uNiolODqoACu7J8sDUzswD4Xo04mUombg=
github.com/siyuan-note/eventbus v0.0.0-20230216103454-41885eac6c2b/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e h1:i3RKrdrddr4AuaHJtoWYAEVNuR7Y9wIsEqPmuFFbJC4=
github.com/siyuan-note/filelock v0.0.0-20221117095924-e1947438a35e/go.mod h1:NmpSIVtIGy8eNWapjDIiiCw5+5r5wxC76k40oG+WRXQ=
github.com/siyuan-note/httpclient v0.0.0-20230211023949-b9d36c2da3ea h1:Q6cuN3L4zWR+MwQVdMKeUnfusd+W0LLu1KeqSB3vfdQ=

1
kernel/job/cron.go
View file

@ -37,6 +37,7 @@ func StartCron() {
go every(3*time.Second, model.FlushUpdateRefTextRenameDocJob)
go every(50*time.Millisecond, model.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushTxJob)
go every(util.SQLFlushInterval, sql.FlushHistoryTxJob)
go every(10*time.Minute, model.FixIndexJob)
go every(10*time.Minute, model.IndexEmbedBlockJob)
go every(10*time.Minute, model.CacheVirtualBlockRefJob)

2
kernel/model/assets.go
View file

@ -356,7 +356,7 @@ func uploadAssets2Cloud(sqlAssets []*sql.Asset, bizType string) (err error) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudFileRequest2m()
resp, reqErr := request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetFile("file[]", absAsset).
SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}).
SetHeader("meta-type", metaType).

2
kernel/model/bookmark.go
View file

@ -158,8 +158,6 @@ func BuildBookmark() (ret *Bookmarks) {
WaitForWritingFiles()
if !sql.IsEmptyQueue() {
sql.WaitForWritingDatabase()
} else {
util.RandomSleep(200, 500)
}
ret = &Bookmarks{}

4
kernel/model/export.go
View file

@ -76,7 +76,7 @@ func Export2Liandi(id string) (err error) {
result := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
resp, getErr := request.
SetResult(result).
SetSuccessResult(result).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Get(util.LiandiServer + "/api/v2/article/update/" + articleId)
if nil != getErr {
@ -115,7 +115,7 @@ func Export2Liandi(id string) (err error) {
result := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
request = request.
SetResult(result).
SetSuccessResult(result).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
SetBody(map[string]interface{}{
"articleTitle": title,

2
kernel/model/graph.go
View file

@ -205,7 +205,7 @@ func BuildGraph(query string) (boxID string, nodes []*GraphNode, links []*GraphL
}
func linkTagBlocks(blocks *[]*Block, nodes *[]*GraphNode, links *[]*GraphLink, p string, style map[string]string) {
tagSpans := sql.QueryTagSpans(p, 1024)
tagSpans := sql.QueryTagSpans(p)
if 1 > len(tagSpans) {
return
}

35
kernel/model/history.go
View file

@ -70,15 +70,12 @@ func generateDocHistory() {
clearOutdatedHistoryDir(historyDir)
// 以下部分是老版本的历史数据，不再保留
for _, box := range Conf.GetBoxes() {
historyDir = filepath.Join(util.DataDir, box.ID, ".siyuan", "history")
os.RemoveAll(historyDir)
}
historyDir = filepath.Join(util.DataDir, "assets", ".siyuan", "history")
os.RemoveAll(historyDir)
historyDir = filepath.Join(util.DataDir, ".siyuan", "history")
os.RemoveAll(historyDir)
}
@ -517,23 +514,7 @@ func clearOutdatedHistoryDir(historyDir string) {
//logging.LogInfof("auto removed history dir [%s]", dir)
// 清理历史库
tx, txErr := sql.BeginHistoryTx()
if nil != txErr {
logging.LogErrorf("begin history tx failed: %s", txErr)
return
}
p := strings.TrimPrefix(dir, util.HistoryDir)
p = filepath.ToSlash(p[1:])
if txErr = sql.DeleteHistoriesByPathPrefix(tx, dir); nil != txErr {
logging.LogErrorf("delete history [%s] failed: %s", dir, txErr)
return
}
if txErr = sql.CommitHistoryTx(tx); nil != txErr {
logging.LogErrorf("commit history tx failed: %s", txErr)
return
}
sql.DeleteHistoriesByPathPrefixQueue(dir)
}
}
@ -678,19 +659,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) {
})
}
tx, txErr := sql.BeginHistoryTx()
if nil != txErr {
logging.LogErrorf("begin transaction failed: %s", txErr)
return
}
if err := sql.InsertHistories(tx, histories); nil != err {
logging.LogErrorf("insert histories failed: %s", err)
return
}
if err := sql.CommitHistoryTx(tx); nil != err {
logging.LogErrorf("commit transaction failed: %s", err)
return
}
sql.IndexHistoriesQueue(histories)
return
}

12
kernel/model/index.go
View file

@ -286,4 +286,16 @@ func init() {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtSQLInsertHistory, func(context map[string]interface{}) {
if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
return
}
current := context["current"].(int) + 1
total := context["total"]
msg := fmt.Sprintf(Conf.Language(191), current, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
}

24
kernel/model/liandi.go
View file

@ -44,7 +44,7 @@ func StartFreeTrial() (err error) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
_, err = request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/user/startFreeTrial")
if nil != err {
@ -61,7 +61,7 @@ func DeactivateUser() (err error) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
resp, err := request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/user/deactivate")
if nil != err {
@ -86,7 +86,7 @@ func SetCloudBlockReminder(id, data string, timed int64) (err error) {
payload := map[string]interface{}{"dataId": id, "data": data, "timed": timed}
request := httpclient.NewCloudRequest30s()
resp, err := request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetBody(payload).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/calendar/setBlockReminder")
@ -119,7 +119,7 @@ func LoadUploadToken() (err error) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
resp, err := request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/upload/token")
if nil != err {
@ -333,7 +333,7 @@ func RemoveCloudShorthands(ids []string) (err error) {
"ids": ids,
}
resp, err := request.
SetResult(&result).
SetSuccessResult(&result).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
SetBody(body).
Post(util.AliyunServer + "/apis/siyuan/inbox/removeCloudShorthands")
@ -361,7 +361,7 @@ func GetCloudShorthand(id string) (ret map[string]interface{}, err error) {
result := map[string]interface{}{}
request := httpclient.NewCloudRequest30s()
resp, err := request.
SetResult(&result).
SetSuccessResult(&result).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/inbox/getCloudShorthand?id=" + id)
if nil != err {
@ -392,7 +392,7 @@ func GetCloudShorthands(page int) (result map[string]interface{}, err error) {
result = map[string]interface{}{}
request := httpclient.NewCloudRequest30s()
resp, err := request.
SetResult(&result).
SetSuccessResult(&result).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/inbox/getCloudShorthands?p=" + strconv.Itoa(page))
if nil != err {
@ -429,7 +429,7 @@ func getUser(token string) (*conf.User, error) {
result := map[string]interface{}{}
request := httpclient.NewCloudRequest30s()
_, err := request.
SetResult(&result).
SetSuccessResult(&result).
SetBody(map[string]string{"token": token}).
Post(util.AliyunServer + "/apis/siyuan/user")
if nil != err {
@ -462,7 +462,7 @@ func UseActivationcode(code string) (err error) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
_, err = request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetBody(map[string]string{"data": code}).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/useActivationcode")
@ -483,7 +483,7 @@ func CheckActivationcode(code string) (retCode int, msg string) {
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudRequest30s()
_, err := request.
SetResult(requestResult).
SetSuccessResult(requestResult).
SetBody(map[string]string{"data": code}).
SetCookies(&http.Cookie{Name: "symphony", Value: Conf.User.UserToken}).
Post(util.AliyunServer + "/apis/siyuan/checkActivationcode")
@ -503,7 +503,7 @@ func Login(userName, password, captcha string) (ret *gulu.Result, err error) {
result := map[string]interface{}{}
request := httpclient.NewCloudRequest30s()
_, err = request.
SetResult(&result).
SetSuccessResult(&result).
SetBody(map[string]string{"userName": userName, "userPassword": password, "captcha": captcha}).
Post(util.AliyunServer + "/apis/siyuan/login")
if nil != err {
@ -529,7 +529,7 @@ func Login2fa(token, code string) (map[string]interface{}, error) {
result := map[string]interface{}{}
request := httpclient.NewCloudRequest30s()
_, err := request.
SetResult(&result).
SetSuccessResult(&result).
SetBody(map[string]string{"twofactorAuthCode": code}).
SetHeader("token", token).
Post(util.AliyunServer + "/apis/siyuan/login/2fa")

2
kernel/model/sync.go
View file

@ -137,6 +137,8 @@ func BootSyncData() {
}
if !util.IsOnline() {
BootSyncSucc = 1
util.PushErrMsg(Conf.Language(28), 7000)
return
}

8
kernel/model/tag.go
View file

@ -41,7 +41,7 @@ func RemoveTag(label string) (err error) {
util.PushEndlessProgress(Conf.Language(116))
util.RandomSleep(1000, 2000)
tags := sql.QueryTagSpansByKeyword(label, 102400)
tags := sql.QueryTagSpansByLabel(label)
treeBlocks := map[string][]string{}
for _, tag := range tags {
if blocks, ok := treeBlocks[tag.RootID]; !ok {
@ -126,7 +126,7 @@ func RenameTag(oldLabel, newLabel string) (err error) {
util.PushEndlessProgress(Conf.Language(110))
util.RandomSleep(500, 1000)
tags := sql.QueryTagSpansByKeyword(oldLabel, 102400)
tags := sql.QueryTagSpansByLabel(oldLabel)
treeBlocks := map[string][]string{}
for _, tag := range tags {
if blocks, ok := treeBlocks[tag.RootID]; !ok {
@ -212,8 +212,6 @@ func BuildTags() (ret *Tags) {
WaitForWritingFiles()
if !sql.IsEmptyQueue() {
sql.WaitForWritingDatabase()
} else {
util.RandomSleep(200, 500)
}
ret = &Tags{}
@ -310,7 +308,7 @@ func labelBlocksByKeyword(keyword string) (ret map[string]TagBlocks) {
func labelTags() (ret map[string]Tags) {
ret = map[string]Tags{}
tagSpans := sql.QueryTagSpans("", 10240)
tagSpans := sql.QueryTagSpans("")
for _, tagSpan := range tagSpans {
label := tagSpan.Content
if _, ok := ret[label]; ok {

30
kernel/model/virutalref.go
View file

@ -21,11 +21,13 @@ import (
"regexp"
"sort"
"strings"
"time"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/ClarkThan/ahocorasick"
"github.com/dgraph-io/ristretto"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
@ -68,18 +70,23 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s
}
contentTmp := blockContent
var keywordsTmp []string
if !Conf.Search.CaseSensitive {
contentTmp = strings.ToLower(blockContent)
for _, keyword := range keywords {
keywordsTmp = append(keywordsTmp, strings.ToLower(keyword))
}
} else {
for _, keyword := range keywords {
keywordsTmp = append(keywordsTmp, keyword)
}
}
for _, keyword := range keywords {
keywordTmp := keyword
if !Conf.Search.CaseSensitive {
keywordTmp = strings.ToLower(keyword)
}
if strings.Contains(contentTmp, keywordTmp) {
ret = append(ret, keyword)
}
m := ahocorasick.NewMatcher()
m.BuildWithPatterns(keywordsTmp)
hits := m.Search(contentTmp)
for _, hit := range hits {
ret = append(ret, hit)
}
if 1 > len(ret) {
@ -87,7 +94,7 @@ func putBlockVirtualRefKeywords(blockContent, blockID, docTitle string) (ret []s
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
virtualBlockRefCache.Set(blockID, ret, 1)
virtualBlockRefCache.SetWithTTL(blockID, ret, 1, 10*time.Minute)
return
}
@ -101,6 +108,11 @@ func CacheVirtualBlockRefJob() {
virtualBlockRefCache.Set("virtual_ref", keywords, 1)
}
func ResetVirtualBlockRefCache() {
virtualBlockRefCache.Clear()
CacheVirtualBlockRefJob()
}
func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
if !Conf.Editor.VirtualBlockRef {
return false

4
kernel/sql/block_query.go
View file

@ -274,8 +274,8 @@ func queryDocIDsByTitle(title string, excludeIDs []string) (ret []string) {
func queryDocTitles() (ret []string) {
ret = []string{}
sqlStmt := "SELECT content FROM blocks WHERE type = 'd' LIMIT ?"
rows, err := query(sqlStmt, 10240)
sqlStmt := "SELECT content FROM blocks WHERE type = 'd'"
rows, err := query(sqlStmt)
if nil != err {
logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err)
return

8
kernel/sql/database.go
View file

@ -230,8 +230,8 @@ func initHistoryDBConnection() {
if nil != err {
logging.LogFatalf("create database failed: %s", err)
}
historyDB.SetMaxIdleConns(1)
historyDB.SetMaxOpenConns(1)
historyDB.SetMaxIdleConns(3)
historyDB.SetMaxOpenConns(3)
historyDB.SetConnMaxLifetime(365 * 24 * time.Hour)
}
@ -1142,7 +1142,7 @@ func beginTx() (tx *sql.Tx, err error) {
return
}
func BeginHistoryTx() (tx *sql.Tx, err error) {
func beginHistoryTx() (tx *sql.Tx, err error) {
if tx, err = historyDB.Begin(); nil != err {
logging.LogErrorf("begin history tx failed: %s\n %s", err, logging.ShortStack())
if strings.Contains(err.Error(), "database is locked") {
@ -1152,7 +1152,7 @@ func BeginHistoryTx() (tx *sql.Tx, err error) {
return
}
func CommitHistoryTx(tx *sql.Tx) (err error) {
func commitHistoryTx(tx *sql.Tx) (err error) {
if nil == tx {
logging.LogErrorf("tx is nil")
return errors.New("tx is nil")

13
kernel/sql/history.go
View file

@ -22,6 +22,7 @@ import (
"fmt"
"strings"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
)
@ -102,7 +103,7 @@ func queryHistory(query string, args ...interface{}) (*sql.Rows, error) {
return historyDB.Query(query, args...)
}
func DeleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string) (err error) {
func deleteHistoriesByPathPrefix(tx *sql.Tx, pathPrefix string, context map[string]interface{}) (err error) {
stmt := "DELETE FROM histories_fts_case_insensitive WHERE path LIKE ?"
if err = execStmtTx(tx, stmt, pathPrefix+"%"); nil != err {
return
@ -115,7 +116,7 @@ const (
HistoriesPlaceholder = "(?, ?, ?, ?, ?, ?)"
)
func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
func insertHistories(tx *sql.Tx, histories []*History, context map[string]interface{}) (err error) {
if 1 > len(histories) {
return
}
@ -127,20 +128,20 @@ func InsertHistories(tx *sql.Tx, histories []*History) (err error) {
continue
}
if err = insertHistories0(tx, bulk); nil != err {
if err = insertHistories0(tx, bulk, context); nil != err {
return
}
bulk = []*History{}
}
if 0 < len(bulk) {
if err = insertHistories0(tx, bulk); nil != err {
if err = insertHistories0(tx, bulk, context); nil != err {
return
}
}
return
}
func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
func insertHistories0(tx *sql.Tx, bulk []*History, context map[string]interface{}) (err error) {
valueStrings := make([]string, 0, len(bulk))
valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(HistoriesPlaceholder, "?"))
for _, b := range bulk {
@ -157,5 +158,7 @@ func insertHistories0(tx *sql.Tx, bulk []*History) (err error) {
if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
return
}
eventbus.Publish(eventbus.EvtSQLInsertHistory, context)
return
}

139
kernel/sql/queue_history.go Normal file
View file

@ -0,0 +1,139 @@
// SiYuan - Build Your Eternal Digital Garden
// Copyright (c) 2020-present, b3log.org
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package sql
import (
"database/sql"
"errors"
"fmt"
"runtime/debug"
"sync"
"time"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/util"
)
var (
historyOperationQueue []*historyDBQueueOperation
historyDBQueueLock = sync.Mutex{}
historyTxLock = sync.Mutex{}
)
type historyDBQueueOperation struct {
inQueueTime time.Time
action string // index/deletePathPrefix
histories []*History // index
pathPrefix string // deletePathPrefix
}
func FlushHistoryTxJob() {
task.AppendTask(task.HistoryDatabaseIndexCommit, FlushHistoryQueue)
}
func FlushHistoryQueue() {
ops := getHistoryOperations()
if 1 > len(ops) {
return
}
txLock.Lock()
defer txLock.Unlock()
start := time.Now()
context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
total := len(ops)
for i, op := range ops {
if util.IsExiting {
return
}
tx, err := beginHistoryTx()
if nil != err {
return
}
context["current"] = i
context["total"] = total
if err = execHistoryOp(op, tx, context); nil != err {
tx.Rollback()
logging.LogErrorf("queue operation failed: %s", err)
continue
}
if err = commitHistoryTx(tx); nil != err {
logging.LogErrorf("commit tx failed: %s", err)
return
}
if 16 < i && 0 == i%128 {
debug.FreeOSMemory()
}
}
if 128 < len(ops) {
debug.FreeOSMemory()
}
elapsed := time.Now().Sub(start).Milliseconds()
if 7000 < elapsed {
logging.LogInfof("database history op tx [%dms]", elapsed)
}
}
func execHistoryOp(op *historyDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
switch op.action {
case "index":
err = insertHistories(tx, op.histories, context)
case "deletePathPrefix":
err = deleteHistoriesByPathPrefix(tx, op.pathPrefix, context)
default:
msg := fmt.Sprintf("unknown history operation [%s]", op.action)
logging.LogErrorf(msg)
err = errors.New(msg)
}
return
}
func DeleteHistoriesByPathPrefixQueue(pathPrefix string) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "deletePathPrefix", pathPrefix: pathPrefix}
historyOperationQueue = append(historyOperationQueue, newOp)
}
func IndexHistoriesQueue(histories []*History) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
newOp := &historyDBQueueOperation{inQueueTime: time.Now(), action: "index", histories: histories}
historyOperationQueue = append(historyOperationQueue, newOp)
}
func getHistoryOperations() (ops []*historyDBQueueOperation) {
historyDBQueueLock.Lock()
defer historyDBQueueLock.Unlock()
ops = historyOperationQueue
historyOperationQueue = nil
return
}

20
kernel/sql/span.go
View file

@ -80,8 +80,23 @@ func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) {
return
}
func QueryTagSpansByLabel(label string) (ret []*Span) {
stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + label + "%' GROUP BY block_id"
rows, err := query(stmt)
if nil != err {
logging.LogErrorf("sql query failed: %s", err)
return
}
defer rows.Close()
for rows.Next() {
span := scanSpanRows(rows)
ret = append(ret, span)
}
return
}
func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {
stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + keyword + "%'"
stmt := "SELECT * FROM spans WHERE type LIKE '%tag%' AND content LIKE '%" + keyword + "%' GROUP BY markdown"
stmt += " LIMIT " + strconv.Itoa(limit)
rows, err := query(stmt)
if nil != err {
@ -96,12 +111,11 @@ func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {
return
}
func QueryTagSpans(p string, limit int) (ret []*Span) {
func QueryTagSpans(p string) (ret []*Span) {
stmt := "SELECT * FROM spans WHERE type LIKE '%tag%'"
if "" != p {
stmt += " AND path = '" + p + "'"
}
stmt += " LIMIT " + strconv.Itoa(limit)
rows, err := query(stmt)
if nil != err {
logging.LogErrorf("sql query failed: %s", err)

22
kernel/task/queue.go
View file

@ -82,16 +82,17 @@ func getCurrentActions() (ret []string) {
}
const (
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
RepoCheckout = "task.repo.checkout" // 从快照中检出
DatabaseIndexFull = "task.database.index.full" // 重建索引
DatabaseIndex = "task.database.index" // 数据库索引
DatabaseIndexCommit = "task.database.index.commit" // 数据库索引提交
DatabaseIndexRef = "task.database.index.ref" // 数据库索引引用
DatabaseIndexFix = "task.database.index.fix" // 数据库索引订正
OCRImage = "task.ocr.image" // 图片 OCR 提取文本
HistoryGenerateDoc = "task.history.generateDoc" // 生成文件历史
HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
DatabaseIndexEmbedBlock = "task.database.index.embedBlock" // 数据库索引嵌入块
ReloadUI = "task.reload.ui" // 重载 UI
)
// uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
@ -101,6 +102,7 @@ var uniqueActions = []string{
DatabaseIndexCommit,
OCRImage,
HistoryGenerateDoc,
HistoryDatabaseIndexCommit,
DatabaseIndexEmbedBlock,
}