✨ Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874

app/appearance/langs/en_US.json

@@ -1260,6 +1260,8 @@
     "212": "There are some defects in the current version of cloud data sync, please upgrade to the latest version. Sorry for the inconvenience",
     "213": "Cloud verification failed, please try to upgrade to the latest version and log in again before syncing",
     "214": "This function needs to be signed in to use",
-    "215": "Save failed: The target file is being used by another program"
+    "215": "Save failed: The target file is being used by another program",
+    "216": "Rebuilding asset content data index, please wait...",
+    "217": "[%d/%d] Created asset content data index"
   }
 }

app/appearance/langs/es_ES.json

@@ -1260,6 +1260,8 @@
     "212": "Hay algunos defectos en la versi\u00f3n actual de sincronizaci\u00f3n de datos en la nube, actualice a la versi\u00f3n m\u00e1s reciente. Disculpe las molestias",
     "213": "La verificaci\u00f3n en la nube fall\u00f3, intente actualizar a la versi\u00f3n m\u00e1s reciente e inicie sesi\u00f3n de nuevo antes de sincronizar",
     "214": "Esta función requiere iniciar sesión en la cuenta antes de poder usarla",
-    "215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa"
+    "215": "Error al guardar: el archivo de destino está siendo utilizado por otro programa",
+    "216": "Reconstruyendo el índice de datos de contenido de recursos, espere...",
+    "217": "[%d/%d] Índice de datos de contenido de activos creado"
   }
 }

app/appearance/langs/fr_FR.json

@@ -1260,6 +1260,8 @@
     "212": "Il y a quelques défauts dans la version actuelle de la synchronisation des données cloud, veuillez mettre à niveau vers la dernière version. Désolé pour le désagrément",
     "213": "Échec de la vérification cloud, veuillez essayer de mettre à niveau vers la dernière version et de vous reconnecter avant de synchroniser",
     "214": "La fonctionnalité nécessite un numéro de compte de connexion avant de pouvoir être utilisée",
-    "215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme"
+    "215": "Échec de l'enregistrement : le fichier de destination est utilisé par un autre programme",
+    "216": "Reconstruction de l'index des données du contenu des ressources, veuillez patienter...",
+    "217": "[%d/%d] Création d'un index de données de contenu d'actif"
   }
 }

app/appearance/langs/zh_CHT.json

@@ -1260,6 +1260,8 @@
     "212": "當前版本雲端數據同步存在一些缺陷，請升級到最新版，帶來不便，敬請諒解",
     "213": "雲端校驗失敗，請嘗試升級到最新版並重新登錄後再進行同步",
     "214": "該功能需要登錄賬號後才能使用",
-    "215": "保存失敗：目標文件正在被其他程序佔用"
+    "215": "保存失敗：目標文件正在被其他程序佔用",
+    "216": "正在重建資源文件內容數據索引，請稍等...",
+    "217": "[%d/%d] 已經建立條資源文件內容數據索引"
   }
 }

app/appearance/langs/zh_CN.json

@@ -1260,6 +1260,8 @@
     "212": "当前版本云端数据同步存在一些缺陷，请升级到最新版，带来不便，敬请谅解",
     "213": "云端校验失败，请尝试升级到最新版并重新登录后再进行同步",
     "214": "该功能需要登录账号后才能使用",
-    "215": "保存失败：目标文件并且正在被其他程序占用"
+    "215": "保存失败：目标文件并且正在被其他程序占用",
+    "216": "正在重建资源文件内容数据索引，请稍等...",
+    "217": "[%d/%d] 已经建立条资源文件内容数据索引"
   }
 }

kernel/cache/asset.go

@@ -79,7 +79,7 @@ func LoadAssets() {
 		assetsCache[path] = &Asset{
 			HName:   hName,
 			Path:    path,
-			Updated: info.ModTime().UnixMilli(),
+			Updated: info.ModTime().Unix(),
 		}
 		return nil
 	})

kernel/go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.23.6
 	github.com/siyuan-note/dejavu v0.0.0-20230801123133-2edc24064c33
 	github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75
-	github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112
+	github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80
 	github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524
 	github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2
 	github.com/siyuan-note/logging v0.0.0-20230327073243-ebe83aec1493

kernel/go.sum

@@ -296,6 +296,8 @@ github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75 h1:Bi7/7f29
 github.com/siyuan-note/encryption v0.0.0-20220713091850-5ecd92177b75/go.mod h1:H8fyqqAbp9XreANjeSbc72zEdFfKTXYN34tc1TjZwtw=
 github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112 h1:lb+8C+XEEEn/lcBtoXlrf5mZEoe0y0KlqiIGG93Gozc=
 github.com/siyuan-note/eventbus v0.0.0-20230702081350-6dde667e7112/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
+github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80 h1:XghjHKJd+SiL0DkGYFVC+UGUDFtnR4v9gkAbPeh9Eq8=
+github.com/siyuan-note/eventbus v0.0.0-20230804030110-cf250f838c80/go.mod h1:Sqo4FYX5lAXu7gWkbEdJF0e6P57tNNVV4WDKYDctokI=
 github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524 h1:ZuxN5gwqtUOd1NkOkNhM4OlVWfjujY98zsR+zFi4x9g=
 github.com/siyuan-note/filelock v0.0.0-20230615140405-d05a21d49524/go.mod h1:jK5lCYfPbFOrW23/HMeU7kmpLdEd5GkennF+kUpy7Vs=
 github.com/siyuan-note/httpclient v0.0.0-20230728124841-53922bac2be2 h1:z6vYbmEOVoytf30Ny6YDjyZTYdCPmazeAl4BN67+308=

kernel/job/cron.go

@@ -38,6 +38,7 @@ func StartCron() {
 	go every(50*time.Millisecond, model.FlushTxJob)
 	go every(util.SQLFlushInterval, sql.FlushTxJob)
 	go every(util.SQLFlushInterval, sql.FlushHistoryTxJob)
+	go every(util.SQLFlushInterval, sql.FlushAssetContentTxJob)
 	go every(10*time.Minute, model.FixIndexJob)
 	go every(10*time.Minute, model.IndexEmbedBlockJob)
 	go every(10*time.Minute, model.CacheVirtualBlockRefJob)

kernel/main.go

@@ -35,6 +35,7 @@ func main() {
 	model.InitAppearance()
 	sql.InitDatabase(false)
 	sql.InitHistoryDatabase(false)
+	sql.InitAssetContentDatabase(false)
 	sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
 	sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath)
 

kernel/mobile/kernel.go

@@ -49,6 +49,7 @@ func StartKernel(container, appDir, workspaceBaseDir, timezoneID, localIPs, lang
 		model.InitAppearance()
 		sql.InitDatabase(false)
 		sql.InitHistoryDatabase(false)
+		sql.InitAssetContentDatabase(false)
 		sql.SetCaseSensitive(model.Conf.Search.CaseSensitive)
 		sql.SetIndexAssetPath(model.Conf.Search.IndexAssetPath)
 

kernel/model/asset_content.go

@@ -0,0 +1,155 @@
+// SiYuan - Refactor your thinking
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package model
+
+import (
+	"github.com/88250/gulu"
+	"github.com/88250/lute/ast"
+	"github.com/siyuan-note/eventbus"
+	"github.com/siyuan-note/filelock"
+	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/sql"
+	"github.com/siyuan-note/siyuan/kernel/task"
+	"github.com/siyuan-note/siyuan/kernel/util"
+	"io/fs"
+	"path/filepath"
+	"strings"
+)
+
+func ReindexAssetContent() {
+	task.AppendTask(task.AssetContentDatabaseIndexFull, fullReindexAssetContent)
+	return
+}
+
+func fullReindexAssetContent() {
+	util.PushMsg(Conf.Language(216), 7*1000)
+	sql.InitAssetContentDatabase(true)
+
+	assetsSearch := NewAssetsSearcher()
+	assetsSearch.Index()
+	return
+}
+
+func init() {
+	subscribeSQLAssetContentEvents()
+}
+
+func subscribeSQLAssetContentEvents() {
+	eventbus.Subscribe(util.EvtSQLAssetContentRebuild, func() {
+		ReindexAssetContent()
+	})
+}
+
+var (
+	AssetsSearchEnabled = true
+)
+
+type AssetsSearcher struct {
+	AssetsDir string
+	Parsers   map[string]AssetParser
+}
+
+func (searcher *AssetsSearcher) Index() {
+	assetsDir := searcher.AssetsDir
+	if !gulu.File.IsDir(assetsDir) {
+		return
+	}
+
+	var results []*AssetParseResult
+	filepath.Walk(assetsDir, func(absPath string, info fs.FileInfo, err error) error {
+		if nil != err {
+			logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		ext := strings.ToLower(filepath.Ext(absPath))
+		parser, found := searcher.Parsers[ext]
+		if !found {
+			return nil
+		}
+
+		result := parser.Parse(absPath)
+		if nil == result {
+			return nil
+		}
+
+		result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
+		result.Size = info.Size()
+		result.Updated = info.ModTime().Unix()
+		results = append(results, result)
+		return nil
+	})
+
+	var assetContents []*sql.AssetContent
+	for _, result := range results {
+		assetContents = append(assetContents, &sql.AssetContent{
+			ID:      ast.NewNodeID(),
+			Name:    filepath.Base(result.Path),
+			Ext:     filepath.Ext(result.Path),
+			Path:    result.Path,
+			Size:    result.Size,
+			Updated: result.Updated,
+			Content: result.Content,
+		})
+	}
+
+	sql.IndexAssetContentsQueue(assetContents)
+}
+
+func NewAssetsSearcher() *AssetsSearcher {
+	return &AssetsSearcher{
+		AssetsDir: util.GetDataAssetsAbsPath(),
+		Parsers: map[string]AssetParser{
+			".txt": &TxtAssetParser{},
+		},
+	}
+}
+
+type AssetParseResult struct {
+	Path    string
+	Size    int64
+	Updated int64
+	Content string
+}
+
+type AssetParser interface {
+	Parse(absPath string) *AssetParseResult
+}
+
+type TxtAssetParser struct {
+}
+
+func (parser *TxtAssetParser) Parse(absPath string) (ret *AssetParseResult) {
+	if !strings.HasSuffix(strings.ToLower(absPath), ".txt") {
+		return
+	}
+
+	data, err := filelock.ReadFile(absPath)
+	if nil != err {
+		logging.LogErrorf("read file [%s] failed: %s", absPath, err)
+		return
+	}
+
+	ret = &AssetParseResult{
+		Content: string(data),
+	}
+	return
+}

kernel/model/index.go

@@ -303,4 +303,16 @@ func subscribeSQLEvents() {
 		util.SetBootDetails(msg)
 		util.ContextPushMsg(context, msg)
 	})
+
+	eventbus.Subscribe(eventbus.EvtSQLInsertAssetContent, func(context map[string]interface{}) {
+		if util.ContainerAndroid == util.Container || util.ContainerIOS == util.Container {
+			return
+		}
+
+		current := context["current"].(int)
+		total := context["total"]
+		msg := fmt.Sprintf(Conf.Language(217), current, total)
+		util.SetBootDetails(msg)
+		util.ContextPushMsg(context, msg)
+	})
 }

kernel/sql/asset_content.go

@@ -0,0 +1,96 @@
+// SiYuan - Refactor your thinking
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+
+	"github.com/siyuan-note/eventbus"
+)
+
+type AssetContent struct {
+	ID      string
+	Name    string
+	Ext     string
+	Path    string
+	Size    int64
+	Updated int64
+	Content string
+}
+
+const (
+	AssetContentsFTSCaseInsensitiveInsert = "INSERT INTO asset_contents_fts_case_insensitive (id, name, ext, path, size, updated, content) VALUES %s"
+	AssetContentsPlaceholder              = "(?, ?, ?, ?, ?, ?, ?)"
+)
+
+func insertAssetContents(tx *sql.Tx, assetContents []*AssetContent, context map[string]interface{}) (err error) {
+	if 1 > len(assetContents) {
+		return
+	}
+
+	var bulk []*AssetContent
+	for _, assetContent := range assetContents {
+		bulk = append(bulk, assetContent)
+		if 512 > len(bulk) {
+			continue
+		}
+
+		if err = insertAssetContents0(tx, bulk, context); nil != err {
+			return
+		}
+		bulk = []*AssetContent{}
+	}
+	if 0 < len(bulk) {
+		if err = insertAssetContents0(tx, bulk, context); nil != err {
+			return
+		}
+	}
+	return
+}
+
+func insertAssetContents0(tx *sql.Tx, bulk []*AssetContent, context map[string]interface{}) (err error) {
+	valueStrings := make([]string, 0, len(bulk))
+	valueArgs := make([]interface{}, 0, len(bulk)*strings.Count(AssetContentsPlaceholder, "?"))
+	for _, b := range bulk {
+		valueStrings = append(valueStrings, AssetContentsPlaceholder)
+		valueArgs = append(valueArgs, b.ID)
+		valueArgs = append(valueArgs, b.Name)
+		valueArgs = append(valueArgs, b.Ext)
+		valueArgs = append(valueArgs, b.Path)
+		valueArgs = append(valueArgs, b.Size)
+		valueArgs = append(valueArgs, b.Updated)
+		valueArgs = append(valueArgs, b.Content)
+	}
+
+	stmt := fmt.Sprintf(AssetContentsFTSCaseInsensitiveInsert, strings.Join(valueStrings, ","))
+	if err = prepareExecInsertTx(tx, stmt, valueArgs); nil != err {
+		return
+	}
+
+	eventbus.Publish(eventbus.EvtSQLInsertAssetContent, context)
+	return
+}
+
+func deleteAssetContentsByPath(tx *sql.Tx, path string, context map[string]interface{}) (err error) {
+	stmt := "DELETE FROM asset_contents_fts_case_insensitive WHERE path = ?"
+	if err = execStmtTx(tx, stmt, path); nil != err {
+		return
+	}
+	return
+}

kernel/sql/database.go

@@ -45,8 +45,9 @@ import (
 )
 
 var (
-	db        *sql.DB
+	db             *sql.DB
-	historyDB *sql.DB
+	historyDB      *sql.DB
+	assetContentDB *sql.DB
 )
 
 func init() {
@@ -193,7 +194,36 @@ func initDBTables() {
 	}
 }
 
+func initDBConnection() {
+	if nil != db {
+		closeDatabase()
+	}
+	dsn := util.DBPath + "?_journal_mode=WAL" +
+		"&_synchronous=OFF" +
+		"&_mmap_size=2684354560" +
+		"&_secure_delete=OFF" +
+		"&_cache_size=-20480" +
+		"&_page_size=32768" +
+		"&_busy_timeout=7000" +
+		"&_ignore_check_constraints=ON" +
+		"&_temp_store=MEMORY" +
+		"&_case_sensitive_like=OFF"
+	var err error
+	db, err = sql.Open("sqlite3_extended", dsn)
+	if nil != err {
+		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
+	}
+	db.SetMaxIdleConns(20)
+	db.SetMaxOpenConns(20)
+	db.SetConnMaxLifetime(365 * 24 * time.Hour)
+}
+
+var initHistoryDatabaseLock = sync.Mutex{}
+
 func InitHistoryDatabase(forceRebuild bool) {
+	initHistoryDatabaseLock.Lock()
+	defer initHistoryDatabaseLock.Unlock()
+
 	initHistoryDBConnection()
 
 	if !forceRebuild && gulu.File.IsExist(util.HistoryDBPath) {
@@ -228,7 +258,7 @@ func initHistoryDBConnection() {
 	var err error
 	historyDB, err = sql.Open("sqlite3_extended", dsn)
 	if nil != err {
-		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
+		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create history database failed: %s", err)
 	}
 	historyDB.SetMaxIdleConns(3)
 	historyDB.SetMaxOpenConns(3)
@@ -243,11 +273,34 @@ func initHistoryDBTables() {
 	}
 }
 
-func initDBConnection() {
+var initAssetContentDatabaseLock = sync.Mutex{}
-	if nil != db {
+
-		closeDatabase()
+func InitAssetContentDatabase(forceRebuild bool) {
+	initAssetContentDatabaseLock.Lock()
+	defer initAssetContentDatabaseLock.Unlock()
+
+	initAssetContentDBConnection()
+
+	if !forceRebuild && gulu.File.IsExist(util.AssetContentDBPath) {
+		return
 	}
-	dsn := util.DBPath + "?_journal_mode=WAL" +
+
+	assetContentDB.Close()
+	if err := os.RemoveAll(util.AssetContentDBPath); nil != err {
+		logging.LogErrorf("remove assets database file [%s] failed: %s", util.AssetContentDBPath, err)
+		return
+	}
+
+	initAssetContentDBConnection()
+	initAssetContentDBTables()
+}
+
+func initAssetContentDBConnection() {
+	if nil != assetContentDB {
+		assetContentDB.Close()
+	}
+
+	dsn := util.AssetContentDBPath + "?_journal_mode=WAL" +
 		"&_synchronous=OFF" +
 		"&_mmap_size=2684354560" +
 		"&_secure_delete=OFF" +
@@ -258,13 +311,21 @@ func initDBConnection() {
 		"&_temp_store=MEMORY" +
 		"&_case_sensitive_like=OFF"
 	var err error
-	db, err = sql.Open("sqlite3_extended", dsn)
+	assetContentDB, err = sql.Open("sqlite3_extended", dsn)
 	if nil != err {
-		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create database failed: %s", err)
+		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create assets database failed: %s", err)
+	}
+	assetContentDB.SetMaxIdleConns(3)
+	assetContentDB.SetMaxOpenConns(3)
+	assetContentDB.SetConnMaxLifetime(365 * 24 * time.Hour)
+}
+
+func initAssetContentDBTables() {
+	assetContentDB.Exec("DROP TABLE asset_contents_fts_case_insensitive")
+	_, err := assetContentDB.Exec("CREATE VIRTUAL TABLE asset_contents_fts_case_insensitive USING fts5(id UNINDEXED, name, ext, path, size UNINDEXED, updated UNINDEXED, content, tokenize=\"siyuan case_insensitive\")")
+	if nil != err {
+		logging.LogFatalf(logging.ExitCodeReadOnlyDatabase, "create table [asset_contents_fts_case_insensitive] failed: %s", err)
 	}
-	db.SetMaxIdleConns(20)
-	db.SetMaxOpenConns(20)
-	db.SetConnMaxLifetime(365 * 24 * time.Hour)
 }
 
 var (
@@ -1161,6 +1222,18 @@ func beginTx() (tx *sql.Tx, err error) {
 	return
 }
 
+func commitTx(tx *sql.Tx) (err error) {
+	if nil == tx {
+		logging.LogErrorf("tx is nil")
+		return errors.New("tx is nil")
+	}
+
+	if err = tx.Commit(); nil != err {
+		logging.LogErrorf("commit tx failed: %s\n  %s", err, logging.ShortStack())
+	}
+	return
+}
+
 func beginHistoryTx() (tx *sql.Tx, err error) {
 	if tx, err = historyDB.Begin(); nil != err {
 		logging.LogErrorf("begin history tx failed: %s\n  %s", err, logging.ShortStack())
@@ -1183,7 +1256,17 @@ func commitHistoryTx(tx *sql.Tx) (err error) {
 	return
 }
 
-func commitTx(tx *sql.Tx) (err error) {
+func beginAssetContentTx() (tx *sql.Tx, err error) {
+	if tx, err = assetContentDB.Begin(); nil != err {
+		logging.LogErrorf("begin asset content tx failed: %s\n  %s", err, logging.ShortStack())
+		if strings.Contains(err.Error(), "database is locked") {
+			os.Exit(logging.ExitCodeReadOnlyDatabase)
+		}
+	}
+	return
+}
+
+func commitAssetContentTx(tx *sql.Tx) (err error) {
 	if nil == tx {
 		logging.LogErrorf("tx is nil")
 		return errors.New("tx is nil")

kernel/sql/queue_asset_content.go

@@ -0,0 +1,147 @@
+// SiYuan - Refactor your thinking
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"runtime/debug"
+	"sync"
+	"time"
+
+	"github.com/siyuan-note/eventbus"
+	"github.com/siyuan-note/logging"
+	"github.com/siyuan-note/siyuan/kernel/task"
+	"github.com/siyuan-note/siyuan/kernel/util"
+)
+
+var (
+	assetContentOperationQueue []*assetContentDBQueueOperation
+	assetContentDBQueueLock    = sync.Mutex{}
+
+	assetContentTxLock = sync.Mutex{}
+)
+
+type assetContentDBQueueOperation struct {
+	inQueueTime time.Time
+	action      string // index/deletePath
+
+	assetContents []*AssetContent // index
+	path          string          // deletePath
+}
+
+func FlushAssetContentTxJob() {
+	task.AppendTask(task.AssetContentDatabaseIndexCommit, FlushAssetContentQueue)
+}
+
+func FlushAssetContentQueue() {
+	ops := getAssetContentOperations()
+	if 1 > len(ops) {
+		return
+	}
+
+	assetContentTxLock.Lock()
+	defer assetContentTxLock.Unlock()
+	start := time.Now()
+
+	groupOpsTotal := map[string]int{}
+	for _, op := range ops {
+		groupOpsTotal[op.action]++
+	}
+
+	context := map[string]interface{}{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar}
+	groupOpsCurrent := map[string]int{}
+	for i, op := range ops {
+		if util.IsExiting {
+			return
+		}
+
+		tx, err := beginAssetContentTx()
+		if nil != err {
+			return
+		}
+
+		groupOpsCurrent[op.action]++
+		context["current"] = groupOpsCurrent[op.action]
+		context["total"] = groupOpsTotal[op.action]
+
+		if err = execAssetContentOp(op, tx, context); nil != err {
+			tx.Rollback()
+			logging.LogErrorf("queue operation failed: %s", err)
+			eventbus.Publish(util.EvtSQLAssetContentRebuild)
+			return
+		}
+
+		if err = commitAssetContentTx(tx); nil != err {
+			logging.LogErrorf("commit tx failed: %s", err)
+			return
+		}
+
+		if 16 < i && 0 == i%128 {
+			debug.FreeOSMemory()
+		}
+	}
+
+	if 128 < len(ops) {
+		debug.FreeOSMemory()
+	}
+
+	elapsed := time.Now().Sub(start).Milliseconds()
+	if 7000 < elapsed {
+		logging.LogInfof("database asset content op tx [%dms]", elapsed)
+	}
+}
+
+func execAssetContentOp(op *assetContentDBQueueOperation, tx *sql.Tx, context map[string]interface{}) (err error) {
+	switch op.action {
+	case "index":
+		err = insertAssetContents(tx, op.assetContents, context)
+	case "delete":
+		err = deleteAssetContentsByPath(tx, op.path, context)
+	default:
+		msg := fmt.Sprintf("unknown asset content operation [%s]", op.action)
+		logging.LogErrorf(msg)
+		err = errors.New(msg)
+	}
+	return
+}
+
+func DeleteAssetContentsByPathQueue(path string) {
+	assetContentTxLock.Lock()
+	defer assetContentTxLock.Unlock()
+
+	newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "deletePath", path: path}
+	assetContentOperationQueue = append(assetContentOperationQueue, newOp)
+}
+
+func IndexAssetContentsQueue(assetContents []*AssetContent) {
+	assetContentTxLock.Lock()
+	defer assetContentTxLock.Unlock()
+
+	newOp := &assetContentDBQueueOperation{inQueueTime: time.Now(), action: "index", assetContents: assetContents}
+	assetContentOperationQueue = append(assetContentOperationQueue, newOp)
+}
+
+func getAssetContentOperations() (ops []*assetContentDBQueueOperation) {
+	assetContentTxLock.Lock()
+	defer assetContentTxLock.Unlock()
+
+	ops = assetContentOperationQueue
+	assetContentOperationQueue = nil
+	return
+}

kernel/sql/queue_history.go

@@ -55,8 +55,8 @@ func FlushHistoryQueue() {
 		return
 	}
 
-	txLock.Lock()
+	historyTxLock.Lock()
-	defer txLock.Unlock()
+	defer historyTxLock.Unlock()
 	start := time.Now()
 
 	groupOpsTotal := map[string]int{}
@@ -145,27 +145,3 @@ func getHistoryOperations() (ops []*historyDBQueueOperation) {
 	historyOperationQueue = nil
 	return
 }
-
-func WaitForWritingHistoryDatabase() {
-	var printLog bool
-	var lastPrintLog bool
-	for i := 0; isWritingHistoryDatabase(); i++ {
-		time.Sleep(50 * time.Millisecond)
-		if 200 < i && !printLog { // 10s 后打日志
-			logging.LogWarnf("history database is writing: \n%s", logging.ShortStack())
-			printLog = true
-		}
-		if 1200 < i && !lastPrintLog { // 60s 后打日志
-			logging.LogWarnf("history database is still writing")
-			lastPrintLog = true
-		}
-	}
-}
-
-func isWritingHistoryDatabase() bool {
-	time.Sleep(util.SQLFlushInterval + 50*time.Millisecond)
-	if 0 < len(historyOperationQueue) || util.IsMutexLocked(&historyTxLock) {
-		return true
-	}
-	return false
-}

kernel/task/queue.go

@@ -82,19 +82,21 @@ func getCurrentActions() (ret []string) {
 }
 
 const (
-	RepoCheckout               = "task.repo.checkout"                 // 从快照中检出
+	RepoCheckout                    = "task.repo.checkout"                 // 从快照中检出
-	DatabaseIndexFull          = "task.database.index.full"           // 重建索引
+	DatabaseIndexFull               = "task.database.index.full"           // 重建索引
-	DatabaseIndex              = "task.database.index"                // 数据库索引
+	DatabaseIndex                   = "task.database.index"                // 数据库索引
-	DatabaseIndexCommit        = "task.database.index.commit"         // 数据库索引提交
+	DatabaseIndexCommit             = "task.database.index.commit"         // 数据库索引提交
-	DatabaseIndexRef           = "task.database.index.ref"            // 数据库索引引用
+	DatabaseIndexRef                = "task.database.index.ref"            // 数据库索引引用
-	DatabaseIndexFix           = "task.database.index.fix"            // 数据库索引订正
+	DatabaseIndexFix                = "task.database.index.fix"            // 数据库索引订正
-	OCRImage                   = "task.ocr.image"                     // 图片 OCR 提取文本
+	OCRImage                        = "task.ocr.image"                     // 图片 OCR 提取文本
-	HistoryGenerateDoc         = "task.history.generateDoc"           // 生成文件历史
+	HistoryGenerateDoc              = "task.history.generateDoc"           // 生成文件历史
-	HistoryDatabaseIndexFull   = "task.history.database.index.full"   // 历史数据库重建索引
+	HistoryDatabaseIndexFull        = "task.history.database.index.full"   // 历史数据库重建索引
-	HistoryDatabaseIndexCommit = "task.history.database.index.commit" // 历史数据库索引提交
+	HistoryDatabaseIndexCommit      = "task.history.database.index.commit" // 历史数据库索引提交
-	DatabaseIndexEmbedBlock    = "task.database.index.embedBlock"     // 数据库索引嵌入块
+	DatabaseIndexEmbedBlock         = "task.database.index.embedBlock"     // 数据库索引嵌入块
-	ReloadUI                   = "task.reload.ui"                     // 重载 UI
+	ReloadUI                        = "task.reload.ui"                     // 重载 UI
-	UpgradeUserGuide           = "task.upgrade.userGuide"             // 升级用户指南文档笔记本
+	UpgradeUserGuide                = "task.upgrade.userGuide"             // 升级用户指南文档笔记本
+	AssetContentDatabaseIndexFull   = "task.asset.database.index.full"     // 资源文件数据库重建索引
+	AssetContentDatabaseIndexCommit = "task.asset.database.index.commit"   // 资源文件数据库索引提交
 )
 
 // uniqueActions 描述了唯一的任务，即队列中只能存在一个在执行的任务。
@@ -107,6 +109,8 @@ var uniqueActions = []string{
 	HistoryDatabaseIndexFull,
 	HistoryDatabaseIndexCommit,
 	DatabaseIndexEmbedBlock,
+	AssetContentDatabaseIndexFull,
+	AssetContentDatabaseIndexCommit,
 }
 
 func Contain(action string, moreActions ...string) bool {

kernel/util/runtime.go

@@ -408,5 +408,6 @@ func existAvailabilityStatus(workspaceAbsPath string) bool {
 const (
 	EvtConfPandocInitialized = "conf.pandoc.initialized"
 
-	EvtSQLHistoryRebuild = "sql.history.rebuild"
+	EvtSQLHistoryRebuild      = "sql.history.rebuild"
+	EvtSQLAssetContentRebuild = "sql.assetContent.rebuild"
 )

kernel/util/working.go

@@ -158,22 +158,23 @@ var (
 	HomeDir, _    = gulu.OS.Home()
 	WorkingDir, _ = os.Getwd()
 
-	WorkspaceDir   string        // 工作空间目录路径
+	WorkspaceDir       string        // 工作空间目录路径
-	WorkspaceLock  *flock.Flock  // 工作空间锁
+	WorkspaceLock      *flock.Flock  // 工作空间锁
-	ConfDir        string        // 配置目录路径
+	ConfDir            string        // 配置目录路径
-	DataDir        string        // 数据目录路径
+	DataDir            string        // 数据目录路径
-	RepoDir        string        // 仓库目录路径
+	RepoDir            string        // 仓库目录路径
-	HistoryDir     string        // 数据历史目录路径
+	HistoryDir         string        // 数据历史目录路径
-	TempDir        string        // 临时目录路径
+	TempDir            string        // 临时目录路径
-	LogPath        string        // 配置目录下的日志文件 siyuan.log 路径
+	LogPath            string        // 配置目录下的日志文件 siyuan.log 路径
-	DBName         = "siyuan.db" // SQLite 数据库文件名
+	DBName             = "siyuan.db" // SQLite 数据库文件名
-	DBPath         string        // SQLite 数据库文件路径
+	DBPath             string        // SQLite 数据库文件路径
-	HistoryDBPath  string        // SQLite 历史数据库文件路径
+	HistoryDBPath      string        // SQLite 历史数据库文件路径
-	BlockTreePath  string        // 区块树文件路径
+	AssetContentDBPath string        // SQLite 资源文件内容数据库文件路径
-	AppearancePath string        // 配置目录下的外观目录 appearance/ 路径
+	BlockTreePath      string        // 区块树文件路径
-	ThemesPath     string        // 配置目录下的外观目录下的 themes/ 路径
+	AppearancePath     string        // 配置目录下的外观目录 appearance/ 路径
-	IconsPath      string        // 配置目录下的外观目录下的 icons/ 路径
+	ThemesPath         string        // 配置目录下的外观目录下的 themes/ 路径
-	SnippetsPath   string        // 数据目录下的 snippets/ 路径
+	IconsPath          string        // 配置目录下的外观目录下的 icons/ 路径
+	SnippetsPath       string        // 数据目录下的 snippets/ 路径
 
 	UIProcessIDs = sync.Map{} // UI 进程 ID
 )
@@ -247,6 +248,7 @@ func initWorkspaceDir(workspaceArg string) {
 	os.Setenv("TMP", osTmpDir)
 	DBPath = filepath.Join(TempDir, DBName)
 	HistoryDBPath = filepath.Join(TempDir, "history.db")
+	AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
 	BlockTreePath = filepath.Join(TempDir, "blocktree")
 	SnippetsPath = filepath.Join(DataDir, "snippets")
 }

kernel/util/working_mobile.go

@@ -159,6 +159,7 @@ func initWorkspaceDirMobile(workspaceBaseDir string) {
 	os.Setenv("TMP", osTmpDir)
 	DBPath = filepath.Join(TempDir, DBName)
 	HistoryDBPath = filepath.Join(TempDir, "history.db")
+	AssetContentDBPath = filepath.Join(TempDir, "asset_content.db")
 	BlockTreePath = filepath.Join(TempDir, "blocktree")
 	SnippetsPath = filepath.Join(DataDir, "snippets")