From b48202bbb8aea06bd7d04ea27c2f74d93ac1e602 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Mon, 9 Mar 2026 11:41:04 +0800 Subject: [PATCH] :art: Improve reindex Signed-off-by: Daniel <845765@qq.com> --- kernel/treenode/blocktree.go | 108 ++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/kernel/treenode/blocktree.go b/kernel/treenode/blocktree.go index ffcac8d89..518f79dbb 100644 --- a/kernel/treenode/blocktree.go +++ b/kernel/treenode/blocktree.go @@ -25,6 +25,7 @@ import ( "runtime/debug" "strings" "sync" + "sync/atomic" "time" "github.com/88250/gulu" @@ -49,7 +50,18 @@ var ( db *sql.DB ) +var ( + initDatabaseLock = sync.Mutex{} + isInitializingDatabase = atomic.Bool{} +) + func initDatabase(forceRebuild bool) (err error) { + initDatabaseLock.Lock() + defer initDatabaseLock.Unlock() + + isInitializingDatabase.Store(true) + defer isInitializingDatabase.Store(false) + initDBConnection() if !forceRebuild { @@ -138,7 +150,7 @@ func closeDatabase() { func GetBlockTreesByType(typ string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE type = ?" - rows, err := db.Query(sqlStmt, typ) + rows, err := query(sqlStmt, typ) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -158,7 +170,7 @@ func GetBlockTreesByType(typ string) (ret []*BlockTree) { func GetBlockTreeByPath(path string) (ret *BlockTree) { ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE path = ?" - err := db.QueryRow(sqlStmt, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) + err := queryRow(sqlStmt, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { @@ -172,7 +184,7 @@ func GetBlockTreeByPath(path string) (ret *BlockTree) { func CountTrees() (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE type = 'd'" - err := db.QueryRow(sqlStmt).Scan(&ret) + err := queryRow(sqlStmt).Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 @@ -184,7 +196,7 @@ func CountTrees() (ret int) { func CountBlocks() (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees" - err := db.QueryRow(sqlStmt).Scan(&ret) + err := queryRow(sqlStmt).Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 @@ -197,7 +209,7 @@ func CountBlocks() (ret int) { func GetBlockTreeRootByPath(boxID, path string) (ret *BlockTree) { ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND path = ? AND type = 'd'" - err := db.QueryRow(sqlStmt, boxID, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) + err := queryRow(sqlStmt, boxID, path).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { @@ -213,7 +225,7 @@ func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) { ret = &BlockTree{} hPath = gulu.Str.RemoveInvisible(hPath) sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND type = 'd'" - err := db.QueryRow(sqlStmt, boxID, hPath).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) + err := queryRow(sqlStmt, boxID, hPath).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { @@ -228,7 +240,7 @@ func GetBlockTreeRootByHPath(boxID, hPath string) (ret *BlockTree) { func GetBlockTreeRootsByHPath(boxID, hPath string) (ret []*BlockTree) { hPath = gulu.Str.RemoveInvisible(hPath) sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND type = 'd'" - rows, err := db.Query(sqlStmt, boxID, hPath) + rows, err := query(sqlStmt, boxID, hPath) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -249,7 +261,7 @@ func GetBlockTreeByHPathPreferredParentID(boxID, hPath, preferredParentID string hPath = gulu.Str.RemoveInvisible(hPath) var roots []*BlockTree sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ? AND hpath = ? AND parent_id = ? LIMIT 1" - rows, err := db.Query(sqlStmt, boxID, hPath, preferredParentID) + rows, err := query(sqlStmt, boxID, hPath, preferredParentID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -285,7 +297,7 @@ func GetBlockTreeByHPathPreferredParentID(boxID, hPath, preferredParentID string func ExistBlockTree(id string) bool { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE id = ?" var count int - err := db.QueryRow(sqlStmt, id).Scan(&count) + err := queryRow(sqlStmt, id).Scan(&count) if err != nil { if errors.Is(err, sql.ErrNoRows) { return false @@ -307,7 +319,7 @@ func ExistBlockTrees(ids []string) (ret map[string]bool) { } sqlStmt := "SELECT id FROM blocktrees WHERE id IN ('" + strings.Join(ids, "','") + "')" - rows, err := db.Query(sqlStmt) + rows, err := query(sqlStmt) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -345,7 +357,7 @@ func GetBlockTrees(ids []string) (ret map[string]*BlockTree) { args = append(args, id) } stmt := stmtBuf.String() - rows, err := db.Query(stmt, args...) + rows, err := query(stmt, args...) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", stmt, err) return @@ -369,7 +381,7 @@ func GetBlockTree(id string) (ret *BlockTree) { ret = &BlockTree{} sqlStmt := "SELECT * FROM blocktrees WHERE id = ?" - err := db.QueryRow(sqlStmt, id).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) + err := queryRow(sqlStmt, id).Scan(&ret.ID, &ret.RootID, &ret.ParentID, &ret.BoxID, &ret.Path, &ret.HPath, &ret.Updated, &ret.Type) if err != nil { ret = nil if errors.Is(err, sql.ErrNoRows) { @@ -388,7 +400,7 @@ func SetBlockTreePath(tree *parse.Tree) { func RemoveBlockTreesByRootID(rootID string) { sqlStmt := "DELETE FROM blocktrees WHERE root_id = ?" - _, err := db.Exec(sqlStmt, rootID) + _, err := exec(sqlStmt, rootID) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return @@ -397,7 +409,7 @@ func RemoveBlockTreesByRootID(rootID string) { func CountBlockTreesByPathPrefix(pathPrefix string) (ret int) { sqlStmt := "SELECT COUNT(*) FROM blocktrees WHERE path LIKE ?" - err := db.QueryRow(sqlStmt, pathPrefix+"%").Scan(&ret) + err := queryRow(sqlStmt, pathPrefix+"%").Scan(&ret) if err != nil { if errors.Is(err, sql.ErrNoRows) { return 0 @@ -409,7 +421,7 @@ func CountBlockTreesByPathPrefix(pathPrefix string) (ret int) { func GetBlockTreesByPathPrefix(pathPrefix string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE path LIKE ?" - rows, err := db.Query(sqlStmt, pathPrefix+"%") + rows, err := query(sqlStmt, pathPrefix+"%") if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -428,7 +440,7 @@ func GetBlockTreesByPathPrefix(pathPrefix string) (ret []*BlockTree) { func GetBlockTreesByRootID(rootID string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE root_id = ?" - rows, err := db.Query(sqlStmt, rootID) + rows, err := query(sqlStmt, rootID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -447,7 +459,7 @@ func GetBlockTreesByRootID(rootID string) (ret []*BlockTree) { func RemoveBlockTreesByPathPrefix(pathPrefix string) { sqlStmt := "DELETE FROM blocktrees WHERE path LIKE ?" - _, err := db.Exec(sqlStmt, pathPrefix+"%") + _, err := exec(sqlStmt, pathPrefix+"%") if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return @@ -456,7 +468,7 @@ func RemoveBlockTreesByPathPrefix(pathPrefix string) { func GetBlockTreesByBoxID(boxID string) (ret []*BlockTree) { sqlStmt := "SELECT * FROM blocktrees WHERE box_id = ?" - rows, err := db.Query(sqlStmt, boxID) + rows, err := query(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -475,7 +487,7 @@ func GetBlockTreesByBoxID(boxID string) (ret []*BlockTree) { func RemoveBlockTreesByBoxID(boxID string) (ids []string) { sqlStmt := "SELECT id FROM blocktrees WHERE box_id = ?" - rows, err := db.Query(sqlStmt, boxID) + rows, err := query(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql query [%s] failed: %s", sqlStmt, err) return @@ -491,7 +503,7 @@ func RemoveBlockTreesByBoxID(boxID string) (ids []string) { } sqlStmt = "DELETE FROM blocktrees WHERE box_id = ?" - _, err = db.Exec(sqlStmt, boxID) + _, err = exec(sqlStmt, boxID) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return @@ -505,7 +517,7 @@ func RemoveBlockTreesByIDs(ids []string) { } sqlStmt := "DELETE FROM blocktrees WHERE id IN ('" + strings.Join(ids, "','") + "')" - _, err := db.Exec(sqlStmt) + _, err := exec(sqlStmt) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return @@ -514,7 +526,7 @@ func RemoveBlockTreesByIDs(ids []string) { func RemoveBlockTree(id string) { sqlStmt := "DELETE FROM blocktrees WHERE id = ?" - _, err := db.Exec(sqlStmt, id) + _, err := exec(sqlStmt, id) if err != nil { logging.LogErrorf("sql exec [%s] failed: %s", sqlStmt, err) return @@ -687,3 +699,55 @@ func CeilBlockCount(count int) int { } return 10000*100 + 1 } + +func queryRow(query string, args ...interface{}) *sql.Row { + query = strings.TrimSpace(query) + if "" == query { + logging.LogErrorf("statement is empty") + return nil + } + + if isInitializingDatabase.Load() { + logging.LogWarnf("database is initializing, ignoring query [%s]", query) + return nil + } + + if nil == db { + return nil + } + return db.QueryRow(query, args...) +} + +func query(query string, args ...interface{}) (*sql.Rows, error) { + query = strings.TrimSpace(query) + if "" == query { + return nil, errors.New("statement is empty") + } + + if isInitializingDatabase.Load() { + logging.LogWarnf("database is initializing, ignoring query [%s]", query) + return nil, errors.New("database is initializing") + } + + if nil == db { + return nil, errors.New("database is nil") + } + return db.Query(query, args...) +} + +func exec(stmt string, args ...interface{}) (sql.Result, error) { + stmt = strings.TrimSpace(stmt) + if "" == stmt { + return nil, errors.New("statement is empty") + } + + if isInitializingDatabase.Load() { + logging.LogWarnf("database is initializing, ignoring exec [%s]", stmt) + return nil, errors.New("database is initializing") + } + + if nil == db { + return nil, errors.New("database is nil") + } + return db.Exec(stmt, args...) +}