From 231cb71a4d8f821c48c617de97fa3da8e1c7c827 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 29 Aug 2022 11:25:13 +0800 Subject: [PATCH 1/3] =?UTF-8?q?:bug:=20=E6=96=87=E6=A1=A3=E5=8F=AA?= =?UTF-8?q?=E6=9C=89=E4=B8=80=E4=B8=AA=E6=A0=87=E9=A2=98=E5=9D=97=E6=97=B6?= =?UTF-8?q?=E5=85=A8=E9=80=89=E5=85=B6=E8=A1=8C=E7=BA=A7=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E5=90=8E=E5=89=AA=E5=88=87=E5=88=B7=E6=96=B0=E8=A7=A6=E5=8F=91?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E5=BC=82=E5=B8=B8=20Fix=20https://github.com?= =?UTF-8?q?/siyuan-note/siyuan/issues/5746?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/filesys/tree.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.go index 54e104585..117aeeb41 100644 --- a/kernel/filesys/tree.go +++ b/kernel/filesys/tree.go @@ -100,6 +100,7 @@ func WriteTree(tree *parse.Tree) (err error) { newP := parse.NewParagraph() tree.Root.AppendChild(newP) tree.Root.SetIALAttr("updated", util.TimeFromID(newP.ID)) + treenode.ReindexBlockTree(tree) } renderer := render.NewJSONRenderer(tree, luteEngine.RenderOptions) From 10032fd154c8a4529f9563f8de58adf02759bb36 Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 29 Aug 2022 11:29:52 +0800 Subject: [PATCH 2/3] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E6=96=87=E6=A1=A3=E5=92=8C=E8=B5=84=E6=BA=90=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E5=88=86=E9=A1=B5=E5=92=8C=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=20https://github.com/siyuan-note/siyuan/issues/4901?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/history.go | 6 ++- kernel/model/history.go | 112 ++++++++++------------------------------ 2 files changed, 32 insertions(+), 86 deletions(-) diff --git a/kernel/api/history.go b/kernel/api/history.go index 28feb9ce6..f6257c8cb 100644 --- a/kernel/api/history.go +++ b/kernel/api/history.go @@ -114,7 +114,11 @@ func getDocHistory(c *gin.Context) { } notebook := arg["notebook"].(string) - histories, err := model.GetDocHistory(notebook) + page := 1 + if nil != arg["page"] { + page = int(arg["page"].(float64)) + } + histories, err := model.GetDocHistory(notebook, page) if nil != err { ret.Code = -1 ret.Msg = err.Error() diff --git a/kernel/model/history.go b/kernel/model/history.go index 0ac0e6139..f108b48fd 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -262,93 +262,22 @@ type HistoryItem struct { Path string `json:"path"` } -const maxHistory = 32 +func GetDocHistory(boxID string, page int) (ret []*History, err error) { + pageSize := 32 + from := (page - 1) * pageSize + to := page * pageSize -func GetDocHistory(boxID string) (ret []*History, err error) { - ret = []*History{} - - historyDir := util.HistoryDir - if !gulu.File.IsDir(historyDir) { - return - } - - historyBoxDirs, err := filepath.Glob(historyDir + "/*/" + boxID) - if nil != err { - logging.LogErrorf("read dir [%s] failed: %s", historyDir, err) - return - } - sort.Slice(historyBoxDirs, func(i, j int) bool { - return historyBoxDirs[i] > historyBoxDirs[j] - }) - - luteEngine := NewLute() - count := 0 - for _, historyBoxDir := range historyBoxDirs { - var docs []*HistoryItem - itemCount := 0 - filepath.Walk(historyBoxDir, func(path string, info fs.FileInfo, err error) error { - if info.IsDir() { - return nil - } - - if !strings.HasSuffix(info.Name(), ".sy") { - return nil - } - - data, err := filelock.NoLockFileRead(path) - if nil != err { - logging.LogErrorf("read file [%s] failed: %s", path, err) - return nil - } - historyTree, err := parse.ParseJSONWithoutFix(data, luteEngine.ParseOptions) - if nil != err { - logging.LogErrorf("parse tree from file [%s] failed, remove it", path) - os.RemoveAll(path) - return nil - } - historyName := historyTree.Root.IALAttr("title") - if "" == historyName { - historyName = info.Name() - } - - docs = append(docs, &HistoryItem{ - Title: historyTree.Root.IALAttr("title"), - Path: path, - }) - itemCount++ - if maxHistory < itemCount { - return io.EOF - } - return nil - }) - - if 1 > len(docs) { - continue - } - - timeDir := filepath.Base(filepath.Dir(historyBoxDir)) - t := timeDir[:strings.LastIndex(timeDir, "-")] - if ti, parseErr := time.Parse("2006-01-02-150405", t); nil == parseErr { - t = ti.Format("2006-01-02 15:04:05") - } - - ret = append(ret, &History{ - HCreated: t, - Items: docs, - }) - - count++ - if maxHistory <= count { - break - } - } - - sort.Slice(ret, func(i, j int) bool { - return ret[i].HCreated > ret[j].HCreated - }) + table := "histories_fts_case_insensitive" + projections := "type, op, title, content, path, created" + stmt := "SELECT " + projections + " FROM " + table + " WHERE path LIKE '%/" + boxID + "/%.sy'" + stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(from) + ", " + strconv.Itoa(to) + sqlHistories := sql.SelectHistoriesRawStmt(stmt) + ret = fromSQLHistories(sqlHistories) return } +const maxHistory = 32 + func GetNotebookHistory() (ret []*History, err error) { ret = []*History{} @@ -642,7 +571,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) (err error) { return } t := name[:strings.LastIndex(name, "-")] - tt, parseErr := time.Parse("2006-01-02-150405", t) + tt, parseErr := time.ParseInLocation("2006-01-02-150405", t, time.Local) if nil != parseErr { logging.LogWarnf("parse history dir time [%s] failed: %s", t, parseErr) return @@ -718,11 +647,20 @@ func FullTextSearchHistory(query string, page int) (ret []*History) { query = gulu.Str.RemoveInvisible(query) query = stringQuery(query) + pageSize := 32 + from := (page - 1) * pageSize + to := page * pageSize + table := "histories_fts_case_insensitive" projections := "type, op, title, content, path, created" stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '{title content}:(" + query + ")'" - stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(page) + stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(from) + ", " + strconv.Itoa(to) sqlHistories := sql.SelectHistoriesRawStmt(stmt) + ret = fromSQLHistories(sqlHistories) + return +} + +func fromSQLHistories(sqlHistories []*sql.History) (ret []*History) { if 1 > len(sqlHistories) { ret = []*History{} return @@ -746,6 +684,10 @@ func FullTextSearchHistory(query string, page int) (ret []*History) { Items: items, }) items = []*HistoryItem{} + items = append(items, &HistoryItem{ + Title: sqlHistory.Title, + Path: filepath.Join(util.HistoryDir, sqlHistory.Path), + }) } } return From 489736645c5358c425e23ed56622bc9ded209d4c Mon Sep 17 00:00:00 2001 From: Liang Ding Date: Mon, 29 Aug 2022 11:57:19 +0800 Subject: [PATCH 3/3] =?UTF-8?q?:art:=20=E6=95=B0=E6=8D=AE=E5=8E=86?= =?UTF-8?q?=E5=8F=B2=E6=96=87=E6=A1=A3=E5=92=8C=E8=B5=84=E6=BA=90=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E6=94=AF=E6=8C=81=E5=88=86=E9=A1=B5=E5=92=8C=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=20https://github.com/siyuan-note/siyuan/issues/4901?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/api/history.go | 16 ++++- kernel/model/history.go | 138 ++++++++++++---------------------------- 2 files changed, 53 insertions(+), 101 deletions(-) diff --git a/kernel/api/history.go b/kernel/api/history.go index f6257c8cb..4f1c50274 100644 --- a/kernel/api/history.go +++ b/kernel/api/history.go @@ -36,7 +36,10 @@ func searchHistory(c *gin.Context) { } query := arg["query"].(string) - page := int(arg["page"].(float64)) + page := 1 + if nil != arg["page"] { + page = int(arg["page"].(float64)) + } histories := model.FullTextSearchHistory(query, page) ret.Data = map[string]interface{}{ "histories": histories, @@ -75,7 +78,16 @@ func getAssetsHistory(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) - histories, err := model.GetAssetsHistory() + arg, ok := util.JsonArg(c, ret) + if !ok { + return + } + + page := 1 + if nil != arg["page"] { + page = int(arg["page"].(float64)) + } + histories, err := model.GetAssetsHistory(page) if nil != err { ret.Code = -1 ret.Msg = err.Error() diff --git a/kernel/model/history.go b/kernel/model/history.go index f108b48fd..f52a9e4a2 100644 --- a/kernel/model/history.go +++ b/kernel/model/history.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -276,7 +275,22 @@ func GetDocHistory(boxID string, page int) (ret []*History, err error) { return } -const maxHistory = 32 +func FullTextSearchHistory(query string, page int) (ret []*History) { + query = gulu.Str.RemoveInvisible(query) + query = stringQuery(query) + + pageSize := 32 + from := (page - 1) * pageSize + to := page * pageSize + + table := "histories_fts_case_insensitive" + projections := "type, op, title, content, path, created" + stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '{title content}:(" + query + ")'" + stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(from) + ", " + strconv.Itoa(to) + sqlHistories := sql.SelectHistoriesRawStmt(stmt) + ret = fromSQLHistories(sqlHistories) + return +} func GetNotebookHistory() (ret []*History, err error) { ret = []*History{} @@ -297,7 +311,6 @@ func GetNotebookHistory() (ret []*History, err error) { return iTimeDir > jTimeDir }) - historyCount := 0 for _, historyNotebookConf := range historyNotebookConfs { timeDir := filepath.Base(filepath.Dir(filepath.Dir(filepath.Dir(historyNotebookConf)))) t := timeDir[:strings.LastIndex(timeDir, "-")] @@ -318,18 +331,11 @@ func GetNotebookHistory() (ret []*History, err error) { ret = append(ret, &History{ HCreated: t, - Items: []*HistoryItem{ - { - Title: c.Name, - Path: filepath.Dir(filepath.Dir(historyNotebookConf)), - }, - }, + Items: []*HistoryItem{{ + Title: c.Name, + Path: filepath.Dir(filepath.Dir(historyNotebookConf)), + }}, }) - - historyCount++ - if maxHistory <= historyCount { - break - } } sort.Slice(ret, func(i, j int) bool { @@ -338,73 +344,17 @@ func GetNotebookHistory() (ret []*History, err error) { return } -func GetAssetsHistory() (ret []*History, err error) { - ret = []*History{} +func GetAssetsHistory(page int) (ret []*History, err error) { + pageSize := 32 + from := (page - 1) * pageSize + to := page * pageSize - historyDir := util.HistoryDir - if !gulu.File.IsDir(historyDir) { - return - } - - historyAssetsDirs, err := filepath.Glob(historyDir + "/*/assets") - if nil != err { - logging.LogErrorf("read dir [%s] failed: %s", historyDir, err) - return - } - sort.Slice(historyAssetsDirs, func(i, j int) bool { - return historyAssetsDirs[i] > historyAssetsDirs[j] - }) - - historyCount := 0 - for _, historyAssetsDir := range historyAssetsDirs { - var assets []*HistoryItem - itemCount := 0 - filepath.Walk(historyAssetsDir, func(path string, info fs.FileInfo, err error) error { - if isSkipFile(info.Name()) { - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - if info.IsDir() { - return nil - } - - assets = append(assets, &HistoryItem{ - Title: info.Name(), - Path: filepath.ToSlash(strings.TrimPrefix(path, util.WorkspaceDir)), - }) - itemCount++ - if maxHistory < itemCount { - return io.EOF - } - return nil - }) - - if 1 > len(assets) { - continue - } - - timeDir := filepath.Base(filepath.Dir(historyAssetsDir)) - t := timeDir[:strings.LastIndex(timeDir, "-")] - if ti, parseErr := time.Parse("2006-01-02-150405", t); nil == parseErr { - t = ti.Format("2006-01-02 15:04:05") - } - - ret = append(ret, &History{ - HCreated: t, - Items: assets, - }) - - historyCount++ - if maxHistory <= historyCount { - break - } - } - - sort.Slice(ret, func(i, j int) bool { - return ret[i].HCreated > ret[j].HCreated - }) + table := "histories_fts_case_insensitive" + projections := "type, op, title, content, path, created" + stmt := "SELECT " + projections + " FROM " + table + " WHERE path LIKE '%/assets/%'" + stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(from) + ", " + strconv.Itoa(to) + sqlHistories := sql.SelectHistoriesRawStmt(stmt) + ret = fromSQLHistories(sqlHistories) return } @@ -529,6 +479,7 @@ const ( HistoryOpUpdate = "update" HistoryOpDelete = "delete" HistoryOpFormat = "format" + HistoryOpSync = "sync" ) func GetHistoryDir(suffix string) (ret string, err error) { @@ -562,7 +513,7 @@ func ReindexHistory() (err error) { return } -var validOps = []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat} +var validOps = []string{HistoryOpClean, HistoryOpUpdate, HistoryOpDelete, HistoryOpFormat, HistoryOpSync} func indexHistoryDir(name string, luteEngine *lute.Lute) (err error) { op := name[strings.LastIndex(name, "-")+1:] @@ -583,7 +534,7 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) (err error) { filepath.Walk(entryPath, func(path string, info os.FileInfo, err error) error { if strings.HasSuffix(info.Name(), ".sy") { docs = append(docs, path) - } else if strings.Contains(path, "assets/") { + } else if strings.Contains(path, "assets"+string(os.PathSeparator)) { assets = append(assets, path) } return nil @@ -643,23 +594,6 @@ func indexHistoryDir(name string, luteEngine *lute.Lute) (err error) { return } -func FullTextSearchHistory(query string, page int) (ret []*History) { - query = gulu.Str.RemoveInvisible(query) - query = stringQuery(query) - - pageSize := 32 - from := (page - 1) * pageSize - to := page * pageSize - - table := "histories_fts_case_insensitive" - projections := "type, op, title, content, path, created" - stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '{title content}:(" + query + ")'" - stmt += " ORDER BY created DESC LIMIT " + strconv.Itoa(from) + ", " + strconv.Itoa(to) - sqlHistories := sql.SelectHistoriesRawStmt(stmt) - ret = fromSQLHistories(sqlHistories) - return -} - func fromSQLHistories(sqlHistories []*sql.History) (ret []*History) { if 1 > len(sqlHistories) { ret = []*History{} @@ -690,5 +624,11 @@ func fromSQLHistories(sqlHistories []*sql.History) (ret []*History) { }) } } + if 1 < len(items) { + ret = append(ret, &History{ + HCreated: time.Unix(tmpTime, 0).Format("2006-01-02 15:04:05"), + Items: items, + }) + } return }