From c96e815aa675222996070cd2f764c0f4d1b097e4 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 11 Aug 2023 10:28:32 +0800 Subject: [PATCH 1/2] :sparkles: Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874 --- kernel/api/router.go | 1 + kernel/api/search.go | 68 +++++++++++++++++++++++++++++++++-- kernel/model/asset_content.go | 36 +++++++++---------- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/kernel/api/router.go b/kernel/api/router.go index 254da533d..5f1ad1f2b 100644 --- a/kernel/api/router.go +++ b/kernel/api/router.go @@ -144,6 +144,7 @@ func ServeAPI(ginServer *gin.Engine) { ginServer.Handle("POST", "/api/search/fullTextSearchBlock", model.CheckAuth, fullTextSearchBlock) ginServer.Handle("POST", "/api/search/searchAsset", model.CheckAuth, searchAsset) ginServer.Handle("POST", "/api/search/findReplace", model.CheckAuth, model.CheckReadonly, findReplace) + ginServer.Handle("POST", "/api/search/fullTextSearchAssetContent", model.CheckAuth, fullTextSearchAssetContent) ginServer.Handle("POST", "/api/block/getBlockInfo", model.CheckAuth, getBlockInfo) ginServer.Handle("POST", "/api/block/getBlockDOM", model.CheckAuth, getBlockDOM) diff --git a/kernel/api/search.go b/kernel/api/search.go index 2db996a5b..65f080c41 100644 --- a/kernel/api/search.go +++ b/kernel/api/search.go @@ -26,6 +26,24 @@ import ( "github.com/siyuan-note/siyuan/kernel/util" ) +func fullTextSearchAssetContent(c *gin.Context) { + ret := gulu.Ret.NewResult() + defer c.JSON(http.StatusOK, ret) + + arg, ok := util.JsonArg(c, ret) + if !ok { + return + } + + page, pageSize, query, types, method, orderBy := parseSearchAssetContentArgs(arg) + assetContents, matchedAssetCount, pageCount := model.FullTextSearchAssetContent(query, types, method, orderBy, page, pageSize) + ret.Data = map[string]interface{}{ + "assetContents": assetContents, + "matchedAssetCount": matchedAssetCount, + "pageCount": pageCount, + } +} + func findReplace(c *gin.Context) { ret := gulu.Ret.NewResult() defer c.JSON(http.StatusOK, ret) @@ -35,7 +53,7 @@ func findReplace(c *gin.Context) { return } - _, _, _, paths, boxes, types, method, orderBy, groupBy := parseSearchArgs(arg) + _, _, _, paths, boxes, types, method, orderBy, groupBy := parseSearchBlockArgs(arg) k := arg["k"].(string) r := arg["r"].(string) @@ -215,7 +233,7 @@ func fullTextSearchBlock(c *gin.Context) { return } - page, pageSize, query, paths, boxes, types, method, orderBy, groupBy := parseSearchArgs(arg) + page, pageSize, query, paths, boxes, types, method, orderBy, groupBy := parseSearchBlockArgs(arg) blocks, matchedBlockCount, matchedRootCount, pageCount := model.FullTextSearchBlock(query, boxes, paths, types, method, orderBy, groupBy, page, pageSize) ret.Data = map[string]interface{}{ "blocks": blocks, @@ -225,7 +243,7 @@ func fullTextSearchBlock(c *gin.Context) { } } -func parseSearchArgs(arg map[string]interface{}) (page, pageSize int, query string, paths, boxes []string, types map[string]bool, method, orderBy, groupBy int) { +func parseSearchBlockArgs(arg map[string]interface{}) (page, pageSize int, query string, paths, boxes []string, types map[string]bool, method, orderBy, groupBy int) { page = 1 if nil != arg["page"] { page = int(arg["page"].(float64)) @@ -291,3 +309,47 @@ func parseSearchArgs(arg map[string]interface{}) (page, pageSize int, query stri } return } + +func parseSearchAssetContentArgs(arg map[string]interface{}) (page, pageSize int, query string, types map[string]bool, method, orderBy int) { + page = 1 + if nil != arg["page"] { + page = int(arg["page"].(float64)) + } + if 0 >= page { + page = 1 + } + + pageSize = 32 + if nil != arg["pageSize"] { + pageSize = int(arg["pageSize"].(float64)) + } + if 0 >= pageSize { + pageSize = 32 + } + + queryArg := arg["query"] + if nil != queryArg { + query = queryArg.(string) + } + + if nil != arg["types"] { + typesArg := arg["types"].(map[string]interface{}) + types = map[string]bool{} + for t, b := range typesArg { + types[t] = b.(bool) + } + } + + // method：0：关键字，1：查询语法，2：SQL，3：正则表达式 + methodArg := arg["method"] + if nil != methodArg { + method = int(methodArg.(float64)) + } + + // orderBy：0：相关度（默认），1：按更新时间升序，2：按更新时间降序 + orderByArg := arg["orderBy"] + if nil != orderByArg { + orderBy = int(orderByArg.(float64)) + } + return +} diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.go index 739c91a98..def9a75c2 100644 --- a/kernel/model/asset_content.go +++ b/kernel/model/asset_content.go @@ -51,24 +51,24 @@ type AssetContent struct { // // method：0：关键字，1：查询语法，2：SQL，3：正则表达式 // orderBy: 0：相关度（默认），1：按更新时间升序，2：按更新时间降序 -func FullTextSearchAssetContent(query string, types map[string]bool, method, orderBy, page, pageSize int) (ret []*AssetContent, matchedAssetsCount, pageCount int) { +func FullTextSearchAssetContent(query string, types map[string]bool, method, orderBy, page, pageSize int) (ret []*AssetContent, matchedAssetCount, pageCount int) { query = strings.TrimSpace(query) beforeLen := 36 orderByClause := buildAssetContentOrderBy(orderBy) switch method { case 1: // 查询语法 filter := buildAssetContentTypeFilter(types) - ret, matchedAssetsCount = fullTextSearchAssetContentByQuerySyntax(query, filter, orderByClause, beforeLen, page, pageSize) + ret, matchedAssetCount = fullTextSearchAssetContentByQuerySyntax(query, filter, orderByClause, beforeLen, page, pageSize) case 2: // SQL - ret, matchedAssetsCount = searchAssetContentBySQL(query, beforeLen, page, pageSize) + ret, matchedAssetCount = searchAssetContentBySQL(query, beforeLen, page, pageSize) case 3: // 正则表达式 typeFilter := buildAssetContentTypeFilter(types) - ret, matchedAssetsCount = fullTextSearchAssetContentByRegexp(query, typeFilter, orderByClause, beforeLen, page, pageSize) + ret, matchedAssetCount = fullTextSearchAssetContentByRegexp(query, typeFilter, orderByClause, beforeLen, page, pageSize) default: // 关键字 filter := buildAssetContentTypeFilter(types) - ret, matchedAssetsCount = fullTextSearchAssetContentByKeyword(query, filter, orderByClause, beforeLen, page, pageSize) + ret, matchedAssetCount = fullTextSearchAssetContentByKeyword(query, filter, orderByClause, beforeLen, page, pageSize) } - pageCount = (matchedAssetsCount + pageSize - 1) / pageSize + pageCount = (matchedAssetCount + pageSize - 1) / pageSize if 1 > len(ret) { ret = []*AssetContent{} @@ -76,18 +76,18 @@ func FullTextSearchAssetContent(query string, types map[string]bool, method, ord return } -func fullTextSearchAssetContentByQuerySyntax(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) { +func fullTextSearchAssetContentByQuerySyntax(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) { query = gulu.Str.RemoveInvisible(query) return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize) } -func fullTextSearchAssetContentByKeyword(query, typeFilter string, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) { +func fullTextSearchAssetContentByKeyword(query, typeFilter string, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) { query = gulu.Str.RemoveInvisible(query) query = stringQuery(query) return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize) } -func fullTextSearchAssetContentByRegexp(exp, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) { +func fullTextSearchAssetContentByRegexp(exp, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) { exp = gulu.Str.RemoveInvisible(exp) fieldFilter := assetContentFieldRegexp(exp) stmt := "SELECT * FROM `asset_contents_fts_case_insensitive` WHERE " + fieldFilter + " AND ext IN " + typeFilter @@ -99,7 +99,7 @@ func fullTextSearchAssetContentByRegexp(exp, typeFilter, orderBy string, beforeL ret = []*AssetContent{} } - matchedAssetsCount = fullTextSearchAssetContentCountByRegexp(exp, typeFilter) + matchedAssetCount = fullTextSearchAssetContentCountByRegexp(exp, typeFilter) return } @@ -113,7 +113,7 @@ func assetContentFieldRegexp(exp string) string { return buf.String() } -func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAssetsCount int) { +func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAssetCount int) { table := "asset_contents_fts_case_insensitive" fieldFilter := fieldRegexp(exp) stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE " + fieldFilter + " AND type IN " + typeFilter @@ -121,11 +121,11 @@ func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAss if 1 > len(result) { return } - matchedAssetsCount = int(result[0]["assets"].(int64)) + matchedAssetCount = int(result[0]["assets"].(int64)) return } -func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) { +func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) { table := "asset_contents_fts_case_insensitive" projections := "id, name, ext, path, size, updated, " + "highlight(" + table + ", 6, '', '') AS content" @@ -139,11 +139,11 @@ func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLe ret = []*AssetContent{} } - matchedAssetsCount = fullTextSearchAssetContentCount(query, typeFilter) + matchedAssetCount = fullTextSearchAssetContentCount(query, typeFilter) return } -func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) { +func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) { stmt = gulu.Str.RemoveInvisible(stmt) stmt = strings.TrimSpace(stmt) assetContents := sql.SelectAssetContentsRawStmt(stmt, page, pageSize) @@ -161,11 +161,11 @@ func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret [] return } - matchedAssetsCount = int(result[0]["assets"].(int64)) + matchedAssetCount = int(result[0]["assets"].(int64)) return } -func fullTextSearchAssetContentCount(query, typeFilter string) (matchedAssetsCount int) { +func fullTextSearchAssetContentCount(query, typeFilter string) (matchedAssetCount int) { query = gulu.Str.RemoveInvisible(query) table := "asset_contents_fts_case_insensitive" @@ -175,7 +175,7 @@ func fullTextSearchAssetContentCount(query, typeFilter string) (matchedAssetsCou if 1 > len(result) { return } - matchedAssetsCount = int(result[0]["assets"].(int64)) + matchedAssetCount = int(result[0]["assets"].(int64)) return } From 1d3e58101bcd7a241e70b3b19af306610ceaa3b9 Mon Sep 17 00:00:00 2001 From: Daniel <845765@qq.com> Date: Fri, 11 Aug 2023 11:04:47 +0800 Subject: [PATCH 2/2] :sparkles: Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874 --- kernel/model/asset_content.go | 12 ++++---- kernel/sql/asset_content_query.go | 48 +++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.go index def9a75c2..61c28f980 100644 --- a/kernel/model/asset_content.go +++ b/kernel/model/asset_content.go @@ -116,8 +116,8 @@ func assetContentFieldRegexp(exp string) string { func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAssetCount int) { table := "asset_contents_fts_case_insensitive" fieldFilter := fieldRegexp(exp) - stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE " + fieldFilter + " AND type IN " + typeFilter - result, _ := sql.QueryNoLimit(stmt) + stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE " + fieldFilter + " AND ext IN " + typeFilter + result, _ := sql.QueryAssetContentNoLimit(stmt) if 1 > len(result) { return } @@ -130,7 +130,7 @@ func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLe projections := "id, name, ext, path, size, updated, " + "highlight(" + table + ", 6, '', '') AS content" stmt := "SELECT " + projections + " FROM " + table + " WHERE (`" + table + "` MATCH '" + buildAssetContentColumnFilter() + ":(" + query + ")'" - stmt += ") AND type IN " + typeFilter + stmt += ") AND ext IN " + typeFilter stmt += " " + orderBy stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize) assetContents := sql.SelectAssetContentsRawStmt(stmt, page, pageSize) @@ -156,7 +156,7 @@ func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret [] stmt = strings.ToLower(stmt) stmt = strings.ReplaceAll(stmt, "select * ", "select COUNT(path) AS `assets` ") stmt = removeLimitClause(stmt) - result, _ := sql.QueryNoLimit(stmt) + result, _ := sql.QueryAssetContentNoLimit(stmt) if 1 > len(ret) { return } @@ -170,8 +170,8 @@ func fullTextSearchAssetContentCount(query, typeFilter string) (matchedAssetCoun table := "asset_contents_fts_case_insensitive" stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE (`" + table + "` MATCH '" + buildAssetContentColumnFilter() + ":(" + query + ")'" - stmt += ") AND type IN " + typeFilter - result, _ := sql.QueryNoLimit(stmt) + stmt += ") AND ext IN " + typeFilter + result, _ := sql.QueryAssetContentNoLimit(stmt) if 1 > len(result) { return } diff --git a/kernel/sql/asset_content_query.go b/kernel/sql/asset_content_query.go index f4d6d1477..4f25dbbd6 100644 --- a/kernel/sql/asset_content_query.go +++ b/kernel/sql/asset_content_query.go @@ -19,6 +19,7 @@ package sql import ( "database/sql" "errors" + "math" "strconv" "strings" @@ -26,6 +27,53 @@ import ( "github.com/siyuan-note/logging" ) +func QueryAssetContentNoLimit(stmt string) (ret []map[string]interface{}, err error) { + return queryAssetContentRawStmt(stmt, math.MaxInt) +} + +func queryAssetContentRawStmt(stmt string, limit int) (ret []map[string]interface{}, err error) { + rows, err := queryAssetContent(stmt) + if nil != err { + if strings.Contains(err.Error(), "syntax error") { + return + } + return + } + defer rows.Close() + + cols, err := rows.Columns() + if nil != err || nil == cols { + return + } + + noLimit := !strings.Contains(strings.ToLower(stmt), " limit ") + var count, errCount int + for rows.Next() { + columns := make([]interface{}, len(cols)) + columnPointers := make([]interface{}, len(cols)) + for i := range columns { + columnPointers[i] = &columns[i] + } + + if err = rows.Scan(columnPointers...); nil != err { + return + } + + m := make(map[string]interface{}) + for i, colName := range cols { + val := columnPointers[i].(*interface{}) + m[colName] = *val + } + + ret = append(ret, m) + count++ + if (noLimit && limit < count) || 0 < errCount { + break + } + } + return +} + func SelectAssetContentsRawStmt(stmt string, page, limit int) (ret []*AssetContent) { parsedStmt, err := sqlparser.Parse(stmt) if nil != err {