Andreas/siyuan
1
0
Fork 0
mirror of https://github.com/siyuan-note/siyuan.git synced 2025-12-16 22:50:13 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

✨ Support for searching asset content https://github.com/siyuan-note/siyuan/issues/8874

Browse source
This commit is contained in:
Daniel 2023-08-11 10:19:11 +08:00
parent d10431d27a
commit 298b95dea3
No known key found for this signature in database
GPG key ID: 86211BA83DF03017
2 changed files with 66 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

76
kernel/model/asset_content.go
View file

@ -37,57 +37,66 @@ import (
"github.com/xuri/excelize/v2" "github.com/xuri/excelize/v2"
) )
type AssetContent struct {
ID string `json:"id"`
Name string `json:"name"`
Ext string `json:"ext"`
Path string `json:"path"`
Size int64 `json:"size"`
Updated int64 `json:"updated"`
Content string `json:"content"`
}
// FullTextSearchAssetContent 搜索资源文件内容。 // FullTextSearchAssetContent 搜索资源文件内容。
// //
// method：0：关键字，1：查询语法，2：SQL，3：正则表达式 // method：0：关键字，1：查询语法，2：SQL，3：正则表达式
// orderBy: 0：相关度（默认），1：按更新时间升序，2：按更新时间降序 // orderBy: 0：相关度（默认），1：按更新时间升序，2：按更新时间降序
func FullTextSearchAssetContent(query string, types map[string]bool, method, orderBy, page, pageSize int) (ret []*Block, matchedBlockCount, matchedRootCount, pageCount int) { func FullTextSearchAssetContent(query string, types map[string]bool, method, orderBy, page, pageSize int) (ret []*AssetContent, matchedAssetsCount, pageCount int) {
query = strings.TrimSpace(query) query = strings.TrimSpace(query)
beforeLen := 36 beforeLen := 36
var blocks []*Block
orderByClause := buildAssetContentOrderBy(orderBy) orderByClause := buildAssetContentOrderBy(orderBy)
switch method { switch method {
case 1: // 查询语法 case 1: // 查询语法
filter := buildAssetContentTypeFilter(types) filter := buildAssetContentTypeFilter(types)
blocks, matchedRootCount = fullTextSearchAssetContentByQuerySyntax(query, filter, orderByClause, beforeLen, page, pageSize) ret, matchedAssetsCount = fullTextSearchAssetContentByQuerySyntax(query, filter, orderByClause, beforeLen, page, pageSize)
case 2: // SQL case 2: // SQL
blocks, matchedRootCount = searchAssetContentBySQL(query, beforeLen, page, pageSize) ret, matchedAssetsCount = searchAssetContentBySQL(query, beforeLen, page, pageSize)
case 3: // 正则表达式 case 3: // 正则表达式
typeFilter := buildAssetContentTypeFilter(types) typeFilter := buildAssetContentTypeFilter(types)
blocks, matchedRootCount = fullTextSearchAssetContentByRegexp(query, typeFilter, orderByClause, beforeLen, page, pageSize) ret, matchedAssetsCount = fullTextSearchAssetContentByRegexp(query, typeFilter, orderByClause, beforeLen, page, pageSize)
default: // 关键字 default: // 关键字
filter := buildAssetContentTypeFilter(types) filter := buildAssetContentTypeFilter(types)
blocks, matchedRootCount = fullTextSearchAssetContentByKeyword(query, filter, orderByClause, beforeLen, page, pageSize) ret, matchedAssetsCount = fullTextSearchAssetContentByKeyword(query, filter, orderByClause, beforeLen, page, pageSize)
} }
pageCount = (matchedRootCount + pageSize - 1) / pageSize pageCount = (matchedAssetsCount + pageSize - 1) / pageSize
if 1 > len(ret) { if 1 > len(ret) {
ret = []*Block{} ret = []*AssetContent{}
} }
return return
} }
func fullTextSearchAssetContentByQuerySyntax(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedAssetsCount int) { func fullTextSearchAssetContentByQuerySyntax(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) {
query = gulu.Str.RemoveInvisible(query) query = gulu.Str.RemoveInvisible(query)
return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize) return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize)
} }
func fullTextSearchAssetContentByKeyword(query, typeFilter string, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedAssetsCount int) { func fullTextSearchAssetContentByKeyword(query, typeFilter string, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) {
query = gulu.Str.RemoveInvisible(query) query = gulu.Str.RemoveInvisible(query)
query = stringQuery(query) query = stringQuery(query)
return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize) return fullTextSearchAssetContentByFTS(query, typeFilter, orderBy, beforeLen, page, pageSize)
} }
func fullTextSearchAssetContentByRegexp(exp, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedAssetsCount int) { func fullTextSearchAssetContentByRegexp(exp, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) {
exp = gulu.Str.RemoveInvisible(exp) exp = gulu.Str.RemoveInvisible(exp)
fieldFilter := assetContentFieldRegexp(exp) fieldFilter := assetContentFieldRegexp(exp)
stmt := "SELECT * FROM `asset_contents_fts_case_insensitive` WHERE " + fieldFilter + " AND ext IN " + typeFilter stmt := "SELECT * FROM `asset_contents_fts_case_insensitive` WHERE " + fieldFilter + " AND ext IN " + typeFilter
stmt += " " + orderBy stmt += " " + orderBy
stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize) stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
blocks := sql.SelectBlocksRawStmtNoParse(stmt, Conf.Search.Limit) assetContents := sql.SelectAssetContentsRawStmtNoParse(stmt, Conf.Search.Limit)
ret = fromSQLBlocks(&blocks, "", beforeLen) ret = fromSQLAssetContents(&assetContents, beforeLen)
if 1 > len(ret) { if 1 > len(ret) {
ret = []*Block{} ret = []*AssetContent{}
} }
matchedAssetsCount = fullTextSearchAssetContentCountByRegexp(exp, typeFilter) matchedAssetsCount = fullTextSearchAssetContentCountByRegexp(exp, typeFilter)
@ -105,8 +114,9 @@ func assetContentFieldRegexp(exp string) string {
} }
func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAssetsCount int) { func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAssetsCount int) {
table := "asset_contents_fts_case_insensitive"
fieldFilter := fieldRegexp(exp) fieldFilter := fieldRegexp(exp)
stmt := "SELECT COUNT(path) AS `assets` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE " + fieldFilter + " AND type IN " + typeFilter
result, _ := sql.QueryNoLimit(stmt) result, _ := sql.QueryNoLimit(stmt)
if 1 > len(result) { if 1 > len(result) {
return return
@ -115,7 +125,7 @@ func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string) (matchedAss
return return
} }
func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*Block, matchedAssetsCount int) { func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) {
table := "asset_contents_fts_case_insensitive" table := "asset_contents_fts_case_insensitive"
projections := "id, name, ext, path, size, updated, " + projections := "id, name, ext, path, size, updated, " +
"highlight(" + table + ", 6, '<mark>', '</mark>') AS content" "highlight(" + table + ", 6, '<mark>', '</mark>') AS content"
@ -123,23 +133,23 @@ func fullTextSearchAssetContentByFTS(query, typeFilter, orderBy string, beforeLe
stmt += ") AND type IN " + typeFilter stmt += ") AND type IN " + typeFilter
stmt += " " + orderBy stmt += " " + orderBy
stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize) stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
blocks := sql.SelectBlocksRawStmt(stmt, page, pageSize) assetContents := sql.SelectAssetContentsRawStmt(stmt, page, pageSize)
ret = fromSQLBlocks(&blocks, "", beforeLen) ret = fromSQLAssetContents(&assetContents, beforeLen)
if 1 > len(ret) { if 1 > len(ret) {
ret = []*Block{} ret = []*AssetContent{}
} }
matchedAssetsCount = fullTextSearchAssetContentCount(query, typeFilter) matchedAssetsCount = fullTextSearchAssetContentCount(query, typeFilter)
return return
} }
func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret []*Block, matchedAssetsCount int) { func searchAssetContentBySQL(stmt string, beforeLen, page, pageSize int) (ret []*AssetContent, matchedAssetsCount int) {
stmt = gulu.Str.RemoveInvisible(stmt) stmt = gulu.Str.RemoveInvisible(stmt)
stmt = strings.TrimSpace(stmt) stmt = strings.TrimSpace(stmt)
blocks := sql.SelectBlocksRawStmt(stmt, page, pageSize) assetContents := sql.SelectAssetContentsRawStmt(stmt, page, pageSize)
ret = fromSQLBlocks(&blocks, "", beforeLen) ret = fromSQLAssetContents(&assetContents, beforeLen)
if 1 > len(ret) { if 1 > len(ret) {
ret = []*Block{} ret = []*AssetContent{}
return return
} }
@ -169,6 +179,26 @@ func fullTextSearchAssetContentCount(query, typeFilter string) (matchedAssetsCou
return return
} }
func fromSQLAssetContents(assetContents *[]*sql.AssetContent, beforeLen int) (ret []*AssetContent) {
ret = []*AssetContent{}
for _, assetContent := range *assetContents {
ret = append(ret, fromSQLAssetContent(assetContent, beforeLen))
}
return
}
func fromSQLAssetContent(assetContent *sql.AssetContent, beforeLen int) *AssetContent {
return &AssetContent{
ID: assetContent.ID,
Name: assetContent.Name,
Ext: assetContent.Ext,
Path: assetContent.Path,
Size: assetContent.Size,
Updated: assetContent.Updated,
Content: assetContent.Content,
}
}
func buildAssetContentColumnFilter() string { func buildAssetContentColumnFilter() string {
return "{name content}" return "{name content}"
} }

23
kernel/sql/asset_content_query.go
View file

@ -26,7 +26,7 @@ import (
"github.com/siyuan-note/logging" "github.com/siyuan-note/logging"
) )
func SelectAssetContentsRawStmt(stmt string, page, limit int) (ret []*Block) { func SelectAssetContentsRawStmt(stmt string, page, limit int) (ret []*AssetContent) {
parsedStmt, err := sqlparser.Parse(stmt) parsedStmt, err := sqlparser.Parse(stmt)
if nil != err { if nil != err {
return selectAssetContentsRawStmt(stmt, limit) return selectAssetContentsRawStmt(stmt, limit)
@ -83,14 +83,18 @@ func SelectAssetContentsRawStmt(stmt string, page, limit int) (ret []*Block) {
} }
defer rows.Close() defer rows.Close()
for rows.Next() { for rows.Next() {
if block := scanAssetContentRows(rows); nil != block { if ac := scanAssetContentRows(rows); nil != ac {
ret = append(ret, block) ret = append(ret, ac)
} }
} }
return return
} }
func selectAssetContentsRawStmt(stmt string, limit int) (ret []*Block) { func SelectAssetContentsRawStmtNoParse(stmt string, limit int) (ret []*AssetContent) {
return selectAssetContentsRawStmt(stmt, limit)
}
func selectAssetContentsRawStmt(stmt string, limit int) (ret []*AssetContent) {
rows, err := queryAssetContent(stmt) rows, err := queryAssetContent(stmt)
if nil != err { if nil != err {
if strings.Contains(err.Error(), "syntax error") { if strings.Contains(err.Error(), "syntax error") {
@ -104,8 +108,8 @@ func selectAssetContentsRawStmt(stmt string, limit int) (ret []*Block) {
var count, errCount int var count, errCount int
for rows.Next() { for rows.Next() {
count++ count++
if block := scanAssetContentRows(rows); nil != block { if ac := scanAssetContentRows(rows); nil != ac {
ret = append(ret, block) ret = append(ret, ac)
} else { } else {
logging.LogWarnf("raw sql query [%s] failed", stmt) logging.LogWarnf("raw sql query [%s] failed", stmt)
errCount++ errCount++
@ -119,13 +123,12 @@ func selectAssetContentsRawStmt(stmt string, limit int) (ret []*Block) {
} }
func scanAssetContentRows(rows *sql.Rows) (ret *AssetContent) { func scanAssetContentRows(rows *sql.Rows) (ret *AssetContent) {
var block Block var ac AssetContent
if err := rows.Scan(&block.ID, &block.ParentID, &block.RootID, &block.Hash, &block.Box, &block.Path, &block.HPath, &block.Name, &block.Alias, &block.Memo, &block.Tag, &block.Content, &block.FContent, &block.Markdown, &block.Length, &block.Type, &block.SubType, &block.IAL, &block.Sort, &block.Created, &block.Updated); nil != err { if err := rows.Scan(&ac.ID, &ac.Name, &ac.Ext, &ac.Path, &ac.Size, &ac.Updated, &ac.Content); nil != err {
logging.LogErrorf("query scan field failed: %s\n%s", err, logging.ShortStack()) logging.LogErrorf("query scan field failed: %s\n%s", err, logging.ShortStack())
return return
} }
ret = &block ret = &ac
putBlockCache(ret)
return return
} }