diff --git a/kernel/api/search.go b/kernel/api/search.go index c5b342ac7..c71298faf 100644 --- a/kernel/api/search.go +++ b/kernel/api/search.go @@ -220,12 +220,17 @@ func fullTextSearchBlock(c *gin.Context) { if nil != methodArg { method = int(methodArg.(float64)) } + orderByArg := arg["orderBy"] + var orderBy int // 0:按块类型(默认),1:按创建时间升序,2:按创建时间降序,3:按更新时间升序,4:按更新时间降序,5:按内容顺序(仅在按文档分组时) + if nil != orderByArg { + orderBy = int(orderByArg.(float64)) + } groupByArg := arg["groupBy"] var groupBy int // 0:不分组,1:按文档分组 if nil != groupByArg { groupBy = int(groupByArg.(float64)) } - blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, groupBy) + blocks, matchedBlockCount, matchedRootCount := model.FullTextSearchBlock(query, boxes, paths, types, method, orderBy, groupBy) ret.Data = map[string]interface{}{ "blocks": blocks, "matchedBlockCount": matchedBlockCount, diff --git a/kernel/model/graph.go b/kernel/model/graph.go index 82ff0b45a..34465259b 100644 --- a/kernel/model/graph.go +++ b/kernel/model/graph.go @@ -18,6 +18,8 @@ package model import ( "bytes" + "github.com/88250/lute/parse" + "github.com/siyuan-note/siyuan/kernel/util" "math" "strings" "unicode/utf8" @@ -685,3 +687,65 @@ func nodeTitleLabel(node *GraphNode, blockContent string) { node.Label = blockContent } } + +func query2Stmt(queryStr string) (ret string) { + buf := bytes.Buffer{} + if util.IsIDPattern(queryStr) { + buf.WriteString("id = '" + queryStr + "'") + } else { + var tags []string + luteEngine := NewLute() + t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) + ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { + if !entering { + return ast.WalkContinue + } + if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) { + tags = append(tags, n.Text()) + } + return ast.WalkContinue + }) + + for _, tag := range tags { + queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "") + } + parts := strings.Split(queryStr, " ") + + for i, part := range parts { + if "" == part { + continue + } + part = strings.ReplaceAll(part, "'", "''") + buf.WriteString("(content LIKE '%" + part + "%'") + buf.WriteString(Conf.Search.NAMFilter(part)) + buf.WriteString(")") + if i < len(parts)-1 { + buf.WriteString(" AND ") + } + } + + if 0 < len(tags) { + if 0 < buf.Len() { + buf.WriteString(" OR ") + } + for i, tag := range tags { + buf.WriteString("(content LIKE '%#" + tag + "#%')") + if i < len(tags)-1 { + buf.WriteString(" AND ") + } + } + buf.WriteString(" OR ") + for i, tag := range tags { + buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'") + if i < len(tags)-1 { + buf.WriteString(" AND ") + } + } + } + } + if 1 > buf.Len() { + buf.WriteString("1=1") + } + ret = buf.String() + return +} diff --git a/kernel/model/search.go b/kernel/model/search.go index f60132beb..36d91ca28 100644 --- a/kernel/model/search.go +++ b/kernel/model/search.go @@ -312,31 +312,32 @@ func FindReplace(keyword, replacement string, ids []string, method int) (err err return } -func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method int, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) { +func FullTextSearchBlock(query string, boxes, paths []string, types map[string]bool, method, orderBy, groupBy int) (ret []*Block, matchedBlockCount, matchedRootCount int) { // method:0:文本,1:查询语法,2:SQL,3:正则表达式 + // orderBy: 0:按块类型(默认),1:按创建时间升序,2:按创建时间降序,3:按更新时间升序,4:按更新时间降序,5:按内容顺序(仅在按文档分组时) // groupBy:0:不分组,1:按文档分组 query = strings.TrimSpace(query) beforeLen := 36 var blocks []*Block - + orderByClause := buildOrderBy(orderBy) switch method { case 1: // 查询语法 filter := buildTypeFilter(types) boxFilter := buildBoxesFilter(boxes) pathFilter := buildPathsFilter(paths) - blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, true) + blocks, matchedBlockCount, matchedRootCount = fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, filter, orderByClause, beforeLen) case 2: // SQL blocks, matchedBlockCount, matchedRootCount = searchBySQL(query, beforeLen) case 3: // 正则表达式 typeFilter := buildTypeFilter(types) boxFilter := buildBoxesFilter(boxes) pathFilter := buildPathsFilter(paths) - blocks, matchedBlockCount, matchedRootCount = fullTextSearchByRegexp(query, boxFilter, pathFilter, typeFilter, beforeLen) + blocks, matchedBlockCount, matchedRootCount = fullTextSearchByRegexp(query, boxFilter, pathFilter, typeFilter, orderByClause, beforeLen) default: // 文本 filter := buildTypeFilter(types) boxFilter := buildBoxesFilter(boxes) pathFilter := buildPathsFilter(paths) - blocks, matchedBlockCount, matchedRootCount = fullTextSearch(query, boxFilter, pathFilter, filter, beforeLen, false) + blocks, matchedBlockCount, matchedRootCount = fullTextSearchByKeyword(query, boxFilter, pathFilter, filter, orderByClause, beforeLen) } switch groupBy { @@ -421,6 +422,21 @@ func buildPathsFilter(paths []string) string { return builder.String() } +func buildOrderBy(orderBy int) string { + switch orderBy { + case 1: + return "ORDER BY created ASC" + case 2: + return "ORDER BY created DESC" + case 3: + return "ORDER BY updated ASC" + case 4: + return "ORDER BY updated DESC" + default: + return "ORDER BY sort ASC" + } +} + func buildTypeFilter(types map[string]bool) string { s := conf.NewSearch() if err := copier.Copy(s, Conf.Search); nil != err { @@ -523,6 +539,85 @@ func fullTextSearchRefBlock(keyword string, beforeLen int) (ret []*Block) { return } +func fullTextSearchByQuerySyntax(query, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + query = gulu.Str.RemoveInvisible(query) + if util.IsIDPattern(query) { + ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) + return + } + return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy, beforeLen) +} + +func fullTextSearchByKeyword(query, boxFilter, pathFilter, typeFilter string, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + query = gulu.Str.RemoveInvisible(query) + if util.IsIDPattern(query) { + ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) + return + } + query = stringQuery(query) + return fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy, beforeLen) +} + +func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + exp = gulu.Str.RemoveInvisible(exp) + exp = regexp.QuoteMeta(exp) + + fieldFilter := fieldRegexp(exp) + stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter + stmt += boxFilter + pathFilter + stmt += " " + orderBy + stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) + blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) + ret = fromSQLBlocks(&blocks, "", beforeLen) + if 1 > len(ret) { + ret = []*Block{} + } + + matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter) + return +} + +func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { + fieldFilter := fieldRegexp(exp) + stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter + stmt += boxFilter + pathFilter + stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) + result, _ := sql.Query(stmt) + if 1 > len(result) { + return + } + matchedBlockCount = int(result[0]["matches"].(int64)) + matchedRootCount = int(result[0]["docs"].(int64)) + return +} + +func fullTextSearchByFTS(query, boxFilter, pathFilter, typeFilter, orderBy string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { + table := "blocks_fts" // 大小写敏感 + if !Conf.Search.CaseSensitive { + table = "blocks_fts_case_insensitive" + } + projections := "id, parent_id, root_id, hash, box, path, " + + "highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS hpath, " + + "highlight(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS name, " + + "highlight(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS alias, " + + "highlight(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS memo, " + + "tag, " + + "highlight(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content, " + + "fcontent, markdown, length, type, subtype, ial, sort, created, updated" + stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter + stmt += boxFilter + pathFilter + stmt += " " + orderBy + stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) + blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) + ret = fromSQLBlocks(&blocks, "", beforeLen) + if 1 > len(ret) { + ret = []*Block{} + } + + matchedBlockCount, matchedRootCount = fullTextSearchCount(query, boxFilter, pathFilter, typeFilter) + return +} + func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { query = gulu.Str.RemoveInvisible(query) if util.IsIDPattern(query) { @@ -551,136 +646,6 @@ func fullTextSearchCount(query, boxFilter, pathFilter, typeFilter string) (match return } -func fullTextSearch(query, boxFilter, pathFilter, typeFilter string, beforeLen int, querySyntax bool) (ret []*Block, matchedBlockCount, matchedRootCount int) { - query = gulu.Str.RemoveInvisible(query) - if util.IsIDPattern(query) { - ret, matchedBlockCount, matchedRootCount = searchBySQL("SELECT * FROM `blocks` WHERE `id` = '"+query+"'", beforeLen) - return - } - - if !querySyntax { - query = stringQuery(query) - } - - table := "blocks_fts" // 大小写敏感 - if !Conf.Search.CaseSensitive { - table = "blocks_fts_case_insensitive" - } - projections := "id, parent_id, root_id, hash, box, path, " + - "highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS hpath, " + - "highlight(" + table + ", 7, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS name, " + - "highlight(" + table + ", 8, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS alias, " + - "highlight(" + table + ", 9, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS memo, " + - "tag, " + - "highlight(" + table + ", 11, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content, " + - "fcontent, markdown, length, type, subtype, ial, sort, created, updated" - stmt := "SELECT " + projections + " FROM " + table + " WHERE " + table + " MATCH '" + columnFilter() + ":(" + query + ")' AND type IN " + typeFilter - stmt += boxFilter + pathFilter - stmt += " ORDER BY sort ASC, rank ASC LIMIT " + strconv.Itoa(Conf.Search.Limit) - blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) - ret = fromSQLBlocks(&blocks, "", beforeLen) - if 1 > len(ret) { - ret = []*Block{} - } - - matchedBlockCount, matchedRootCount = fullTextSearchCount(query, boxFilter, pathFilter, typeFilter) - return -} - -func fullTextSearchByRegexp(exp, boxFilter, pathFilter, typeFilter string, beforeLen int) (ret []*Block, matchedBlockCount, matchedRootCount int) { - exp = gulu.Str.RemoveInvisible(exp) - exp = regexp.QuoteMeta(exp) - - fieldFilter := fieldRegexp(exp) - stmt := "SELECT * FROM `blocks` WHERE (" + fieldFilter + ") AND type IN " + typeFilter - stmt += boxFilter + pathFilter - stmt += " ORDER BY sort ASC LIMIT " + strconv.Itoa(Conf.Search.Limit) - blocks := sql.SelectBlocksRawStmt(stmt, Conf.Search.Limit) - ret = fromSQLBlocks(&blocks, "", beforeLen) - if 1 > len(ret) { - ret = []*Block{} - } - - matchedBlockCount, matchedRootCount = fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter) - return -} - -func fullTextSearchCountByRegexp(exp, boxFilter, pathFilter, typeFilter string) (matchedBlockCount, matchedRootCount int) { - fieldFilter := fieldRegexp(exp) - stmt := "SELECT COUNT(id) AS `matches`, COUNT(DISTINCT(root_id)) AS `docs` FROM `blocks` WHERE " + fieldFilter + " AND type IN " + typeFilter - stmt += boxFilter + pathFilter - stmt += " LIMIT " + strconv.Itoa(Conf.Search.Limit) - result, _ := sql.Query(stmt) - if 1 > len(result) { - return - } - matchedBlockCount = int(result[0]["matches"].(int64)) - matchedRootCount = int(result[0]["docs"].(int64)) - return -} - -func query2Stmt(queryStr string) (ret string) { - buf := bytes.Buffer{} - if util.IsIDPattern(queryStr) { - buf.WriteString("id = '" + queryStr + "'") - } else { - var tags []string - luteEngine := NewLute() - t := parse.Inline("", []byte(queryStr), luteEngine.ParseOptions) - ast.Walk(t.Root, func(n *ast.Node, entering bool) ast.WalkStatus { - if !entering { - return ast.WalkContinue - } - if ast.NodeTag == n.Type || (n.IsTextMarkType("tag")) { - tags = append(tags, n.Text()) - } - return ast.WalkContinue - }) - - for _, tag := range tags { - queryStr = strings.ReplaceAll(queryStr, "#"+tag+"#", "") - } - parts := strings.Split(queryStr, " ") - - for i, part := range parts { - if "" == part { - continue - } - part = strings.ReplaceAll(part, "'", "''") - buf.WriteString("(content LIKE '%" + part + "%'") - buf.WriteString(Conf.Search.NAMFilter(part)) - buf.WriteString(")") - if i < len(parts)-1 { - buf.WriteString(" AND ") - } - } - - if 0 < len(tags) { - if 0 < buf.Len() { - buf.WriteString(" OR ") - } - for i, tag := range tags { - buf.WriteString("(content LIKE '%#" + tag + "#%')") - if i < len(tags)-1 { - buf.WriteString(" AND ") - } - } - buf.WriteString(" OR ") - for i, tag := range tags { - buf.WriteString("ial LIKE '%tags=\"%" + tag + "%\"%'") - if i < len(tags)-1 { - buf.WriteString(" AND ") - } - } - } - } - if 1 > buf.Len() { - buf.WriteString("1=1") - } - ret = buf.String() - return -} - func markSearch(text string, keyword string, beforeLen int) (marked string, score float64) { if 0 == len(keyword) { marked = text