diff --git a/kernel/go.mod b/kernel/go.mod index 569940b57..6c0251baa 100644 --- a/kernel/go.mod +++ b/kernel/go.mod @@ -48,9 +48,10 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/radovskyb/watcher v1.0.7 github.com/rqlite/sql v0.0.0-20240102050638-e741e9f54197 + github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sashabaranov/go-openai v1.17.10 github.com/shirou/gopsutil/v3 v3.23.12 - github.com/siyuan-note/dejavu v0.0.0-20240105084039-9867350d60a1 + github.com/siyuan-note/dejavu v0.0.0-20240106123310-d41af94d3e67 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 github.com/siyuan-note/eventbus v0.0.0-20231210025112-82fdedd8e374 github.com/siyuan-note/filelock v0.0.0-20231211015131-3b3dfabdce9c @@ -79,7 +80,7 @@ require ( github.com/andybalholm/brotli v1.0.6 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef // indirect - github.com/aws/aws-sdk-go v1.49.15 // indirect + github.com/aws/aws-sdk-go v1.49.16 // indirect github.com/bytedance/sonic v1.10.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect @@ -140,7 +141,6 @@ require ( github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect diff --git a/kernel/go.sum b/kernel/go.sum index 04d09ee4e..8ef964788 100644 --- a/kernel/go.sum +++ b/kernel/go.sum @@ -49,8 +49,8 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhP github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= -github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= -github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.16 h1:KAQwhLg296hfffRdh+itA9p7Nx/3cXS/qOa3uF9ssig= +github.com/aws/aws-sdk-go v1.49.16/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -354,8 +354,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d h1:lvCTyBbr36+tqMccdGMwuEU+hjux/zL6xSmf5S9ITaA= github.com/shurcooL/gofontwoff v0.0.0-20181114050219-180f79e6909d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= github.com/simplereach/timeutils v1.2.0/go.mod h1:VVbQDfN/FHRZa1LSqcwo4kNZ62OOyqLLGQKYB3pB0Q8= -github.com/siyuan-note/dejavu v0.0.0-20240105084039-9867350d60a1 h1:DtHawQtKy2N1ocpGCjoB4q8UqxYYwYqVaFlSo++59qY= -github.com/siyuan-note/dejavu v0.0.0-20240105084039-9867350d60a1/go.mod h1:XFB3igKt0cE8QByhcw5VKnEZK9rHAQzHH4F/dJ17GHU= +github.com/siyuan-note/dejavu v0.0.0-20240106123310-d41af94d3e67 h1:foJRxYDhG81UHzLmE+4oIHON918n+yi/xff7dIqru7Y= +github.com/siyuan-note/dejavu v0.0.0-20240106123310-d41af94d3e67/go.mod h1:3uStV30HTpFN1Zvyaf/thqheOh4RJ7WIVeTqUpBzNhc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/evyW6LcB9IQT8PR4ppx8JVqOFP9Ix3rfwSrc= github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis= github.com/siyuan-note/eventbus v0.0.0-20231210025112-82fdedd8e374 h1:haPIOZBOAEQaXsUPUpyZeOeMgw6g/9O7RBbS3d39Ta8= diff --git a/kernel/sql/upsert.go b/kernel/sql/upsert.go index 52811d107..a022ffaaa 100644 --- a/kernel/sql/upsert.go +++ b/kernel/sql/upsert.go @@ -21,12 +21,20 @@ import ( "crypto/sha256" "database/sql" "fmt" - "github.com/siyuan-note/siyuan/kernel/filesys" + "os" + "path" + "path/filepath" "strings" + "sync" + "time" + "github.com/88250/gulu" "github.com/88250/lute/parse" "github.com/emirpasic/gods/sets/hashset" + ignore "github.com/sabhiram/go-gitignore" "github.com/siyuan-note/eventbus" + "github.com/siyuan-note/logging" + "github.com/siyuan-note/siyuan/kernel/filesys" "github.com/siyuan-note/siyuan/kernel/util" ) @@ -400,11 +408,6 @@ func indexTree(tx *sql.Tx, box, p string, context map[string]interface{}) (err e return } - err = insertTree(tx, tree, context) - return -} - -func insertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (err error) { blocks, spans, assets, attributes := fromTree(tree.Root, tree) refs, fileAnnotationRefs := refsFromTree(tree) err = insertTree0(tx, tree, context, blocks, spans, assets, attributes, refs, fileAnnotationRefs) @@ -470,6 +473,14 @@ func upsertTree(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}) (e func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}, blocks []*Block, spans []*Span, assets []*Asset, attributes []*Attribute, refs []*Ref, fileAnnotationRefs []*FileAnnotationRef) (err error) { + if ignoreLines := getIndexIgnoreLines(); 0 < len(ignoreLines) { + // Support ignore index https://github.com/siyuan-note/siyuan/issues/9198 + matcher := ignore.CompileIgnoreLines(ignoreLines...) + if matcher.MatchesPath("/" + path.Join(tree.Box, tree.Path)) { + return + } + } + if err = insertBlocks(tx, blocks, context); nil != err { return } @@ -498,3 +509,51 @@ func insertTree0(tx *sql.Tx, tree *parse.Tree, context map[string]interface{}, } return } + +var ( + indexIgnoreLastModified int64 + indexIgnore []string + indexIgnoreLock = sync.Mutex{} +) + +func getIndexIgnoreLines() (ret []string) { + now := time.Now().UnixMilli() + if now-indexIgnoreLastModified < 30*1000 { + return indexIgnore + } + + indexIgnoreLock.Lock() + defer indexIgnoreLock.Unlock() + + indexIgnoreLastModified = now + + indexIgnorePath := filepath.Join(util.DataDir, ".siyuan", "indexignore") + err := os.MkdirAll(filepath.Dir(indexIgnorePath), 0755) + if nil != err { + return + } + if !gulu.File.IsExist(indexIgnorePath) { + if err = gulu.File.WriteFileSafer(indexIgnorePath, nil, 0644); nil != err { + logging.LogErrorf("create indexignore [%s] failed: %s", indexIgnorePath, err) + return + } + } + data, err := os.ReadFile(indexIgnorePath) + if nil != err { + logging.LogErrorf("read indexignore [%s] failed: %s", indexIgnorePath, err) + return + } + dataStr := string(data) + dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n") + ret = strings.Split(dataStr, "\n") + + ret = gulu.Str.RemoveDuplicatedElem(ret) + if 0 < len(ret) && "" == ret[0] { + ret = ret[1:] + } + indexIgnore = nil + for _, line := range ret { + indexIgnore = append(indexIgnore, line) + } + return +}