🎨 同步忽略文件配置使用 gitignore 规则 https://github.com/siyuan-note/siyuan/issues/5295

kernel/go.mod

@@ -93,6 +93,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/restic/chunker v0.4.0 // indirect
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect

kernel/go.sum

@@ -409,6 +409,8 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
+github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=

kernel/model/sync.go

@@ -34,8 +34,7 @@ import (
 
 	"github.com/88250/gulu"
 	"github.com/dustin/go-humanize"
-	"github.com/emirpasic/gods/sets/hashset"
-	"github.com/mattn/go-zglob"
+	gitignore "github.com/sabhiram/go-gitignore"
 	"github.com/siyuan-note/encryption"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/siyuan/kernel/cache"
@@ -1253,9 +1252,8 @@ func IsValidCloudDirName(cloudDirName string) bool {
 func getSyncExcludedList(localDirPath string) (ret map[string]bool) {
 	syncIgnoreList := getSyncIgnoreList()
 	ret = map[string]bool{}
-	ignores := syncIgnoreList.Values()
-	for _, p := range ignores {
-		relPath := p.(string)
+	for _, p := range syncIgnoreList {
+		relPath := p
 		relPath = pathSha256Short(relPath, "/")
 		relPath = filepath.Join(localDirPath, relPath)
 		ret[relPath] = true
@@ -1263,8 +1261,7 @@ func getSyncExcludedList(localDirPath string) (ret map[string]bool) {
 	return
 }
 
-func getSyncIgnoreList() (ret *hashset.Set) {
-	ret = hashset.New()
+func getSyncIgnoreList() (ret []string) {
 	ignore := filepath.Join(util.DataDir, ".siyuan", "syncignore")
 	os.MkdirAll(filepath.Dir(ignore), 0755)
 	if !gulu.File.IsExist(ignore) {
@@ -1287,37 +1284,16 @@ func getSyncIgnoreList() (ret *hashset.Set) {
 	lines = append(lines, "20210808180117-czj9bvb/**/*")
 	lines = append(lines, "20211226090932-5lcq56f/**/*")
 
-	var parents []string
-	for _, line := range lines {
-		if idx := strings.Index(line, "/*"); -1 < idx {
-			parent := line[:idx]
-			parents = append(parents, parent)
+	lines = gulu.Str.RemoveDuplicatedElem(lines)
+	gi := gitignore.CompileIgnoreLines(lines...)
+	filepath.Walk(util.DataDir, func(p string, info os.FileInfo, err error) error {
+		p = strings.TrimPrefix(p, util.DataDir+string(os.PathSeparator))
+		p = filepath.ToSlash(p)
+		if gi.MatchesPath(p) {
+			ret = append(ret, p)
 		}
-	}
-	lines = append(lines, parents...)
-
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
-		if "" == line {
-			continue
-		}
-		pattern := filepath.Join(util.DataDir, line)
-		pattern = filepath.FromSlash(pattern)
-		matches, globErr := zglob.Glob(pattern)
-		if nil != globErr && globErr != os.ErrNotExist {
-			util.LogErrorf("glob [%s] failed: %s", line, globErr)
-			continue
-		}
-		for _, m := range matches {
-			m = filepath.ToSlash(m)
-			if strings.Contains(m, ".siyuan/history") {
-				continue
-			}
-
-			m = strings.TrimPrefix(m, filepath.ToSlash(util.DataDir+string(os.PathSeparator)))
-			ret.Add(m)
-		}
-	}
+		return nil
+	})
 	return
 }
 

kernel/util/ignore.go

@@ -0,0 +1,204 @@
+// SiYuan - Build Your Eternal Digital Garden
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package util
+
+// 该文件代码来自 https://github.com/go-git/go-git 项目，Apache-2.0 license
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// https://github.com/go-git/go-git/blob/master/plumbing/format/gitignore/matcher.go
+
+// Matcher defines a global multi-pattern matcher for gitignore patterns
+type Matcher interface {
+	// Match matches patterns in the order of priorities. As soon as an inclusion or
+	// exclusion is found, not further matching is performed.
+	Match(path []string, isDir bool) bool
+}
+
+// NewMatcher constructs a new global matcher. Patterns must be given in the order of
+// increasing priority. That is most generic settings files first, then the content of
+// the repo .gitignore, then content of .gitignore down the path or the repo and then
+// the content command line arguments.
+func NewMatcher(ps []Pattern) Matcher {
+	return &matcher{ps}
+}
+
+type matcher struct {
+	patterns []Pattern
+}
+
+func (m *matcher) Match(path []string, isDir bool) bool {
+	n := len(m.patterns)
+	for i := n - 1; i >= 0; i-- {
+		if match := m.patterns[i].Match(path, isDir); match > NoMatch {
+			return match == Exclude
+		}
+	}
+	return false
+}
+
+// https://github.com/go-git/go-git/blob/master/plumbing/format/gitignore/pattern.go
+
+// MatchResult defines outcomes of a match, no match, exclusion or inclusion.
+type MatchResult int
+
+const (
+	// NoMatch defines the no match outcome of a match check
+	NoMatch MatchResult = iota
+	// Exclude defines an exclusion of a file as a result of a match check
+	Exclude
+	// Include defines an explicit inclusion of a file as a result of a match check
+	Include
+)
+
+const (
+	inclusionPrefix = "!"
+	zeroToManyDirs  = "**"
+	patternDirSep   = "/"
+)
+
+// Pattern defines a single gitignore pattern.
+type Pattern interface {
+	// Match matches the given path to the pattern.
+	Match(path []string, isDir bool) MatchResult
+}
+
+type pattern struct {
+	domain    []string
+	pattern   []string
+	inclusion bool
+	dirOnly   bool
+	isGlob    bool
+}
+
+// ParsePattern parses a gitignore pattern string into the Pattern structure.
+func ParsePattern(p string, domain []string) Pattern {
+	res := pattern{domain: domain}
+
+	if strings.HasPrefix(p, inclusionPrefix) {
+		res.inclusion = true
+		p = p[1:]
+	}
+
+	if !strings.HasSuffix(p, "\\ ") {
+		p = strings.TrimRight(p, " ")
+	}
+
+	if strings.HasSuffix(p, patternDirSep) {
+		res.dirOnly = true
+		p = p[:len(p)-1]
+	}
+
+	if strings.Contains(p, patternDirSep) {
+		res.isGlob = true
+	}
+
+	res.pattern = strings.Split(p, patternDirSep)
+	return &res
+}
+
+func (p *pattern) Match(path []string, isDir bool) MatchResult {
+	if len(path) <= len(p.domain) {
+		return NoMatch
+	}
+	for i, e := range p.domain {
+		if path[i] != e {
+			return NoMatch
+		}
+	}
+
+	path = path[len(p.domain):]
+	if p.isGlob && !p.globMatch(path, isDir) {
+		return NoMatch
+	} else if !p.isGlob && !p.simpleNameMatch(path, isDir) {
+		return NoMatch
+	}
+
+	if p.inclusion {
+		return Include
+	} else {
+		return Exclude
+	}
+}
+
+func (p *pattern) simpleNameMatch(path []string, isDir bool) bool {
+	for i, name := range path {
+		if match, err := filepath.Match(p.pattern[0], name); err != nil {
+			return false
+		} else if !match {
+			continue
+		}
+		if p.dirOnly && !isDir && i == len(path)-1 {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+func (p *pattern) globMatch(path []string, isDir bool) bool {
+	matched := false
+	canTraverse := false
+	for i, pattern := range p.pattern {
+		if pattern == "" {
+			canTraverse = false
+			continue
+		}
+		if pattern == zeroToManyDirs {
+			if i == len(p.pattern)-1 {
+				break
+			}
+			canTraverse = true
+			continue
+		}
+		if strings.Contains(pattern, zeroToManyDirs) {
+			return false
+		}
+		if len(path) == 0 {
+			return false
+		}
+		if canTraverse {
+			canTraverse = false
+			for len(path) > 0 {
+				e := path[0]
+				path = path[1:]
+				if match, err := filepath.Match(pattern, e); err != nil {
+					return false
+				} else if match {
+					matched = true
+					break
+				} else if len(path) == 0 {
+					// if nothing left then fail
+					matched = false
+				}
+			}
+		} else {
+			if match, err := filepath.Match(pattern, path[0]); err != nil || !match {
+				return false
+			}
+			matched = true
+			path = path[1:]
+		}
+	}
+	if matched && p.dirOnly && !isDir && len(path) == 0 {
+		matched = false
+	}
+	return matched
+}