feat: 增加数据库支持及增量更新功能

README.md

@@ -8,16 +8,21 @@ NAME:
    AlistAutoStrm - Auto generate .strm file for EMBY or Jellyfin server use Alist API
 
 USAGE:
-   AlistAutoStrm [global options] command [command options] [arguments...]
+   AlistAutoStrm [global options] command [command options]
 
 VERSION:
-   1.1.2
+   1.2.0
 
 DESCRIPTION:
    Auto generate .strm file for EMBY or Jellyfin server use Alist API
 
 COMMANDS:
-   help, h  Shows a list of commands or help for one command
+   fresh-all        generate all strm files from alist server, whatever the file has been generated or not
+   update           update strm file with choosed mode
+   update-database  clean database and get all local strm files stored in database
+   check            check if strm file is valid
+   version          show version
+   help, h          Shows a list of commands or help for one command
 
 GLOBAL OPTIONS:
    --config FILE, -c FILE  Load configuration from FILE (default: "config.json") [%ALIST_AUTO_STRM_CONFIG%]

config.go

@@ -1,13 +1,15 @@
 package main
 
 type Config struct {
-	Database           string     `json:"database" yaml:"database"`
-	Endpoints          []Endpoint `json:"endpoints" yaml:"endpoints"`
-	Loglevel           string     `json:"loglevel" yaml:"loglevel"`
-	ColoredLog         bool       `json:"colored-log" yaml:"colored-log"`
-	Timeout            int        `json:"timeout" yaml:"timeout"`
-	Exts               []string   `json:"exts" yaml:"exts"`
-	CreateSubDirectory bool       `json:"create-sub-directory" yaml:"create-sub-directory"`
+	Database            string     `json:"database" yaml:"database"`
+	Endpoints           []Endpoint `json:"endpoints" yaml:"endpoints"`
+	Loglevel            string     `json:"loglevel" yaml:"loglevel"`
+	ColoredLog          bool       `json:"colored-log" yaml:"colored-log"`
+	Timeout             int        `json:"timeout" yaml:"timeout"`
+	Exts                []string   `json:"exts" yaml:"exts"`
+	CreateSubDirectory  bool       `json:"create-sub-directory" yaml:"create-sub-directory"`
+	isIncrementalUpdate bool
+	records             map[string]int
 }
 
 type Endpoint struct {

functions.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/boltdb/bolt"
 	sdk "github.com/imshuai/alistsdk-go"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -57,6 +58,14 @@ func loadConfig(c *cli.Context) error {
 			return errors.New("unmarshal yaml type config file error: " + err.Error())
 		}
 	}
+	db, err = bolt.Open(config.Database, 0600, nil)
+	if err != nil {
+		return errors.New("open database error: " + err.Error())
+	}
+	config.records, err = GetRecordCollection()
+	if err != nil {
+		return errors.New("get record collection error: " + err.Error())
+	}
 	return nil
 }
 
@@ -195,7 +204,7 @@ func readStrmFile(file string) *Strm {
 	// TODO 读取strm文件
 	strm := &Strm{}
 	strm.Name = filepath.Base(file)
-	strm.Dir = filepath.Dir(file)
+	strm.LocalDir = filepath.Dir(file)
 	strm.RawURL = func() string {
 		byts, err := os.ReadFile(file)
 		if err != nil {
@@ -204,5 +213,10 @@ func readStrmFile(file string) *Strm {
 		//返回的字符串应该有且只有一行，且不会以\n或者\r\n结束
 		return strings.TrimRight(strings.Split(string(byts), "\n")[0], "\r")
 	}()
+	strm.RemoteDir = func() string {
+		paths := strings.Split(strings.Split(strm.RawURL, "/d/")[1], "/")
+		paths = paths[:len(paths)-1]
+		return "/" + strings.Join(paths, "/")
+	}()
 	return strm
 }

main.go

@@ -13,7 +13,7 @@ const (
 	// 定义常量
 	NAME        = "AlistAutoStrm"
 	DESCRIPTION = "Auto generate .strm file for EMBY or Jellyfin server use Alist API"
-	VERSION     = "1.1.2"
+	VERSION     = "1.2.0"
 )
 
 var (
@@ -27,6 +27,9 @@ func main() {
 	// 程序退出时显示光标
 	defer func() {
 		fmt.Print("\033[?25h")
+		if db != nil {
+			db.Close()
+		}
 	}()
 
 	// 初始化一个mpb.Progress实例
@@ -175,6 +178,11 @@ func main() {
 					Usage: "update mode, support: local, remote. when strm content is same but filename changed, local: keep local filename, remote: rename local filename to remote filename",
 					Value: "local",
 				},
+				&cli.BoolFlag{
+					Name:  "no-incremental-update",
+					Usage: "when this flag is set, will not use incremental update, will update all files",
+					Value: false,
+				},
 			},
 			Action: func(c *cli.Context) error {
 				//TODO 实现strm文件更新功能
@@ -216,6 +224,8 @@ func main() {
 
 				mode := c.String("mode")
 				logger.Debugf("[MAIN]: update mode: %s", mode)
+				config.isIncrementalUpdate = !c.Bool("no-incremental-update") //是否使用增量更新
+				logger.Debugf("[MAIN]: incremental update: %t", config.isIncrementalUpdate)
 				localStrms := make(map[string]*Strm, 0)
 				remoteStrms := make(map[string]*Strm, 0)
 				addStrms := make([]*Strm, 0)
@@ -278,7 +288,7 @@ func main() {
 						continue
 					}
 					added++
-					logger.Infof("[MAIN]: generate file %s success", v.Dir+"/"+v.Name)
+					logger.Infof("[MAIN]: generate file %s success", v.LocalDir+"/"+v.Name)
 				}
 
 				for _, v := range deleteStrms {
@@ -297,6 +307,29 @@ func main() {
 				return nil
 			},
 		},
+		{
+			Name:  "update-database",
+			Usage: "clean database and get all local strm files stored in database",
+			Action: func(c *cli.Context) error {
+				err := loadConfig(c)
+				if err != nil {
+					return err
+				}
+				records := make(map[string]int, 0)
+				for _, e := range config.Endpoints {
+					strms := fetchLocalFiles(e)
+					for _, v := range strms {
+						records[v.RemoteDir] = 0
+					}
+				}
+				err = SaveRecordCollection(records)
+				if err != nil {
+					return err
+				}
+				logger.Infof("[MAIN]: database has been cleaned, and %d records saved", len(records))
+				return nil
+			},
+		},
 		{
 			Name:  "check",
 			Usage: "check if strm file is valid",

mission.go

@@ -72,14 +72,15 @@ func (m *Mission) walk() {
 						//return replaceSpaceToDash(name) + ".strm"
 						return name + ".strm"
 					}(),
-					Dir:    m.LocalPath,
-					RawURL: m.BaseURL + "/d" + urlEncode(m.CurrentRemotePath+"/"+f.Name),
+					LocalDir:  m.LocalPath,
+					RemoteDir: m.CurrentRemotePath,
+					RawURL:    m.BaseURL + "/d" + m.CurrentRemotePath + "/" + f.Name,
 				}
 				err := strm.GenStrm(true)
 				if err != nil {
 					logger.Errorf("[thread %2d]: save file [%s] error: %s", idx, m.CurrentRemotePath+"/"+f.Name, err.Error())
 				}
-				logger.Tracef("[thread %2d]: generate [%s] ==> [%s] success", idx, path.Join(strm.Dir, strm.Name), strm.RawURL)
+				logger.Tracef("[thread %2d]: generate [%s] ==> [%s] success", idx, path.Join(strm.LocalDir, strm.Name), strm.RawURL)
 				logger.Add(1)
 			}
 		}
@@ -116,7 +117,11 @@ func (m *Mission) getStrm(strmChan chan *Strm) {
 	logger.Debugf("[thread %2d]: get %d files from [%s]", threadIdx, len(alistFiles), m.CurrentRemotePath)
 	for _, f := range alistFiles {
 		if f.IsDir && m.IsRecursive {
-			logger.Debugf("[thread %2d]: found sub directory [%s]", threadIdx, m.CurrentRemotePath+"/"+f.Name)
+			logger.Debugf("[thread %2d]: found directory [%s]", threadIdx, m.CurrentRemotePath+"/"+f.Name)
+			if _, ok := config.records[m.CurrentRemotePath+"/"+f.Name]; ok && config.isIncrementalUpdate {
+				logger.Debugf("[thread %2d]: directory [%s] already processed and use incremental update, skip", threadIdx, m.CurrentRemotePath+"/"+f.Name)
+				continue
+			}
 			mm := &Mission{
 				BaseURL:           m.BaseURL,
 				CurrentRemotePath: m.CurrentRemotePath + "/" + f.Name,
@@ -148,8 +153,10 @@ func (m *Mission) getStrm(strmChan chan *Strm) {
 						//return replaceSpaceToDash(name) + ".strm"
 						return name + ".strm"
 					}(),
-					Dir:    m.LocalPath,
-					RawURL: m.BaseURL + "/d" + urlEncode(m.CurrentRemotePath+"/"+f.Name),
+					RemoteDir: m.CurrentRemotePath,
+					LocalDir:  m.LocalPath,
+					RawURL:    m.BaseURL + "/d" + m.CurrentRemotePath + "/" + f.Name,
+					//RawURL:    m.BaseURL + "/d" + urlEncode(m.CurrentRemotePath+"/"+f.Name), //urlEncode is not necessary
 				}
 				strmChan <- strm
 				logger.Add(1)

types.go → strm.go

@@ -15,9 +15,10 @@ var (
 )
 
 type Strm struct {
-	Name   string `json:"name"`
-	Dir    string `json:"dir"`
-	RawURL string `json:"raw_url"`
+	Name      string `json:"name"`
+	LocalDir  string `json:"local_dir"`
+	RemoteDir string `json:"remote_dir"`
+	RawURL    string `json:"raw_url"`
 }
 
 func (s *Strm) Key() string {
@@ -38,7 +39,7 @@ func (s *Strm) Value() []byte {
 
 func (s *Strm) Delete() error {
 	//TODO 使用boltdb实现Strm对象的删除逻辑
-	err := os.RemoveAll(path.Join(s.Dir, s.Name))
+	err := os.RemoveAll(path.Join(s.LocalDir, s.Name))
 	if err != nil {
 		return err
 	}
@@ -70,17 +71,17 @@ func (s *Strm) Save() error {
 
 func (s *Strm) GenStrm(overwrite bool) error {
 	//创建s.Dir目录
-	err := os.MkdirAll(s.Dir, 0666)
+	err := os.MkdirAll(s.LocalDir, 0666)
 	if err != nil {
 		return err
 	}
 	// 如果s.Dir目录下已经存在s.Name文件，并且overwrite为false，则返回错误
-	_, err = os.Stat(path.Join(s.Dir, s.Name))
+	_, err = os.Stat(path.Join(s.LocalDir, s.Name))
 	if !overwrite && !os.IsNotExist(err) {
-		return fmt.Errorf("file %s already exists and overwrite is false", path.Join(s.Dir, s.Name))
+		return fmt.Errorf("file %s already exists and overwrite is false", path.Join(s.LocalDir, s.Name))
 	}
 	// 将s.RawURL写入s.Dir目录下的s.Name文件中
-	return os.WriteFile(path.Join(s.Dir, s.Name), []byte(s.RawURL), 0666)
+	return os.WriteFile(path.Join(s.LocalDir, s.Name), []byte(s.RawURL), 0666)
 }
 
 func GetStrm(rawUrl string) (*Strm, error) {
@@ -108,3 +109,31 @@ func GetStrm(rawUrl string) (*Strm, error) {
 	// 返回Strm对象和错误
 	return &strm, err
 }
+
+func GetRecordCollection() (map[string]int, error) {
+	var records map[string]int
+	err := db.View(func(tx *bolt.Tx) error {
+		// Assume bucket exists and has keys
+		b := tx.Bucket([]byte("strm"))
+		byts := b.Get([]byte("records"))
+		return json.Unmarshal(byts, &records)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return records, nil
+}
+
+func SaveRecordCollection(records map[string]int) error {
+	return db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("strm"))
+		if err != nil {
+			return err
+		}
+		byts, err := json.Marshal(records)
+		if err != nil {
+			return err
+		}
+		return b.Put([]byte("records"), byts)
+	})
+}