feat: 支持删除对象存储过期的文件 (#27)

client/backup.go

@@ -103,7 +103,9 @@ func run(conf entity.Config, backupConf entity.BackupConfig) {
 			result.FileSize = fmt.Sprintf("%d MB", outFileName.Size()/1000/1000)
 			result.Result = "成功"
 			// send file to s3
-			go conf.UploadFile(backupConf.GetProjectPath() + string(os.PathSeparator) + outFileName.Name())
+			if conf.S3Config.CheckNotEmpty() {
+				go conf.S3Config.UploadFile(backupConf.GetProjectPath() + string(os.PathSeparator) + outFileName.Name())
+			}
 		}
 		conf.ExecWebhook(result)
 	}
@@ -125,7 +127,7 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 	projectName := backupConf.ProjectName
 	log.Printf("正在备份项目: %s ...", projectName)
 
-	todayString := time.Now().Format("2006-01-02_15_04")
+	todayString := time.Now().Format(util.FileNameFormatStr)
 	shellString := strings.ReplaceAll(backupConf.Command, "#{DATE}", todayString)
 
 	// 解密pwd
@@ -144,9 +146,9 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 	// create shell file
 	var shellName string
 	if runtime.GOOS == "windows" {
-		shellName = time.Now().Format("shell-2006-01-02-15-04-") + "backup.bat"
+		shellName = time.Now().Format("shell-"+util.FileNameFormatStr+"-") + "backup.bat"
 	} else {
-		shellName = time.Now().Format("shell-2006-01-02-15-04-") + "backup.sh"
+		shellName = time.Now().Format("shell-"+util.FileNameFormatStr+"-") + "backup.sh"
 	}
 
 	shellFile, err := os.Create(backupConf.GetProjectPath() + string(os.PathSeparator) + shellName)

client/delete_old_file.go

@@ -6,7 +6,6 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"strconv"
 	"time"
 )
 
@@ -26,30 +25,57 @@ func DeleteOldBackup() {
 			if !backupConf.NotEmptyProject() {
 				continue
 			}
-			// read from current path
-			backupFiles, err := ioutil.ReadDir(backupConf.GetProjectPath())
-			if err != nil {
-				log.Printf("读取项目 %s 目录失败! ERR: %s\n", backupConf.ProjectName, err)
-				continue
-			}
+			// 删除本地的过时文件
+			deleteLocalOlderFiles(backupConf)
 
-			// delete client files
-			subDuration, _ := time.ParseDuration("-" + strconv.Itoa(backupConf.SaveDays*24) + "h")
-			before := time.Now().Add(subDuration)
-
-			// delete older file when file numbers gt MaxSaveDays
-			for _, backupFile := range backupFiles {
-				if backupFile.ModTime().Before(before) {
-					filepath := backupConf.GetProjectPath() + string(os.PathSeparator) + backupFile.Name()
-					err := os.Remove(filepath)
-					if err != nil {
-						log.Printf("删除过期的文件 %s 失败", filepath)
-					} else {
-						log.Printf("删除过期的文件 %s 成功", filepath)
-					}
-				}
-			}
+			// 删除对象存储中的过时文件
+			deleteS3OlderFiles(conf.S3Config, backupConf)
 		}
 	}
 
 }
+
+// deleteLocalOlderFiles 删除本地的过时文件
+func deleteLocalOlderFiles(backupConf entity.BackupConfig) {
+	backupFiles, err := ioutil.ReadDir(backupConf.GetProjectPath())
+	if err != nil {
+		log.Printf("读取项目 %s 的本地目录失败! ERR: %s\n", backupConf.ProjectName, err)
+	}
+	backupFileNames := make([]string, len(backupFiles))
+	for _, backupFile := range backupFiles {
+		backupFileNames = append(backupFileNames, backupFile.Name())
+	}
+
+	tobeDeleteFiles := util.FileNameBeforeDays(backupConf.SaveDays, backupFileNames)
+
+	for i := 0; i < len(tobeDeleteFiles); i++ {
+		err := os.Remove(backupConf.GetProjectPath() + string(os.PathSeparator) + tobeDeleteFiles[i])
+		if err == nil {
+			log.Printf("删除过期的文件(本地) %s 成功", backupConf.ProjectName+string(os.PathSeparator)+tobeDeleteFiles[i])
+		} else {
+			log.Printf("删除过期的文件(本地) %s 失败: %s", backupConf.ProjectName+string(os.PathSeparator)+tobeDeleteFiles[i], err)
+		}
+	}
+}
+
+// deleteS3OlderFiles 删除对象存储的过时文件
+func deleteS3OlderFiles(s3Conf entity.S3Config, backupConf entity.BackupConfig) {
+	if !s3Conf.CheckNotEmpty() {
+		return
+	}
+	fileNames, err := s3Conf.ListFiles(backupConf.GetProjectPath())
+	if err != nil {
+		log.Printf("读取项目 %s 的对象存储目录失败! ERR: %s\n", backupConf.ProjectName, err)
+	}
+
+	tobeDeleteFiles := util.FileNameBeforeDays(backupConf.SaveDaysS3, fileNames)
+
+	for i := 0; i < len(tobeDeleteFiles); i++ {
+		err := s3Conf.DeleteFile(tobeDeleteFiles[i])
+		if err == nil {
+			log.Printf("删除过期的文件(对象存储) %s 成功", tobeDeleteFiles[i])
+		} else {
+			log.Printf("删除过期的文件(对象存储) %s 失败: %s", tobeDeleteFiles[i], err)
+		}
+	}
+}

entity/config_backup.go

@@ -4,7 +4,8 @@ package entity
 type BackupConfig struct {
 	ProjectName string // 项目名称
 	Command     string // 命令
-	SaveDays    int
+	SaveDays    int    // 本地保存天数
+	SaveDaysS3  int    // 对象存储保存天数
 	StartTime   int    // 开始时间(0-23)
 	Period      int    // 间隔周期(分钟)
 	Pwd         string // 密码

entity/s3_config.go

@@ -23,14 +23,14 @@ type S3Config struct {
 
 var ErrS3Empty = errors.New("s3 config is empty")
 
-func (s3Config S3Config) checkNotEmpty() bool {
+func (s3Config S3Config) CheckNotEmpty() bool {
 	return s3Config.Endpoint != "" && s3Config.AccessKey != "" &&
 		s3Config.SecretKey != "" && s3Config.BucketName != ""
 }
 
 func (s3Config S3Config) getSession() (*session.Session, error) {
 
-	if !s3Config.checkNotEmpty() {
+	if !s3Config.CheckNotEmpty() {
 		return nil, ErrS3Empty
 	}
 
@@ -89,6 +89,7 @@ func (s3Config S3Config) CreateBucketIfNotExist() {
 	}
 }
 
+// UploadFile 上传
 func (s3Config S3Config) UploadFile(fileName string) {
 	mySession, err := s3Config.getSession()
 	if err != nil {
@@ -117,3 +118,52 @@ func (s3Config S3Config) UploadFile(fileName string) {
 		log.Printf("%s 上传到对象存储成功\n", fileName)
 	}
 }
+
+// ListFiles 列出文件
+func (s3Config S3Config) ListFiles(projectPath string) (fileNames []string, err error) {
+	mySession, err := s3Config.getSession()
+	if err != nil {
+		if err != ErrS3Empty {
+			log.Printf("创建对象存储会话失败, ERR: %s\n", err)
+		}
+		return nil, err
+	}
+
+	svc := s3.New(mySession)
+	params := &s3.ListObjectsInput{
+		Bucket: aws.String(s3Config.BucketName),
+		Prefix: aws.String(projectPath),
+	}
+	resp, err := svc.ListObjects(params)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range resp.Contents {
+		fileNames = append(fileNames, *item.Key)
+	}
+
+	return fileNames, err
+}
+
+// DeleteFile 删除文件
+func (s3Config S3Config) DeleteFile(filePath string) error {
+	mySession, err := s3Config.getSession()
+	if err != nil {
+		if err != ErrS3Empty {
+			log.Printf("创建对象存储会话失败, ERR: %s\n", err)
+		}
+		return err
+	}
+
+	svc := s3.New(mySession)
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s3Config.BucketName), Key: aws.String(filePath)})
+	if err != nil {
+		return err
+	}
+
+	return svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
+		Bucket: aws.String(s3Config.BucketName),
+		Key:    aws.String(filePath),
+	})
+}

util/file_name_util.go

@@ -0,0 +1,28 @@
+package util
+
+import (
+	"regexp"
+	"strconv"
+	"time"
+)
+
+const FileNameFormatStr = "2006-01-02_15_04"
+
+// FileNameBeforeDays 查找文件名中有多少在指定天数之前的
+func FileNameBeforeDays(days int, fileNames []string) []string {
+	oldFiles := make([]string, 0)
+	// 2006-01-02_15_04
+	fileRegxp := regexp.MustCompile(`([\d]{4})-([\d]{2})-([\d]{2})_([\d]{2})_([\d]{2})`)
+	subDuration, _ := time.ParseDuration("-" + strconv.Itoa(days*24) + "h")
+	before := time.Now().Add(subDuration)
+	for i := 0; i < len(fileNames); i++ {
+		dateString := fileRegxp.FindString(fileNames[i])
+		if dateString != "" {
+			if fileTime, err := time.Parse(FileNameFormatStr, dateString); err == nil && fileTime.Before(before) {
+				oldFiles = append(oldFiles, fileNames[i])
+			}
+		}
+
+	}
+	return oldFiles
+}

util/file_name_util_test.go

@@ -0,0 +1,21 @@
+package util
+
+import (
+	"strconv"
+	"testing"
+	"time"
+)
+
+// TestFileNameUtil
+func TestFileNameUtil(t *testing.T) {
+	const days = 10
+	beforeTomorrow, _ := time.ParseDuration("-" + strconv.Itoa((days+1)*24) + "h")
+	fileNames := []string{
+		"a2020-10-10_11_12b.sql",
+		"测试2021-10-10_11_12测试.sql",
+		time.Now().Add(beforeTomorrow).Format(FileNameFormatStr) + ".sql",
+	}
+	if len(fileNames) != len(FileNameBeforeDays(10, fileNames)) {
+		t.Error("TestFileNameUtil Test failed!")
+	}
+}

web/writing.go

@@ -29,7 +29,7 @@ func WritingConfig(writer http.ResponseWriter, request *http.Request) {
 	// 获得环境变量
 	backupConf := []entity.BackupConfig{}
 	for i := 0; i < 16; i++ {
-		backupConf = append(backupConf, entity.BackupConfig{SaveDays: 30, StartTime: 1, Period: 1440})
+		backupConf = append(backupConf, entity.BackupConfig{SaveDays: 30, SaveDaysS3: 60, StartTime: 1, Period: 1440})
 	}
 	conf = entity.Config{
 		BackupConfig: backupConf,

web/writing.html

@@ -83,10 +83,14 @@
                   </div>
 
                   <div class="form-group row">
-                    <label for="SaveDays_{{$i}}" class="col-sm-2 col-form-label">保存天数</label>
-                    <div class="col-sm-10">
+                    <label for="SaveDays_{{$i}}" class="col-sm-2 col-form-label">本地保存(天)</label>
+                    <div class="col-sm-4">
                       <input type="number" class="form-control" name="SaveDays" id="SaveDays_{{$i}}" value="{{$v.SaveDays}}" min="1">
                     </div>
+                    <label for="SaveDaysS3_{{$i}}" class="col-sm-2 col-form-label">对象存储保存(天)</label>
+                    <div class="col-sm-4">
+                      <input type="number" class="form-control" name="SaveDaysS3" id="SaveDaysS3_{{$i}}" value="{{$v.SaveDaysS3}}" min="1">
+                    </div>
                   </div>
 
                   <div class="form-group row">