feat: 文件同步(s3sync) (#30)

Dockerfile

@@ -7,6 +7,18 @@ RUN go env -w GO111MODULE=on \
     && go env -w GOPROXY=https://goproxy.cn,direct \
     && make clean test build
 
+# build s3sync
+FROM golang:1.17 AS s3sync
+
+WORKDIR /src/
+RUN git clone --branch 2.33 https://github.com/larrabee/s3sync.git
+
+WORKDIR /src/s3sync
+ENV CGO_ENABLED 0
+COPY . ./
+RUN go mod vendor && \
+    go build -o s3sync ./cli
+
 # final stage
 FROM debian:stable-slim
 
@@ -23,5 +35,6 @@ WORKDIR /app
 VOLUME /app/backup-x-files
 ENV TZ=Asia/Shanghai
 COPY --from=builder /app/backup-x /app/backup-x
+COPY --from=s3sync /src/s3sync/s3sync /usr/local/bin/s3sync
 EXPOSE 9977
 ENTRYPOINT ["/app/backup-x"]

README.md

@@ -21,7 +21,7 @@
     jeessy/backup-x
   ```
 - 登录 http://your_docker_ip:9977 并配置
-- docker容器默认安装default-mysql-client/postgres-client
+- docker容器默认安装default-mysql-client/postgres-client/[s3sync](https://github.com/larrabee/s3sync)
 
 ## 系统中使用
 - 下载并解压[https://github.com/jeessy2/backup-x/releases](https://github.com/jeessy2/backup-x/releases)
@@ -59,9 +59,19 @@
 
     |  说明   | 备份脚本  |
     |  ----  | ----  |
-    | tar压缩备份 | tar -zcvf #{DATE}.tar.gz /home/projects |
-    | 还原 | tar -zxvf 2021-11-12_10_29.tar.gz |
+    | 备份本地文件到对象存储 [s3sync](https://github.com/larrabee/s3sync) | s3sync --fs-disable-xattr --filter-not-exist --tk #{AccessKey} --ts #{SecretKey} --te #{Endpoint} fs:///opt/test/ s3://#{BucketName}/test/ |
+    | 备份对象存储到对象存储 [s3sync](https://github.com/larrabee/s3sync) | s3sync --filter-not-exist --sk source_key -ss #{PWD} --se https://s3.source.com --tk #{AccessKey} --ts #{SecretKey} --te #{Endpoint} s3://backup/ s3://#{BucketName}/ |
 
+  - 变量说明
+
+    |  变量名   | 说明  |
+    |  ----  | ----  |
+    |  #{DATE}  | 年-月-日_时_分  |
+    |  #{PWD}   | 下方的密码变量  |
+    |  #{Endpoint}  | 下方的对象存储变量 Endpoint  |
+    |  #{AccessKey}  | 下方的对象存储变量 AccessKey  |
+    |  #{SecretKey}  | 下方的对象存储变量 SecretKey  |
+    |  #{BucketName}  | 下方的对象存储变量 BucketName  |
 ## webhook
 - 支持webhook, 备份更新成功或不成功时, 会回调填写的URL
 - 支持的变量

client/backup.go

@@ -95,17 +95,19 @@ func run(conf entity.Config, backupConf entity.BackupConfig) {
 			return
 		}
 		// backup
-		outFileName, err := backup(backupConf, conf.EncryptKey)
+		outFileName, err := backup(backupConf, conf.EncryptKey, conf.S3Config)
 		result := entity.BackupResult{ProjectName: backupConf.ProjectName, Result: "失败"}
 		if err == nil {
 			// webhook
-			result.FileName = outFileName.Name()
-			result.FileSize = fmt.Sprintf("%d MB", outFileName.Size()/1000/1000)
-			result.Result = "成功"
-			// send file to s3
-			if conf.S3Config.CheckNotEmpty() {
-				go conf.S3Config.UploadFile(backupConf.GetProjectPath() + string(os.PathSeparator) + outFileName.Name())
+			if outFileName != nil {
+				result.FileName = outFileName.Name()
+				result.FileSize = fmt.Sprintf("%d MB", outFileName.Size()/1000/1000)
+				// send file to s3
+				if conf.S3Config.CheckNotEmpty() {
+					go conf.S3Config.UploadFile(backupConf.GetProjectPath() + string(os.PathSeparator) + outFileName.Name())
+				}
 			}
+			result.Result = "成功"
 		}
 		conf.ExecWebhook(result)
 	}
@@ -116,14 +118,10 @@ func prepare(backupConf entity.BackupConfig) (err error) {
 	// create floder
 	os.MkdirAll(backupConf.GetProjectPath(), 0750)
 
-	if !strings.Contains(backupConf.Command, "#{DATE}") {
-		err = errors.New("项目: " + backupConf.ProjectName + "的备份脚本须包含#{DATE}")
-	}
-
 	return
 }
 
-func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.FileInfo, err error) {
+func backup(backupConf entity.BackupConfig, encryptKey string, s3Conf entity.S3Config) (outFileName os.FileInfo, err error) {
 	projectName := backupConf.ProjectName
 	log.Printf("正在备份项目: %s ...", projectName)
 
@@ -140,8 +138,22 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 			return nil, err
 		}
 	}
+	// 解密s3 SecretKey
+	secretKey := ""
+	if s3Conf.SecretKey != "" {
+		secretKey, err = util.DecryptByEncryptKey(encryptKey, s3Conf.SecretKey)
+		if err != nil {
+			err = fmt.Errorf("解密失败")
+			log.Println(err)
+			return nil, err
+		}
+	}
 
 	shellString = strings.ReplaceAll(shellString, "#{PWD}", pwd)
+	shellString = strings.ReplaceAll(shellString, "#{AccessKey}", s3Conf.AccessKey)
+	shellString = strings.ReplaceAll(shellString, "#{SecretKey}", secretKey)
+	shellString = strings.ReplaceAll(shellString, "#{Endpoint}", s3Conf.Endpoint)
+	shellString = strings.ReplaceAll(shellString, "#{BucketName}", s3Conf.BucketName)
 
 	// create shell file
 	var shellName string
@@ -170,7 +182,7 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 	shell.Dir = backupConf.GetProjectPath()
 	outputBytes, err := shell.CombinedOutput()
 	if len(outputBytes) > 0 {
-		log.Printf("<span title=\"%s\">%s 执行shell的输出：鼠标移动此处查看</span>\n", util.EscapeShell(string(outputBytes)), backupConf.ProjectName)
+		log.Printf("<span title=\"%s\">%s 执行shell的输出: 鼠标移动此处查看</span>\n", util.EscapeShell(string(outputBytes)), backupConf.ProjectName)
 	} else {
 		log.Printf("执行shell的输出为空\n")
 	}
@@ -179,23 +191,30 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 	if err == nil {
 		// find backup file by todayString
 		outFileName, err = findBackupFile(backupConf, todayString)
-
-		// check file size
-		if err != nil {
-			log.Println(err)
-		} else if outFileName.Size() >= 200 {
-			log.Printf("成功备份项目: %s, 文件名: %s\n", projectName, outFileName.Name())
-			// success, remove shell file
-			os.Remove(shellFile.Name())
+		if backupConf.BackupType == 0 {
+			// 备份数据库
+			// check file size
+			if err != nil {
+				log.Println(err)
+			} else if outFileName.Size() >= 200 {
+				log.Printf("成功备份项目: %s, 文件名: %s\n", projectName, outFileName.Name())
+			} else {
+				err = errors.New(projectName + " 备份后的文件大小小于200字节, 当前大小：" + strconv.Itoa(int(outFileName.Size())))
+				log.Println(err)
+			}
 		} else {
-			err = errors.New(projectName + " 备份后的文件大小小于200字节, 当前大小：" + strconv.Itoa(int(outFileName.Size())))
-			log.Println(err)
+			// 1 同步文件
+			// err = nil
+			err = nil
 		}
 	} else {
 		err = fmt.Errorf("执行备份shell失败: %s", util.EscapeShell(string(outputBytes)))
 		log.Println(err)
 	}
 
+	// remove shell file
+	os.Remove(shellFile.Name())
+
 	return
 }
 
@@ -203,11 +222,13 @@ func backup(backupConf entity.BackupConfig, encryptKey string) (outFileName os.F
 func findBackupFile(backupConf entity.BackupConfig, todayString string) (backupFile os.FileInfo, err error) {
 	files, err := ioutil.ReadDir(backupConf.GetProjectPath())
 	for _, file := range files {
-		if strings.Contains(file.Name(), todayString) {
+		if strings.Contains(file.Name(), todayString) && !strings.HasPrefix(file.Name(), "shell-") {
 			backupFile = file
 			return
 		}
 	}
-	err = errors.New("不能找到备份后的文件，没有找到包含 " + todayString + " 的文件名")
+
+	err = fmt.Errorf("项目 %s 没有输出包含 %s 的文件名", backupConf.ProjectName, todayString)
+
 	return
 }

entity/config_backup.go

@@ -9,6 +9,7 @@ type BackupConfig struct {
 	StartTime   int    // 开始时间(0-23)
 	Period      int    // 间隔周期(分钟)
 	Pwd         string // 密码
+	BackupType  int    // 备份类型 0 数据库备份 1 文件同步
 }
 
 // GetProjectPath 获得项目路径

web/save.go

@@ -47,6 +47,7 @@ func Save(writer http.ResponseWriter, request *http.Request) {
 		saveDaysS3, _ := strconv.Atoi(forms["SaveDaysS3"][index])
 		startTime, _ := strconv.Atoi(forms["StartTime"][index])
 		period, _ := strconv.Atoi(forms["Period"][index])
+		backupType, _ := strconv.Atoi(forms["BackupType"][index])
 		conf.BackupConfig = append(
 			conf.BackupConfig,
 			entity.BackupConfig{
@@ -57,6 +58,7 @@ func Save(writer http.ResponseWriter, request *http.Request) {
 				StartTime:   startTime,
 				Period:      period,
 				Pwd:         forms["Pwd"][index],
+				BackupType:  backupType,
 			},
 		)
 	}

web/writing.go

@@ -37,7 +37,7 @@ func WritingConfig(writer http.ResponseWriter, request *http.Request) {
 	// 获得环境变量
 	backupConf := []entity.BackupConfig{}
 	for i := 0; i < 16; i++ {
-		backupConf = append(backupConf, entity.BackupConfig{SaveDays: 30, SaveDaysS3: 60, StartTime: 1, Period: 1440})
+		backupConf = append(backupConf, entity.BackupConfig{SaveDays: 30, SaveDaysS3: 60, StartTime: 1, Period: 1440, BackupType: 0})
 	}
 	conf = entity.Config{
 		BackupConfig: backupConf,

web/writing.html

@@ -70,12 +70,23 @@
                     <div class="col-sm-10">
                       <textarea class="form-control" name="Command" id="Command_{{$i}}" rows="3" aria-describedby="Command_help">{{$v.Command}}</textarea>
                       <small id="Command_help" class="form-text text-muted">
-                        须包含#{DATE}变量, #{PWD}为下方的密码变量 <a target="blank" href="https://github.com/jeessy2/backup-x#备份脚本参考">备份脚本参考</a> 
-                        <br/>例：mysqldump -h192.168.1.11 -uroot -p#{PWD} db-name > #{DATE}.sql
+                        日期变量 #{DATE} ,下方的密码变量 #{PWD} ,下方的对象存储变量: #{Endpoint} #{AccessKey} #{SecretKey} #{BucketName}
+                        <br/>例: mysqldump -h192.168.1.11 -uroot -p#{PWD} db-name > #{DATE}.sql <a target="blank" href="https://github.com/jeessy2/backup-x#备份脚本参考">备份脚本参考</a> 
                       </small>
                     </div>
                   </div>
 
+                  <div class="form-group row">
+                    <label for="BackupType_{{$i}}" class="col-sm-2">备份类型</label>
+                    <div class="col-sm-10">
+                      <select class="form-control" name="BackupType" id="BackupType_{{$i}}" value="{{$v.BackupType}}">
+                        <option value="0" {{if eq $v.BackupType 0}}selected{{end}}>备份数据库</option>
+                        <option value="1" {{if eq $v.BackupType 1}}selected{{end}}>同步文件</option>
+                      </select>
+                      <small id="BackupType_help" class="form-text text-muted">如果没有输出文件, 请选择同步文件</small>
+                    </div>
+                  </div>
+
                   <div class="form-group row">
                     <label for="Pwd_{{$i}}" class="col-sm-2 col-form-label">密码变量</label>
                     <div class="col-sm-10">
@@ -96,7 +107,7 @@
 
                   <div class="form-group row">
                     <label for="StartTime_{{$i}}" class="col-sm-2 col-form-label">备份起始时间</label>
-                    <div class="col-sm-10">
+                    <div class="col-sm-4">
                       <select class="form-control" name="StartTime" id="StartTime_{{$i}}" value="{{$v.StartTime}}">
                         <option value="0" {{if eq $v.StartTime 0}}selected{{end}}>0:00</option>
                         <option value="1" {{if eq $v.StartTime 1}}selected{{end}}>1:00</option>
@@ -124,11 +135,8 @@
                         <option value="23" {{if eq $v.StartTime 23}}selected{{end}}>23:00</option>
                       </select>
                     </div>
-                  </div>
-
-                  <div class="form-group row">
                     <label for="Period_{{$i}}" class="col-sm-2 col-form-label">备份周期(分钟)</label>
-                    <div class="col-sm-10">
+                    <div class="col-sm-4">
                       <input type="number" class="form-control" name="Period" id="Period_{{$i}}" value="{{$v.Period}}" min="1">
                     </div>
                   </div>