From 0f5c099a17ce9ec490ad6d6a5140ad6a256e40ce Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Tue, 21 Jul 2020 11:45:02 +0200 Subject: [PATCH 1/7] =?UTF-8?q?Vrs=20funcional=20sem=20Restic=20para=20ava?= =?UTF-8?q?lia=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 34 +++++ LICENSE | 2 +- README.md | 74 ++++++++++- docker-compose.yml | 32 +++++ main.go | 19 +++ schelly-mysql/main.go | 289 ++++++++++++++++++++++++++++++++++++++++++ startup.sh | 7 + 7 files changed, 454 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 main.go create mode 100644 schelly-mysql/main.go create mode 100644 startup.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..71bc209 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM golang:1.10 AS BUILD + +#install external dependencies first +ADD /main.go $GOPATH/src/schelly-mysql/main.go +RUN go get -v schelly-mysql + +#now build source code +ADD schelly-mysql $GOPATH/src/schelly-mysql +RUN go get -v schelly-mysql + + +FROM ubuntu:18.04 + +RUN apt-get update +RUN apt-get install -y ca-certificates + +EXPOSE 7070 + +ENV LOG_LEVEL 'debug' + +ENV S3_PATH /mysql +ENV S3_BUCKET bem-backups-dev +ENV S3_REGION us-west-1 +ENV DUMP_CONNECTION_NAME bem_saude +ENV DUMP_CONNECTION_HOST docker.for.win.localhost:3306 +ENV DUMP_CONNECTION_AUTH_USERNAME bem_saude +ENV DUMP_CONNECTION_AUTH_PASSWORD bem_saude +ENV AWS_ACCESS_KEY_ID aaaaaaaaaa +ENV AWS_SECRET_ACCESS_KEY aaaaaaaaaa + +COPY --from=BUILD /go/bin/* /bin/ +ADD startup.sh / + +CMD [ "/startup.sh" ] diff --git a/LICENSE b/LICENSE index 0619b91..f8b2a09 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 stutzlab +Copyright (c) 2018 Flavio Stutz Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index e205c48..a45d8a6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,72 @@ -# schelly-restic-mysql -Schelly Restic MySQL container for doing MySQL backups using dumps and Restic storages +# schelly-mysql + +Backup MySQL + +See more about Schelly at http://github.com/flaviostutz/schelly + +# Usage + +docker-compose .yml + +``` +version: '3.5' + +services: + + # schelly: + # image: flaviostutz/schelly + # ports: + # - 8080:8080 + # environment: + # - LOG_LEVEL=debug + # - BACKUP_NAME=schelly-pgdump + # - WEBHOOK_URL=http://localhost:7070/backups + # - BACKUP_CRON_STRING=0 */1 * * * * + # - RETENTION_MINUTELY=5 + # - WEBHOOK_GRACE_TIME=20 + + mysql-api: + build: . + ports: + - 7070:7070 + environment: + - LOG_LEVEL=debug + - S3_PATH=/mysql + - S3_BUCKET=bucket + - S3_REGION=us-west-1 + - DUMP_CONNECTION_NAME=name + - DUMP_CONNECTION_HOST=server:port + - DUMP_CONNECTION_AUTH_USERNAME=user + - DUMP_CONNECTION_AUTH_PASSWORD=pass + - AWS_ACCESS_KEY_ID=key + - AWS_SECRET_ACCESS_KEY=sec +``` + +* execute ```docker-compose up``` and see logs + +* run: + +``` +// #create a new backup +// curl POST localhost:7070/backups + +// #list all backups +// curl localhost:7070/backups + +// #list existing backup +// curl localhost:7070/backups/abc123 + +// #remove all backups +// curl DELETE localhost:7070/backups + +// #remove existing backup +// curl DELETE localhost:7070/backups/abc123 + +// #download all backups +// curl COPY localhost:7070/backups + +// #download existing backup +// curl COPY localhost:7070/backups/abc123 + +``` + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1d18471 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3.5' + +services: + + # schelly: + # image: flaviostutz/schelly + # ports: + # - 8080:8080 + # environment: + # - LOG_LEVEL=debug + # - BACKUP_NAME=schelly-pgdump + # - WEBHOOK_URL=http://localhost:7070/backups + # - BACKUP_CRON_STRING=0 */1 * * * * + # - RETENTION_MINUTELY=5 + # - WEBHOOK_GRACE_TIME=20 + + mysql-api: + build: . + ports: + - 7070:7070 + environment: + - LOG_LEVEL=debug + - S3_PATH=/mysql + - S3_BUCKET=bem-backups-dev + - S3_REGION=us-west-1 + - DUMP_CONNECTION_NAME=bem_saude + - DUMP_CONNECTION_HOST=docker.for.win.localhost:3306 + - DUMP_CONNECTION_AUTH_USERNAME=bem_saude + - DUMP_CONNECTION_AUTH_PASSWORD=bem_saude + - AWS_ACCESS_KEY_ID=aaaaaaaaaa + - AWS_SECRET_ACCESS_KEY=aaaaaaaaaa + \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..45f2522 --- /dev/null +++ b/main.go @@ -0,0 +1,19 @@ +package main + +import ( + _ "encoding/json" + _ "flag" + _ "fmt" + _ "io/ioutil" + "log" + _ "net/http" + _ "os" + _ "regexp" + _ "strings" + + _ "github.com/sirupsen/logrus" +) + +func main() { + log.Print("Should not start this class.") +} diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go new file mode 100644 index 0000000..b5b184a --- /dev/null +++ b/schelly-mysql/main.go @@ -0,0 +1,289 @@ + +package main + +import ( + "fmt" + "log" + "net/http" + "os" + "strings" + "bytes" + "path" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + + "github.com/sirupsen/logrus" + + "database/sql" + "github.com/go-sql-driver/mysql" + "github.com/jamf/go-mysqldump" +) + +// ---------------------------------------------------------------------------------------------------- + +func main() { + http.HandleFunc("/backups", router) + http.HandleFunc("/backups/", router) + log.Fatal(http.ListenAndServe(":7070", nil)) +} +func router(w http.ResponseWriter, r *http.Request) { + S3_Key_MySQL := "" + if strings.Compare(r.URL.Path, "/backups") == 0 { + S3_Key_MySQL = strings.Replace(r.URL.Path, "/backups", "", 1) + } else { + S3_Key_MySQL = strings.Replace(r.URL.Path, "/backups/", "", 1) + } + switch r.Method { + case "GET": + List(S3_Key_MySQL); + case "POST": + Mysqldump(); + case "DELETE": + Delete(S3_Key_MySQL); + case "COPY": + Download(S3_Key_MySQL); + default: + fmt.Fprintf(w, "Sorry, only GET, POST, DELETE & DOWNLOAD methods are supported.") + } +} + +// ---------------------------------------------------------------------------------------------------- + +var ( + S3_REGION = os.Getenv("S3_REGION") + S3_BUCKET = os.Getenv("S3_BUCKET") + S3_PATH = os.Getenv("S3_PATH") + DUMP_CONNECTION_NAME = os.Getenv("DUMP_CONNECTION_NAME") + DUMP_CONNECTION_HOST = os.Getenv("DUMP_CONNECTION_HOST") + DUMP_CONNECTION_AUTH_USERNAME = os.Getenv("DUMP_CONNECTION_AUTH_USERNAME") + DUMP_CONNECTION_AUTH_PASSWORD = os.Getenv("DUMP_CONNECTION_AUTH_PASSWORD") + AWS_ACCESS_KEY_ID = os.Getenv("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = os.Getenv("AWS_SECRET_ACCESS_KEY") +) +var sess = connectAWS() +func connectAWS() *session.Session { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(S3_REGION), + Credentials: credentials.NewStaticCredentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, ""), + }) + if err != nil { + panic(err) + } + return sess +} + +func ClearDir(dir string) error { + dirRead, err := os.Open(dir) + if err != nil { + return err + } + dirFiles, err := dirRead.Readdir(0) + if err != nil { + return err + } + for index := range dirFiles { + entery := dirFiles[index] + filename := entery.Name() + fullPath := path.Join(dir, filename) + os.Remove(fullPath) + } + return nil +} + +// ---------------------------------------------------------------------------------------------------- + +func Mysqldump(){ + // Open connection to database + config := mysql.NewConfig() + config.User = DUMP_CONNECTION_AUTH_USERNAME + config.Passwd = DUMP_CONNECTION_AUTH_PASSWORD + config.DBName = DUMP_CONNECTION_NAME + config.Net = "tcp" + config.Addr = DUMP_CONNECTION_HOST + dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", config.DBName) // accepts time layout string and add .sql at the end of file + err := os.Remove(S3_PATH) + if err := os.MkdirAll(S3_PATH, 0755); err != nil { + logrus.Errorf("Error mkdir: %s", err) + return + } + db, err := sql.Open("mysql", config.FormatDSN()) + if err != nil { + logrus.Errorf("Error opening database: %s", err) + return + } + // Register database with mysqldump + dumper, err := mysqldump.Register(db, S3_PATH, dumpFilenameFormat) + if err != nil { + logrus.Errorf("Error registering databse: %s", err) + return + } + // Dump database to file + if err := dumper.Dump(); err != nil { + logrus.Errorf("Error dumping: %s", err) + return + } + if file, ok := dumper.Out.(*os.File); ok { + logrus.Infof("Successfully mysqldump...") + UploadS3(file.Name()) + err := ClearDir(S3_PATH) + if err!=nil{ + logrus.Errorf("Error ClearDir: %s", err) + } + } else { + logrus.Errorf("It's not part of *os.File, but dump is done") + } + // Close dumper, connected database and file stream. + dumper.Close() +} + +// ----------------------------------------------------------------- + +func UploadS3(S3_Key_MySQL string){ + file, err := os.Open(S3_Key_MySQL) + if err != nil { + logrus.Errorf("File not opened: %q", err) + return + } + // Get file size and read the file content into a buffer + fileInfo, _ := file.Stat() + var size int64 = fileInfo.Size() + buffer := make([]byte, size) + file.Read(buffer) + svc := s3.New(sess) + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(S3_BUCKET), + Key: aws.String(S3_Key_MySQL), + Body: bytes.NewReader(buffer), + ContentLength: aws.Int64(size), + ContentType: aws.String(http.DetectContentType(buffer)), + ContentDisposition: aws.String("attachment"), + //ServerSideEncryption: aws.String("AES256"), + }) + if err != nil { + logrus.Errorf("Something went wrong uploading the file: %q", err) + return + } + logrus.Infof("Successfully uploaded to %s", S3_BUCKET) + return +} + +// ---------------------------------------------------------------------------------------------------- + +func ListAll() []*s3.Object { + return List("") +} +func List(S3_Key_MySQL string) []*s3.Object { + svc := s3.New(sess) + resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(S3_BUCKET)}) + if err != nil { + logrus.Errorf("Unable to list items in bucket: %s", err) + } + i := 1 + fmt.Println("------------------- Start List ----------------------") + for _, item := range resp.Contents { + if strings.Compare(S3_Key_MySQL, *item.Key) == 0 { + fmt.Println("S3_Key: ", *item.Key) + fmt.Println("Last modified:", *item.LastModified) + fmt.Println("Size: ", *item.Size) + fmt.Println("Storage class:", *item.StorageClass) + fmt.Println("") + } + if len(S3_Key_MySQL) == 0 { + fmt.Println("", i) + fmt.Println("S3_Key: ", *item.Key) + fmt.Println("Last modified:", *item.LastModified) + fmt.Println("Size: ", *item.Size) + fmt.Println("Storage class:", *item.StorageClass) + fmt.Println("") + } + i++ + } + fmt.Println("------------------- End List ----------------------") + return resp.Contents +} + +// ---------------------------------------------------------------------------------------------------- + +func DownloadAll(){ + Download("") + return +} +func Download(S3_Key_MySQL string){ + downloader := s3manager.NewDownloader(sess) + if len(S3_Key_MySQL) == 0 { + listAll := ListAll() + for _, item := range listAll { + S3_Key_MySQL := *item.Key + file, err := os.Create(S3_Key_MySQL) + if err != nil { + logrus.Errorf("Unable to open file: %q", err) + } + defer file.Close() + + _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(S3_BUCKET), + Key: aws.String(S3_Key_MySQL), + }) + if err != nil { + logrus.Errorf("Something went wrong retrieving the file from S3> %q", err) + return + } + } + } else { + file, err := os.Create(S3_Key_MySQL) + if err != nil { + logrus.Errorf("Unable to open file: %q", err) + } + defer file.Close() + + _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(S3_BUCKET), + Key: aws.String(S3_Key_MySQL), + }) + if err != nil { + logrus.Errorf("Something went wrong retrieving the file from S3: %q", err) + return + } + } + logrus.Infof("Downloaded") + return +} + +// ---------------------------------------------------------------------------------------------------- + +func DeleteAll(){ + Delete("") + return +} +func Delete(S3_Key_MySQL string){ + logrus.Infof("S3_Key_MySQL: %s", S3_Key_MySQL) + svc := s3.New(sess) + if len(S3_Key_MySQL) == 0 { + iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ + Bucket: aws.String(S3_BUCKET), + }) + if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { + logrus.Errorf("Unable to delete objects: %q", err) + } + } else { + var err error + _, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(S3_BUCKET), Key: aws.String(S3_Key_MySQL)}) + if err != nil { + logrus.Errorf("Unable to delete object: %q", err) + } + err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(S3_BUCKET), + Key: aws.String(S3_Key_MySQL), + }) + if err != nil { + logrus.Errorf("Unable to delete objects: %q", err) + return + } + } + logrus.Infof("Deleted object(s) from bucket: %s", S3_BUCKET) + return +} \ No newline at end of file diff --git a/startup.sh b/startup.sh new file mode 100644 index 0000000..5a05fa7 --- /dev/null +++ b/startup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +# set -x + +echo "Starting Mysqldump..." +schelly-mysql \ + --log-level=$LOG_LEVEL From dbd373d9ac116e2338a136b91c3b2a18663b1d08 Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Tue, 21 Jul 2020 15:07:36 +0200 Subject: [PATCH 2/7] Ajustes Docker com Schelly - Issue #120 --- README.md | 22 +++++++++++----------- docker-compose.yml | 26 +++++++++++++------------- schelly-mysql/main.go | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index a45d8a6..6ad6ee2 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ version: '3.5' services: - # schelly: - # image: flaviostutz/schelly - # ports: - # - 8080:8080 - # environment: - # - LOG_LEVEL=debug - # - BACKUP_NAME=schelly-pgdump - # - WEBHOOK_URL=http://localhost:7070/backups - # - BACKUP_CRON_STRING=0 */1 * * * * - # - RETENTION_MINUTELY=5 - # - WEBHOOK_GRACE_TIME=20 + schelly: + image: flaviostutz/schelly + ports: + - 8080:8080 + environment: + - LOG_LEVEL=debug + - BACKUP_NAME=schelly-pgdump + - WEBHOOK_URL=http://localhost:7070/backups + - BACKUP_CRON_STRING=0 */1 * * * * + - RETENTION_MINUTELY=5 + - WEBHOOK_GRACE_TIME=20 mysql-api: build: . diff --git a/docker-compose.yml b/docker-compose.yml index 1d18471..85efc47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,17 +2,17 @@ version: '3.5' services: - # schelly: - # image: flaviostutz/schelly - # ports: - # - 8080:8080 - # environment: - # - LOG_LEVEL=debug - # - BACKUP_NAME=schelly-pgdump - # - WEBHOOK_URL=http://localhost:7070/backups - # - BACKUP_CRON_STRING=0 */1 * * * * - # - RETENTION_MINUTELY=5 - # - WEBHOOK_GRACE_TIME=20 + schelly: + image: flaviostutz/schelly + ports: + - 8080:8080 + environment: + - LOG_LEVEL=debug + - BACKUP_NAME=schelly-pgdump + - WEBHOOK_URL=http://localhost:7070/backups + - BACKUP_CRON_STRING=0 */1 * * * * + - RETENTION_MINUTELY=5 + - WEBHOOK_GRACE_TIME=20 mysql-api: build: . @@ -27,6 +27,6 @@ services: - DUMP_CONNECTION_HOST=docker.for.win.localhost:3306 - DUMP_CONNECTION_AUTH_USERNAME=bem_saude - DUMP_CONNECTION_AUTH_PASSWORD=bem_saude - - AWS_ACCESS_KEY_ID=aaaaaaaaaa - - AWS_SECRET_ACCESS_KEY=aaaaaaaaaa + - AWS_ACCESS_KEY_ID=aaaaaaaa + - AWS_SECRET_ACCESS_KEY=aaaaaaaa \ No newline at end of file diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index b5b184a..8138300 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -47,7 +47,7 @@ func router(w http.ResponseWriter, r *http.Request) { case "COPY": Download(S3_Key_MySQL); default: - fmt.Fprintf(w, "Sorry, only GET, POST, DELETE & DOWNLOAD methods are supported.") + fmt.Fprintf(w, "Sorry, only GET, POST, DELETE & COPY methods are supported.") } } From 44d528aad625214390363c30c95a845ac2bc0efc Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Tue, 21 Jul 2020 16:42:14 +0200 Subject: [PATCH 3/7] Retirado Delete All - Issue #120 --- README.md | 3 --- schelly-mysql/main.go | 15 +++------------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6ad6ee2..767096a 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,6 @@ services: // #list existing backup // curl localhost:7070/backups/abc123 -// #remove all backups -// curl DELETE localhost:7070/backups - // #remove existing backup // curl DELETE localhost:7070/backups/abc123 diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index 8138300..9e678a5 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -255,20 +255,11 @@ func Download(S3_Key_MySQL string){ // ---------------------------------------------------------------------------------------------------- -func DeleteAll(){ - Delete("") - return -} func Delete(S3_Key_MySQL string){ logrus.Infof("S3_Key_MySQL: %s", S3_Key_MySQL) svc := s3.New(sess) if len(S3_Key_MySQL) == 0 { - iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ - Bucket: aws.String(S3_BUCKET), - }) - if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { - logrus.Errorf("Unable to delete objects: %q", err) - } + logrus.Errorf("Unable to delete without 'key'") } else { var err error _, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(S3_BUCKET), Key: aws.String(S3_Key_MySQL)}) @@ -280,10 +271,10 @@ func Delete(S3_Key_MySQL string){ Key: aws.String(S3_Key_MySQL), }) if err != nil { - logrus.Errorf("Unable to delete objects: %q", err) + logrus.Errorf("Unable to delete object: %q", err) return } } - logrus.Infof("Deleted object(s) from bucket: %s", S3_BUCKET) + logrus.Infof("Deleted object from bucket: %s", S3_BUCKET) return } \ No newline at end of file From e2652ac6e00efa6a541369bc7aba4e30c6826df3 Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Wed, 22 Jul 2020 07:57:59 +0200 Subject: [PATCH 4/7] + return json GET & POST --- README.md | 7 ---- schelly-mysql/main.go | 92 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 767096a..26e74d0 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,5 @@ services: // #remove existing backup // curl DELETE localhost:7070/backups/abc123 - -// #download all backups -// curl COPY localhost:7070/backups - -// #download existing backup -// curl COPY localhost:7070/backups/abc123 - ``` diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index 9e678a5..d59490d 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -21,6 +21,8 @@ import ( "database/sql" "github.com/go-sql-driver/mysql" "github.com/jamf/go-mysqldump" + + "encoding/json" ) // ---------------------------------------------------------------------------------------------------- @@ -38,21 +40,76 @@ func router(w http.ResponseWriter, r *http.Request) { S3_Key_MySQL = strings.Replace(r.URL.Path, "/backups/", "", 1) } switch r.Method { - case "GET": - List(S3_Key_MySQL); + case "GET": + resp := List(S3_Key_MySQL); + listsJson := ListsJson{}; listJson := ListJson{}; + i := 0; ir := 0; + for _, item := range resp { + if strings.Compare(S3_Key_MySQL, *item.Key) == 0 { + listJson = ListJson{Id: *item.Key, Data_id:*item.Key, Status: "available", Message: *item.StorageClass, Size_mb: *item.Size,} + ir++ + } + if len(S3_Key_MySQL) == 0 { + listsJson = append(listsJson, ListJson{Id: *item.Key, Data_id:*item.Key, Status: "available", Message: *item.StorageClass, Size_mb: *item.Size,}) + } + i++ + } + w.Header().Set("Content-Type","application/json") + if i == 0 { + w.WriteHeader(404) // 404 notFoundHtmlFile + } else { + w.WriteHeader(200) // 200 http.StatusOK + if ir == 1 { + if err := json.NewEncoder(w).Encode(listJson); err != nil {panic(err)} + } else { + if err := json.NewEncoder(w).Encode(listsJson); err != nil {panic(err)} + } + } case "POST": - Mysqldump(); + resp := Mysqldump(); + dumpJson := DumpJson{} + if len(resp) == 0 { + dumpJson = DumpJson{Id: "0", Status: "error", Message: "",} + } else { + dumpJson = DumpJson{Id: resp, Status: "available", Message: "",} + } + w.Header().Set("Content-Type","application/json") + if len(resp) == 0 { + w.WriteHeader(404) // 404 notFoundHtmlFile + } else { + w.WriteHeader(200) // 200 http.StatusOK + } + if err := json.NewEncoder(w).Encode(dumpJson); err != nil { + panic(err) + } case "DELETE": Delete(S3_Key_MySQL); case "COPY": - Download(S3_Key_MySQL); + //Download(S3_Key_MySQL); + fmt.Fprintf(w, "Sorry, only GET, POST & DELETE methods are supported.") default: - fmt.Fprintf(w, "Sorry, only GET, POST, DELETE & COPY methods are supported.") + fmt.Fprintf(w, "Sorry, only GET, POST & DELETE methods are supported.") } } // ---------------------------------------------------------------------------------------------------- +type DumpJson struct { + Id string `json:"id"` + Status string `json:"status"` + Message string `json:"message"` +} +type ListJson struct { + Id string `json:"id"` + Data_id string `json:"data_id"` + Status string `json:"status"` + Message string `json:"message"` + Size_mb int64 `json:"size_mb"` +} +type ListsJson []ListJson + +// ---------------------------------------------------------------------------------------------------- + var ( S3_REGION = os.Getenv("S3_REGION") S3_BUCKET = os.Getenv("S3_BUCKET") @@ -96,7 +153,7 @@ func ClearDir(dir string) error { // ---------------------------------------------------------------------------------------------------- -func Mysqldump(){ +func Mysqldump() string{ // Open connection to database config := mysql.NewConfig() config.User = DUMP_CONNECTION_AUTH_USERNAME @@ -108,45 +165,42 @@ func Mysqldump(){ err := os.Remove(S3_PATH) if err := os.MkdirAll(S3_PATH, 0755); err != nil { logrus.Errorf("Error mkdir: %s", err) - return + return "" } db, err := sql.Open("mysql", config.FormatDSN()) if err != nil { logrus.Errorf("Error opening database: %s", err) - return + return "" } // Register database with mysqldump dumper, err := mysqldump.Register(db, S3_PATH, dumpFilenameFormat) if err != nil { logrus.Errorf("Error registering databse: %s", err) - return + return "" } // Dump database to file if err := dumper.Dump(); err != nil { logrus.Errorf("Error dumping: %s", err) - return + return "" } if file, ok := dumper.Out.(*os.File); ok { logrus.Infof("Successfully mysqldump...") - UploadS3(file.Name()) - err := ClearDir(S3_PATH) - if err!=nil{ - logrus.Errorf("Error ClearDir: %s", err) - } + return UploadS3(file.Name()) } else { logrus.Errorf("It's not part of *os.File, but dump is done") } // Close dumper, connected database and file stream. dumper.Close() + return "" } // ----------------------------------------------------------------- -func UploadS3(S3_Key_MySQL string){ +func UploadS3(S3_Key_MySQL string) string{ file, err := os.Open(S3_Key_MySQL) if err != nil { logrus.Errorf("File not opened: %q", err) - return + return "" } // Get file size and read the file content into a buffer fileInfo, _ := file.Stat() @@ -165,10 +219,10 @@ func UploadS3(S3_Key_MySQL string){ }) if err != nil { logrus.Errorf("Something went wrong uploading the file: %q", err) - return + return "" } logrus.Infof("Successfully uploaded to %s", S3_BUCKET) - return + return S3_Key_MySQL } // ---------------------------------------------------------------------------------------------------- From 66b49c414ef8334a142f0d6c7b7321894561b12d Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Sun, 16 Aug 2020 17:46:28 +0200 Subject: [PATCH 5/7] =?UTF-8?q?commit=20inicial=20com=20shellyhook=20para?= =?UTF-8?q?=20avalia=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schelly-mysql/main.go | 145 +++++++++++++++++++------------------ schelly-mysql/main_test.go | 15 ++++ 2 files changed, 88 insertions(+), 72 deletions(-) create mode 100644 schelly-mysql/main_test.go diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index d59490d..28ad683 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -1,9 +1,7 @@ - package main import ( "fmt" - "log" "net/http" "os" "strings" @@ -22,91 +20,94 @@ import ( "github.com/go-sql-driver/mysql" "github.com/jamf/go-mysqldump" - "encoding/json" + "github.com/flaviostutz/schelly-webhook/schellyhook" + "time" ) // ---------------------------------------------------------------------------------------------------- -func main() { - http.HandleFunc("/backups", router) - http.HandleFunc("/backups/", router) - log.Fatal(http.ListenAndServe(":7070", nil)) +type MySQLBackuper struct{} + +func (sb MySQLBackuper) CreateNewBackup(apiID string, timeout time.Duration, shellContext *schellyhook.ShellContext) error { + // resp := Mysqldump(); + // backup := schellyhook.SchellyResponse{ + // ID: "0", + // Status: "error", + // Message: "", + // } + // if len(resp) != 0 { + // backup = schellyhook.SchellyResponse{ + // ID: resp, + // Status: "available", + // Message: "", + // } + // } + + return nil } -func router(w http.ResponseWriter, r *http.Request) { - S3_Key_MySQL := "" - if strings.Compare(r.URL.Path, "/backups") == 0 { - S3_Key_MySQL = strings.Replace(r.URL.Path, "/backups", "", 1) - } else { - S3_Key_MySQL = strings.Replace(r.URL.Path, "/backups/", "", 1) - } - switch r.Method { - case "GET": + +func (sb MySQLBackuper) GetAllBackups() ([]schellyhook.SchellyResponse, error) { + S3_Key_MySQL := "" resp := List(S3_Key_MySQL); - listsJson := ListsJson{}; listJson := ListJson{}; - i := 0; ir := 0; + if len(resp) == 0 { + return nil, nil + } + backups := make([]schellyhook.SchellyResponse, 0) for _, item := range resp { - if strings.Compare(S3_Key_MySQL, *item.Key) == 0 { - listJson = ListJson{Id: *item.Key, Data_id:*item.Key, Status: "available", Message: *item.StorageClass, Size_mb: *item.Size,} - ir++ - } - if len(S3_Key_MySQL) == 0 { - listsJson = append(listsJson, ListJson{Id: *item.Key, Data_id:*item.Key, Status: "available", Message: *item.StorageClass, Size_mb: *item.Size,}) - } - i++ - } - w.Header().Set("Content-Type","application/json") - if i == 0 { - w.WriteHeader(404) // 404 notFoundHtmlFile - } else { - w.WriteHeader(200) // 200 http.StatusOK - if ir == 1 { - if err := json.NewEncoder(w).Encode(listJson); err != nil {panic(err)} - } else { - if err := json.NewEncoder(w).Encode(listsJson); err != nil {panic(err)} + S3key := *item.Key + S3Size := *item.Size + S3Msg := *item.StorageClass + sr := schellyhook.SchellyResponse{ + ID: S3key, + DataID: S3key, + Status: "available", + Message: S3Msg, + SizeMB: float64(S3Size), } - } - case "POST": - resp := Mysqldump(); - dumpJson := DumpJson{} - if len(resp) == 0 { - dumpJson = DumpJson{Id: "0", Status: "error", Message: "",} - } else { - dumpJson = DumpJson{Id: resp, Status: "available", Message: "",} + backups = append(backups, sr) } - w.Header().Set("Content-Type","application/json") + return backups, nil +} + +func (sb MySQLBackuper) GetBackup(apiID string) (*schellyhook.SchellyResponse, error) { + S3_Key_MySQL := apiID + resp := List(S3_Key_MySQL); if len(resp) == 0 { - w.WriteHeader(404) // 404 notFoundHtmlFile - } else { - w.WriteHeader(200) // 200 http.StatusOK - } - if err := json.NewEncoder(w).Encode(dumpJson); err != nil { - panic(err) - } - case "DELETE": - Delete(S3_Key_MySQL); - case "COPY": - //Download(S3_Key_MySQL); - fmt.Fprintf(w, "Sorry, only GET, POST & DELETE methods are supported.") - default: - fmt.Fprintf(w, "Sorry, only GET, POST & DELETE methods are supported.") - } + return nil, nil + } + S3key := *resp[0].Key + S3Size := *resp[0].Size + S3Msg := *resp[0].StorageClass + return &schellyhook.SchellyResponse{ + ID: S3key, + DataID: S3key, + Status: "available", + Message: S3Msg, + SizeMB: float64(S3Size), + }, nil } -// ---------------------------------------------------------------------------------------------------- +func (sb MySQLBackuper) DeleteBackup(apiID string) error { + Delete(apiID); + return nil +} -type DumpJson struct { - Id string `json:"id"` - Status string `json:"status"` - Message string `json:"message"` +func main() { + logrus.Info("====Starting server====") + mySQLBackuper := MySQLBackuper{} + err := schellyhook.Initialize(mySQLBackuper) + if err != nil { + logrus.Errorf("Error initializating Schellyhook. err=%s", err) + os.Exit(1) + } } -type ListJson struct { - Id string `json:"id"` - Data_id string `json:"data_id"` - Status string `json:"status"` - Message string `json:"message"` - Size_mb int64 `json:"size_mb"` + +func (sb MySQLBackuper) Init() error { + return nil +} +func (sb MySQLBackuper) RegisterFlags() error { + return nil } -type ListsJson []ListJson // ---------------------------------------------------------------------------------------------------- diff --git a/schelly-mysql/main_test.go b/schelly-mysql/main_test.go new file mode 100644 index 0000000..524d412 --- /dev/null +++ b/schelly-mysql/main_test.go @@ -0,0 +1,15 @@ +package main + +import ( + "testing" + + "go.uber.org/zap" +) + +func Test(t *testing.T) { + logger, _ := zap.NewDevelopment() + defer logger.Sync() // flushes buffer, if any + sugar := logger.Sugar() + + sugar.Infof("Starting Test...") +} From 38f14fbf4b40f967de590e59a448b17112d4936a Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Sun, 16 Aug 2020 21:14:11 +0200 Subject: [PATCH 6/7] Implementado com schellyhook --- Dockerfile | 6 +- README.md | 2 +- docker-compose.yml | 6 +- schelly-mysql/main.go | 218 +++++++++++++++--------------------------- 4 files changed, 86 insertions(+), 146 deletions(-) diff --git a/Dockerfile b/Dockerfile index 71bc209..8ca4e1d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,15 +18,15 @@ EXPOSE 7070 ENV LOG_LEVEL 'debug' -ENV S3_PATH /mysql +ENV S3_PATH mysql ENV S3_BUCKET bem-backups-dev ENV S3_REGION us-west-1 ENV DUMP_CONNECTION_NAME bem_saude ENV DUMP_CONNECTION_HOST docker.for.win.localhost:3306 ENV DUMP_CONNECTION_AUTH_USERNAME bem_saude ENV DUMP_CONNECTION_AUTH_PASSWORD bem_saude -ENV AWS_ACCESS_KEY_ID aaaaaaaaaa -ENV AWS_SECRET_ACCESS_KEY aaaaaaaaaa +ENV AWS_ACCESS_KEY_ID key +ENV AWS_SECRET_ACCESS_KEY sec COPY --from=BUILD /go/bin/* /bin/ ADD startup.sh / diff --git a/README.md b/README.md index 26e74d0..12e09a6 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ services: - 7070:7070 environment: - LOG_LEVEL=debug - - S3_PATH=/mysql + - S3_PATH=mysql - S3_BUCKET=bucket - S3_REGION=us-west-1 - DUMP_CONNECTION_NAME=name diff --git a/docker-compose.yml b/docker-compose.yml index 85efc47..2e0a986 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,13 +20,13 @@ services: - 7070:7070 environment: - LOG_LEVEL=debug - - S3_PATH=/mysql + - S3_PATH=mysql - S3_BUCKET=bem-backups-dev - S3_REGION=us-west-1 - DUMP_CONNECTION_NAME=bem_saude - DUMP_CONNECTION_HOST=docker.for.win.localhost:3306 - DUMP_CONNECTION_AUTH_USERNAME=bem_saude - DUMP_CONNECTION_AUTH_PASSWORD=bem_saude - - AWS_ACCESS_KEY_ID=aaaaaaaa - - AWS_SECRET_ACCESS_KEY=aaaaaaaa + - AWS_ACCESS_KEY_ID=key + - AWS_SECRET_ACCESS_KEY=sec \ No newline at end of file diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index 28ad683..6ca867f 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "net/http" "os" "strings" @@ -12,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/sirupsen/logrus" @@ -20,7 +18,7 @@ import ( "github.com/go-sql-driver/mysql" "github.com/jamf/go-mysqldump" - "github.com/flaviostutz/schelly-webhook/schellyhook" + "github.com/flaviostutz/schelly-webhook" "time" ) @@ -29,67 +27,44 @@ import ( type MySQLBackuper struct{} func (sb MySQLBackuper) CreateNewBackup(apiID string, timeout time.Duration, shellContext *schellyhook.ShellContext) error { - // resp := Mysqldump(); - // backup := schellyhook.SchellyResponse{ - // ID: "0", - // Status: "error", - // Message: "", - // } - // if len(resp) != 0 { - // backup = schellyhook.SchellyResponse{ - // ID: resp, - // Status: "available", - // Message: "", - // } - // } - - return nil + // Remove the server files + e := ClearDir(S3_PATH) + if e != nil { + logrus.Errorf("Could not remove the server files: %q", e) + } + return Mysqldump(apiID); } func (sb MySQLBackuper) GetAllBackups() ([]schellyhook.SchellyResponse, error) { - S3_Key_MySQL := "" - resp := List(S3_Key_MySQL); + resp, err := List(""); + if err != nil { + return nil, err + } if len(resp) == 0 { return nil, nil - } - backups := make([]schellyhook.SchellyResponse, 0) - for _, item := range resp { - S3key := *item.Key - S3Size := *item.Size - S3Msg := *item.StorageClass - sr := schellyhook.SchellyResponse{ - ID: S3key, - DataID: S3key, - Status: "available", - Message: S3Msg, - SizeMB: float64(S3Size), - } - backups = append(backups, sr) - } - return backups, nil + } + return resp, nil } func (sb MySQLBackuper) GetBackup(apiID string) (*schellyhook.SchellyResponse, error) { - S3_Key_MySQL := apiID - resp := List(S3_Key_MySQL); + resp, err := List(getKey(apiID)); + if err != nil { + return nil, err + } if len(resp) == 0 { return nil, nil - } - S3key := *resp[0].Key - S3Size := *resp[0].Size - S3Msg := *resp[0].StorageClass + } return &schellyhook.SchellyResponse{ - ID: S3key, - DataID: S3key, - Status: "available", - Message: S3Msg, - SizeMB: float64(S3Size), + ID: resp[0].ID, + DataID: resp[0].DataID, + Status: resp[0].Status, + Message: resp[0].Message, + SizeMB: resp[0].SizeMB, }, nil } func (sb MySQLBackuper) DeleteBackup(apiID string) error { - Delete(apiID); - return nil + return Delete(getKey(apiID)); } func main() { @@ -133,6 +108,12 @@ func connectAWS() *session.Session { } return sess } +func lastString(ss []string) string { + return ss[len(ss)-1] +} +func getKey(t string) string { + return S3_PATH + "/" + t + ".sql" +} func ClearDir(dir string) error { dirRead, err := os.Open(dir) @@ -154,7 +135,7 @@ func ClearDir(dir string) error { // ---------------------------------------------------------------------------------------------------- -func Mysqldump() string{ +func Mysqldump(S3_Key string) error{ // Open connection to database config := mysql.NewConfig() config.User = DUMP_CONNECTION_AUTH_USERNAME @@ -162,27 +143,27 @@ func Mysqldump() string{ config.DBName = DUMP_CONNECTION_NAME config.Net = "tcp" config.Addr = DUMP_CONNECTION_HOST - dumpFilenameFormat := fmt.Sprintf("%s-20060102T150405", config.DBName) // accepts time layout string and add .sql at the end of file err := os.Remove(S3_PATH) if err := os.MkdirAll(S3_PATH, 0755); err != nil { logrus.Errorf("Error mkdir: %s", err) - return "" + return err } db, err := sql.Open("mysql", config.FormatDSN()) if err != nil { logrus.Errorf("Error opening database: %s", err) - return "" + return err } // Register database with mysqldump - dumper, err := mysqldump.Register(db, S3_PATH, dumpFilenameFormat) + dumper, err := mysqldump.Register(db, S3_PATH, S3_Key) + if err != nil { logrus.Errorf("Error registering databse: %s", err) - return "" + return err } // Dump database to file if err := dumper.Dump(); err != nil { logrus.Errorf("Error dumping: %s", err) - return "" + return err } if file, ok := dumper.Out.(*os.File); ok { logrus.Infof("Successfully mysqldump...") @@ -192,16 +173,16 @@ func Mysqldump() string{ } // Close dumper, connected database and file stream. dumper.Close() - return "" + return nil } // ----------------------------------------------------------------- -func UploadS3(S3_Key_MySQL string) string{ - file, err := os.Open(S3_Key_MySQL) +func UploadS3(S3_Key string) error{ + file, err := os.Open(S3_Key) if err != nil { logrus.Errorf("File not opened: %q", err) - return "" + return err } // Get file size and read the file content into a buffer fileInfo, _ := file.Stat() @@ -211,7 +192,7 @@ func UploadS3(S3_Key_MySQL string) string{ svc := s3.New(sess) _, err = svc.PutObject(&s3.PutObjectInput{ Bucket: aws.String(S3_BUCKET), - Key: aws.String(S3_Key_MySQL), + Key: aws.String(S3_Key), Body: bytes.NewReader(buffer), ContentLength: aws.Int64(size), ContentType: aws.String(http.DetectContentType(buffer)), @@ -220,116 +201,75 @@ func UploadS3(S3_Key_MySQL string) string{ }) if err != nil { logrus.Errorf("Something went wrong uploading the file: %q", err) - return "" + return err } logrus.Infof("Successfully uploaded to %s", S3_BUCKET) - return S3_Key_MySQL + file.Close() + return nil } // ---------------------------------------------------------------------------------------------------- -func ListAll() []*s3.Object { - return List("") -} -func List(S3_Key_MySQL string) []*s3.Object { +func List(S3_Key string) ([]schellyhook.SchellyResponse, error) { svc := s3.New(sess) resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(S3_BUCKET)}) if err != nil { logrus.Errorf("Unable to list items in bucket: %s", err) + return nil, err } - i := 1 - fmt.Println("------------------- Start List ----------------------") + backups := make([]schellyhook.SchellyResponse, 0) for _, item := range resp.Contents { - if strings.Compare(S3_Key_MySQL, *item.Key) == 0 { - fmt.Println("S3_Key: ", *item.Key) - fmt.Println("Last modified:", *item.LastModified) - fmt.Println("Size: ", *item.Size) - fmt.Println("Storage class:", *item.StorageClass) - fmt.Println("") - } - if len(S3_Key_MySQL) == 0 { - fmt.Println("", i) - fmt.Println("S3_Key: ", *item.Key) - fmt.Println("Last modified:", *item.LastModified) - fmt.Println("Size: ", *item.Size) - fmt.Println("Storage class:", *item.StorageClass) - fmt.Println("") - } - i++ - } - fmt.Println("------------------- End List ----------------------") - return resp.Contents -} - -// ---------------------------------------------------------------------------------------------------- - -func DownloadAll(){ - Download("") - return -} -func Download(S3_Key_MySQL string){ - downloader := s3manager.NewDownloader(sess) - if len(S3_Key_MySQL) == 0 { - listAll := ListAll() - for _, item := range listAll { - S3_Key_MySQL := *item.Key - file, err := os.Create(S3_Key_MySQL) - if err != nil { - logrus.Errorf("Unable to open file: %q", err) - } - defer file.Close() + S3key := *item.Key + S3Size := *item.Size + S3Msg := *item.StorageClass - _, err = downloader.Download(file, &s3.GetObjectInput{ - Bucket: aws.String(S3_BUCKET), - Key: aws.String(S3_Key_MySQL), - }) - if err != nil { - logrus.Errorf("Something went wrong retrieving the file from S3> %q", err) - return - } - } - } else { - file, err := os.Create(S3_Key_MySQL) - if err != nil { - logrus.Errorf("Unable to open file: %q", err) + if len(S3_Key) == 0 { + sr := schellyhook.SchellyResponse{ + ID: lastString(strings.Split(S3key, "/")), + DataID: S3key, + Status: "available", + Message: S3Msg, + SizeMB: float64(S3Size), + } + backups = append(backups, sr) } - defer file.Close() - - _, err = downloader.Download(file, &s3.GetObjectInput{ - Bucket: aws.String(S3_BUCKET), - Key: aws.String(S3_Key_MySQL), - }) - if err != nil { - logrus.Errorf("Something went wrong retrieving the file from S3: %q", err) - return + if strings.Compare(S3_Key, *item.Key) == 0 { + sr := schellyhook.SchellyResponse{ + ID: lastString(strings.Split(S3key, "/")), + DataID: S3key, + Status: "available", + Message: S3Msg, + SizeMB: float64(S3Size), + } + backups = append(backups, sr) } - } - logrus.Infof("Downloaded") - return + } + return backups, nil } // ---------------------------------------------------------------------------------------------------- -func Delete(S3_Key_MySQL string){ - logrus.Infof("S3_Key_MySQL: %s", S3_Key_MySQL) +func Delete(S3_Key string) error { + logrus.Infof("S3_Key: %s", S3_Key) svc := s3.New(sess) - if len(S3_Key_MySQL) == 0 { + if len(S3_Key) == 0 { logrus.Errorf("Unable to delete without 'key'") } else { var err error - _, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(S3_BUCKET), Key: aws.String(S3_Key_MySQL)}) + _, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(S3_BUCKET), Key: aws.String(S3_Key)}) if err != nil { logrus.Errorf("Unable to delete object: %q", err) + return err } err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ Bucket: aws.String(S3_BUCKET), - Key: aws.String(S3_Key_MySQL), + Key: aws.String(S3_Key), }) if err != nil { logrus.Errorf("Unable to delete object: %q", err) - return + return err } } logrus.Infof("Deleted object from bucket: %s", S3_BUCKET) - return + return nil } \ No newline at end of file From 11230d9eccabd066bf2b19777cd7d79cc547be8b Mon Sep 17 00:00:00 2001 From: RWdesenv Studio Date: Mon, 17 Aug 2020 07:54:37 +0200 Subject: [PATCH 7/7] Get list of objects from 'mysql' sub folder of Amazon S3 bucket --- schelly-mysql/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schelly-mysql/main.go b/schelly-mysql/main.go index 6ca867f..6c5754c 100644 --- a/schelly-mysql/main.go +++ b/schelly-mysql/main.go @@ -212,7 +212,7 @@ func UploadS3(S3_Key string) error{ func List(S3_Key string) ([]schellyhook.SchellyResponse, error) { svc := s3.New(sess) - resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(S3_BUCKET)}) + resp, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(S3_BUCKET), Prefix: aws.String(S3_PATH),}) if err != nil { logrus.Errorf("Unable to list items in bucket: %s", err) return nil, err