Option for retries when downloading the repo.yaml file

CHANGELOG.md

@@ -6,13 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 ## [Unreleased]
+
+
+## [0.2.1] - 2021-08-18
+
+### Added
+
 * Option for own slack prefix/emoji in notifications via `REPOW_SLACK_PREFIX`
+* Option for retries when downloading the repo.yaml file `REPOW_GITLAB_DOWNLOAD_RETRIES`/`GITLAB_DOWNLOAD_RETRIES`
 
 
 ## [0.2.0] - 2021-06-20
 
 ### Added
-* Also support REPOW_GITLAB_SECRET_TOKEN env variable additional to GITLAB_SECRET_TOKEN
+* Also support `REPOW_GITLAB_SECRET_TOKEN` env variable additional to `GITLAB_SECRET_TOKEN`
 * Quite option for validate command
 * Support for `gitlab.forking_access_level`
 * Optional contacts are allowed with the option `-c` in validate/apply and via `REPOW_OPTIONAL_CONTACTS` env variable in serve
@@ -55,7 +62,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
-[Unreleased]: https://github.com/galan/repow/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/galan/repow/compare/v0.2.1...HEAD
+[0.2.1]: https://github.com/galan/repow/compare/v0.2.0...v0.2.1
 [0.2.0]: https://github.com/galan/repow/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/galan/repow/compare/v0.0.2...v0.1.0
 [0.0.2]: https://github.com/galan/repow/compare/v0.0.1...v0.0.2

internal/hoster/gitlab/gitlab.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"strconv"
+	"time"
 
 	"repo/internal/hoster"
 	"repo/internal/model"
@@ -19,6 +21,10 @@ import (
 const REPOW_GITLAB_API_TOKEN = "REPOW_GITLAB_API_TOKEN"
 const GITLAB_API_TOKEN = "GITLAB_API_TOKEN"
 
+const REPOW_GITLAB_DOWNLOAD_RETRIES = "REPOW_GITLAB_DOWNLOAD_RETRIES"
+const GITLAB_DOWNLOAD_RETRIES = "GITLAB_DOWNLOAD_RETRIES"
+const GITLAB_DOWNLOAD_RETRIES_DEFAULT = 6 // lower values didn't solve the issue
+
 func MakeHoster() (*Gitlab, error) {
 	result := &Gitlab{}
 	value := util.GetEnv(REPOW_GITLAB_API_TOKEN, util.GetEnv(GITLAB_API_TOKEN, ""))
@@ -148,14 +154,14 @@ func (g Gitlab) ProjectState(projectPath string) (hoster.CleanupState, error) {
 func (g Gitlab) Validate(repo model.RepoMeta, optionalContacts bool) []error {
 	var errs []error
 	// repo.yaml itself
-	if repo.RepoYaml == nil {
-		errs = append(errs, errors.New("No repo.yaml file exists"))
-		return errs
-	}
 	if !repo.RepoYamlValid {
 		errs = append(errs, errors.New("Invalid repo.yaml file"))
 		return errs
 	}
+	if repo.RepoYaml == nil {
+		errs = append(errs, errors.New("No repo.yaml file exists"))
+		return errs
+	}
 
 	pattern := `^[a-z][a-z0-9-]{0,99}$`
 
@@ -229,24 +235,10 @@ func (g Gitlab) contactExists(remotePath, contact string) bool {
 func (g Gitlab) DownloadRepoyaml(remotePath string, branch string) (*model.RepoYaml, bool, error) {
 	say.Verbose("Downloading repo.yaml for project %s", remotePath)
 
-	gfo := &gg.GetFileOptions{
-		Ref: gg.String(branch),
-	}
-	file, response, err := g.client.RepositoryFiles.GetFile(remotePath, model.RepoYamlFilename, gfo)
-
+	file, err := downloadFile(g, remotePath, branch)
 	if err != nil {
 		return nil, false, err
 	}
-	if response == nil {
-		return nil, false, errors.New("No gitlab server response")
-	}
-	if response.StatusCode == 404 {
-		return nil, false, nil
-	}
-	if response.StatusCode != 200 {
-		say.Verbose("Unable to download repository manifest: %d", response.StatusCode)
-		return nil, false, errors.New(fmt.Sprintf("Statuscode %d", response.StatusCode))
-	}
 
 	contentDecoded, errDecode := base64.StdEncoding.DecodeString(file.Content)
 	if errDecode != nil {
@@ -263,6 +255,46 @@ func (g Gitlab) DownloadRepoyaml(remotePath string, branch string) (*model.RepoY
 	return result, true, nil
 }
 
+func downloadFile(g Gitlab, remotePath string, branch string) (*gg.File, error) {
+	gfo := &gg.GetFileOptions{
+		Ref: gg.String(branch),
+	}
+	downloadFileRetries, errRetries := strconv.Atoi(util.GetEnv(REPOW_GITLAB_DOWNLOAD_RETRIES, util.GetEnv(GITLAB_DOWNLOAD_RETRIES, strconv.Itoa(GITLAB_DOWNLOAD_RETRIES_DEFAULT))))
+	if errRetries != nil {
+		downloadFileRetries = GITLAB_DOWNLOAD_RETRIES_DEFAULT
+	}
+
+	var file *gg.File
+	var response *gg.Response
+	var err error
+
+	for attempts := 0; attempts < downloadFileRetries; attempts++ {
+		file, response, err = g.client.RepositoryFiles.GetFile(remotePath, model.RepoYamlFilename, gfo)
+		if err == nil {
+			break
+		}
+		// retry mostly because of unreliable gitlab api due to "net/http: TLS handshake timeout"
+		say.Error("Downloading file encountered error (retrying %d): %s", attempts+1, err)
+		time.Sleep(2 * time.Second)
+	}
+
+	if response != nil && response.StatusCode == 404 {
+		return nil, errors.New("repo.yaml does not exist")
+	}
+	if err != nil {
+		return nil, err
+	}
+	if response == nil {
+		return nil, errors.New("No gitlab server response")
+	}
+	if response.StatusCode != 200 {
+		say.Verbose("Unable to download repository manifest: %d", response.StatusCode)
+		return nil, errors.New(fmt.Sprintf("Gitlab API returned statuscode %d", response.StatusCode))
+	}
+
+	return file, err
+}
+
 func (g Gitlab) Apply(repo model.RepoMeta) error {
 	say.InfoLn("Apply %s", repo.Name)