From 8fd8be4582d13124aac423f2cb157cbfdd9f38e8 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Mon, 30 Oct 2023 17:17:02 +0000 Subject: [PATCH 1/3] chore(http): send token using standard `Bearer` authorization scheme * the Bearer scheme also allows to use JSON web tokens (JWT) * ref: --- dl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dl.go b/dl.go index 0cbb100..b84939b 100644 --- a/dl.go +++ b/dl.go @@ -50,7 +50,7 @@ func SetAuthHeader(req *http.Request) *http.Request { fmt.Fprintln(os.Stderr, "error: cannot use GitHub token if SSL verification is disabled") os.Exit(1) } - req.Header.Set("Authorization", fmt.Sprintf("token %s", token)) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) } return req From 3734c1f08e9bae3228003aba1d8e033b384f37c9 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Mon, 30 Oct 2023 18:45:31 +0000 Subject: [PATCH 2/3] chore(http): allow to provide `Accept` header in `Get()` * all binary downloads should use the `application/octet-stream` media type * all GitHub API calls should use the `application/vnd.github+json` media type * also refactor `GetRateLimit()` to use `Get()` instead of own HTTP request * ref: --- dl.go | 25 ++++++++++--------------- find.go | 6 +++--- verify.go | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/dl.go b/dl.go index b84939b..52e4536 100644 --- a/dl.go +++ b/dl.go @@ -15,6 +15,11 @@ import ( "github.com/zyedidia/eget/home" ) +const ( + AcceptBinary = "application/octet-stream" + AcceptGitHubJSON = "application/vnd.github+json" +) + func tokenFrom(s string) (string, error) { if strings.HasPrefix(s, "@") { f, err := home.Expand(s[1:]) @@ -56,13 +61,14 @@ func SetAuthHeader(req *http.Request) *http.Request { return req } -func Get(url string) (*http.Response, error) { +func Get(url, accept string) (*http.Response, error) { req, err := http.NewRequest("GET", url, nil) - if err != nil { return nil, err } + req.Header.Set("Accept", accept) + req = SetAuthHeader(req) proxyClient := &http.Client{Transport: &http.Transport{ @@ -101,21 +107,10 @@ func (r RateLimit) String() string { } func GetRateLimit() (RateLimit, error) { - url := "https://api.github.com/rate_limit" - req, err := http.NewRequest("GET", url, nil) + resp, err := Get("https://api.github.com/rate_limit", AcceptGitHubJSON) if err != nil { return RateLimit{}, err } - - req = SetAuthHeader(req) - - req.Header.Set("Accept", "application/vnd.github.v3+json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return RateLimit{}, err - } - defer resp.Body.Close() b, err := io.ReadAll(resp.Body) @@ -144,7 +139,7 @@ func Download(url string, out io.Writer, getbar func(size int64) *pb.ProgressBar return err } - resp, err := Get(url) + resp, err := Get(url, AcceptBinary) if err != nil { return err } diff --git a/find.go b/find.go index cffb9e1..8dcc210 100644 --- a/find.go +++ b/find.go @@ -69,7 +69,7 @@ func (f *GithubAssetFinder) Find() ([]string, error) { // query github's API for this repo/tag pair. url := fmt.Sprintf("https://api.github.com/repos/%s/releases/%s", f.Repo, f.Tag) - resp, err := Get(url) + resp, err := Get(url, AcceptGitHubJSON) if err != nil { return nil, err } @@ -122,7 +122,7 @@ func (f *GithubAssetFinder) FindMatch() ([]string, error) { for page := 1; ; page++ { url := fmt.Sprintf("https://api.github.com/repos/%s/releases?page=%d", f.Repo, page) - resp, err := Get(url) + resp, err := Get(url, AcceptGitHubJSON) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (f *GithubAssetFinder) FindMatch() ([]string, error) { // finds the latest pre-release and returns the tag func (f *GithubAssetFinder) getLatestTag() (string, error) { url := fmt.Sprintf("https://api.github.com/repos/%s/releases", f.Repo) - resp, err := Get(url) + resp, err := Get(url, AcceptGitHubJSON) if err != nil { return "", fmt.Errorf("pre-release finder: %w", err) } diff --git a/verify.go b/verify.go index 923b03d..1aa3750 100644 --- a/verify.go +++ b/verify.go @@ -65,7 +65,7 @@ type Sha256AssetVerifier struct { } func (s256 *Sha256AssetVerifier) Verify(b []byte) error { - resp, err := Get(s256.AssetURL) + resp, err := Get(s256.AssetURL, AcceptBinary) if err != nil { return err } From 156c0343907def7138593e9ec5fa09f3f1c409e2 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Mon, 30 Oct 2023 19:03:33 +0000 Subject: [PATCH 3/3] feat: add support for assets in private repositories * download assets through the GitHub API instead of browser download links * allows to download from private repositories when combined with a GitHub token * added new Asset type with the Name (necessary for matching) and DownloadURL of an asset * ref: --- detect.go | 52 ++++++++++++++++++++++++++-------------------------- eget.go | 42 +++++++++++++++++++++++------------------- find.go | 42 +++++++++++++++++++++++++++++------------- 3 files changed, 78 insertions(+), 58 deletions(-) diff --git a/detect.go b/detect.go index e883bee..3846efe 100644 --- a/detect.go +++ b/detect.go @@ -12,7 +12,7 @@ type Detector interface { // Detect takes a list of possible assets and returns a direct match. If a // single direct match is not found, it returns a list of candidates and an // error explaining what happened. - Detect(assets []string) (string, []string, error) + Detect(assets []Asset) (Asset, []Asset, error) } type DetectorChain struct { @@ -20,11 +20,11 @@ type DetectorChain struct { system Detector } -func (dc *DetectorChain) Detect(assets []string) (string, []string, error) { +func (dc *DetectorChain) Detect(assets []Asset) (Asset, []Asset, error) { for _, d := range dc.detectors { choice, candidates, err := d.Detect(assets) if len(candidates) == 0 && err != nil { - return "", nil, err + return Asset{}, nil, err } else if len(candidates) == 0 { return choice, nil, nil } else { @@ -33,13 +33,13 @@ func (dc *DetectorChain) Detect(assets []string) (string, []string, error) { } choice, candidates, err := dc.system.Detect(assets) if len(candidates) == 0 && err != nil { - return "", nil, err + return Asset{}, nil, err } else if len(candidates) == 0 { return choice, nil, nil } else if len(candidates) >= 1 { assets = candidates } - return "", assets, fmt.Errorf("%d candidates found for asset chain", len(assets)) + return Asset{}, assets, fmt.Errorf("%d candidates found for asset chain", len(assets)) } // An OS represents a target operating system. @@ -170,11 +170,11 @@ var goarchmap = map[string]Arch{ // candidates. type AllDetector struct{} -func (a *AllDetector) Detect(assets []string) (string, []string, error) { +func (a *AllDetector) Detect(assets []Asset) (Asset, []Asset, error) { if len(assets) == 1 { return assets[0], nil, nil } - return "", assets, fmt.Errorf("%d matches found", len(assets)) + return Asset{}, assets, fmt.Errorf("%d matches found", len(assets)) } // SingleAssetDetector finds a single named asset. If Anti is true it finds all @@ -184,25 +184,25 @@ type SingleAssetDetector struct { Anti bool } -func (s *SingleAssetDetector) Detect(assets []string) (string, []string, error) { - var candidates []string +func (s *SingleAssetDetector) Detect(assets []Asset) (Asset, []Asset, error) { + var candidates []Asset for _, a := range assets { - if !s.Anti && path.Base(a) == s.Asset { + if !s.Anti && path.Base(a.Name) == s.Asset { return a, nil, nil } - if !s.Anti && strings.Contains(path.Base(a), s.Asset) { + if !s.Anti && strings.Contains(path.Base(a.Name), s.Asset) { candidates = append(candidates, a) } - if s.Anti && !strings.Contains(path.Base(a), s.Asset) { + if s.Anti && !strings.Contains(path.Base(a.Name), s.Asset) { candidates = append(candidates, a) } } if len(candidates) == 1 { return candidates[0], nil, nil } else if len(candidates) > 1 { - return "", candidates, fmt.Errorf("%d candidates found for asset `%s`", len(candidates), s.Asset) + return Asset{}, candidates, fmt.Errorf("%d candidates found for asset `%s`", len(candidates), s.Asset) } - return "", nil, fmt.Errorf("asset `%s` not found", s.Asset) + return Asset{}, nil, fmt.Errorf("asset `%s` not found", s.Asset) } // A SystemDetector matches a particular OS/Arch system pair. @@ -234,22 +234,22 @@ func NewSystemDetector(sos, sarch string) (*SystemDetector, error) { // match the OS are found, and no full OS/Arch matches are found, the OS // matches are returned as candidates. Otherwise all assets are returned as // candidates. -func (d *SystemDetector) Detect(assets []string) (string, []string, error) { - var priority []string - var matches []string - var candidates []string - all := make([]string, 0, len(assets)) +func (d *SystemDetector) Detect(assets []Asset) (Asset, []Asset, error) { + var priority []Asset + var matches []Asset + var candidates []Asset + all := make([]Asset, 0, len(assets)) for _, a := range assets { - if strings.HasSuffix(a, ".sha256") || strings.HasSuffix(a, ".sha256sum") { + if strings.HasSuffix(a.Name, ".sha256") || strings.HasSuffix(a.Name, ".sha256sum") { // skip checksums (they will be checked later by the verifier) continue } - os, extra := d.Os.Match(a) + os, extra := d.Os.Match(a.Name) if extra { priority = append(priority, a) } - arch := d.Arch.Match(a) + arch := d.Arch.Match(a.Name) if os && arch { matches = append(matches, a) } @@ -261,17 +261,17 @@ func (d *SystemDetector) Detect(assets []string) (string, []string, error) { if len(priority) == 1 { return priority[0], nil, nil } else if len(priority) > 1 { - return "", priority, fmt.Errorf("%d priority matches found", len(matches)) + return Asset{}, priority, fmt.Errorf("%d priority matches found", len(matches)) } else if len(matches) == 1 { return matches[0], nil, nil } else if len(matches) > 1 { - return "", matches, fmt.Errorf("%d matches found", len(matches)) + return Asset{}, matches, fmt.Errorf("%d matches found", len(matches)) } else if len(candidates) == 1 { return candidates[0], nil, nil } else if len(candidates) > 1 { - return "", candidates, fmt.Errorf("%d candidates found (unsure architecture)", len(candidates)) + return Asset{}, candidates, fmt.Errorf("%d candidates found (unsure architecture)", len(candidates)) } else if len(all) == 1 { return all[0], nil, nil } - return "", all, fmt.Errorf("no candidates found") + return Asset{}, all, fmt.Errorf("no candidates found") } diff --git a/eget.go b/eget.go index bd46219..cbee44b 100644 --- a/eget.go +++ b/eget.go @@ -60,15 +60,15 @@ func IsDirectory(path string) bool { return fileInfo.IsDir() } -// searches for an asset thaat has the same name as the requested one but +// searches for an asset that has the same name as the requested one but // ending with .sha256 or .sha256sum -func checksumAsset(asset string, assets []string) string { +func checksumAsset(asset Asset, assets []Asset) Asset { for _, a := range assets { - if a == asset+".sha256sum" || a == asset+".sha256" { + if a.Name == asset.Name+".sha256sum" || a.Name == asset.Name+".sha256" { return a } } - return "" + return Asset{} } // Determine the appropriate Finder to use. If opts.URL is provided, we use @@ -135,12 +135,12 @@ func getFinder(project string, opts *Flags) (finder Finder, tool string) { return finder, tool } -func getVerifier(sumAsset string, opts *Flags) (verifier Verifier, err error) { +func getVerifier(sumAsset Asset, opts *Flags) (verifier Verifier, err error) { if opts.Verify != "" { verifier, err = NewSha256Verifier(opts.Verify) - } else if sumAsset != "" { + } else if sumAsset != (Asset{}) { verifier = &Sha256AssetVerifier{ - AssetURL: sumAsset, + AssetURL: sumAsset.DownloadURL, } } else if opts.Hash { verifier = &Sha256Printer{} @@ -428,27 +428,31 @@ func main() { fatal(err) } - // get the url and candidates from the detector - url, candidates, err := detector.Detect(assets) + // get the asset and candidates from the detector + asset, candidates, err := detector.Detect(assets) if len(candidates) != 0 && err != nil { // if multiple candidates are returned, the user must select manually which one to download fmt.Fprintf(os.Stderr, "%v: please select manually\n", err) choices := make([]interface{}, len(candidates)) for i := range candidates { - choices[i] = path.Base(candidates[i]) + choices[i] = path.Base(candidates[i].Name) } choice := userSelect(choices) - url = candidates[choice-1] + asset = candidates[choice-1] } else if err != nil { fatal(err) } - // print the URL - fmt.Fprintf(output, "%s\n", url) + // print the download URL of the asset + if asset.Name != asset.DownloadURL { + fmt.Fprintf(output, "%s (%s)\n", asset.DownloadURL, asset.Name) + } else { + fmt.Fprintf(output, "%s\n", asset.DownloadURL) + } // download with progress bar buf := &bytes.Buffer{} - err = Download(url, buf, func(size int64) *pb.ProgressBar { + err = Download(asset.DownloadURL, buf, func(size int64) *pb.ProgressBar { var pbout io.Writer = os.Stderr if opts.Quiet { pbout = io.Discard @@ -474,12 +478,12 @@ func main() { })) }) if err != nil { - fatal(fmt.Sprintf("%s (URL: %s)", err, url)) + fatal(fmt.Sprintf("%s (URL: %s)", err, asset.DownloadURL)) } body := buf.Bytes() - sumAsset := checksumAsset(url, assets) + sumAsset := checksumAsset(asset, assets) verifier, err := getVerifier(sumAsset, &opts) if err != nil { fatal(err) @@ -487,13 +491,13 @@ func main() { err = verifier.Verify(body) if err != nil { fatal(err) - } else if opts.Verify == "" && sumAsset != "" { - fmt.Fprintf(output, "Checksum verified with %s\n", path.Base(sumAsset)) + } else if opts.Verify == "" && sumAsset != (Asset{}) { + fmt.Fprintf(output, "Checksum verified with %s\n", path.Base(sumAsset.Name)) } else if opts.Verify != "" { fmt.Fprintf(output, "Checksum verified\n") } - extractor, err := getExtractor(url, tool, &opts) + extractor, err := getExtractor(asset.Name, tool, &opts) if err != nil { fatal(err) } diff --git a/find.go b/find.go index 8dcc210..997ebbc 100644 --- a/find.go +++ b/find.go @@ -10,15 +10,22 @@ import ( "time" ) -// A Finder returns a list of URLs making up a project's assets. +// A Finder returns a list of assets for a project. type Finder interface { - Find() ([]string, error) + Find() ([]Asset, error) +} + +// An Asset is the name (if any) and download URL for an asset of a project. +type Asset struct { + Name string + DownloadURL string } // A GithubRelease matches the Assets portion of Github's release API json. type GithubRelease struct { Assets []struct { - DownloadURL string `json:"browser_download_url"` + Name string `json:"name"` + URL string `json:"url"` } `json:"assets"` Prerelease bool `json:"prerelease"` @@ -58,7 +65,7 @@ type GithubAssetFinder struct { var ErrNoUpgrade = errors.New("requested release is not more recent than current version") -func (f *GithubAssetFinder) Find() ([]string, error) { +func (f *GithubAssetFinder) Find() ([]Asset, error) { if f.Prerelease && f.Tag == "latest" { tag, err := f.getLatestTag() if err != nil { @@ -109,15 +116,15 @@ func (f *GithubAssetFinder) Find() ([]string, error) { } // accumulate all assets from the json into a slice - assets := make([]string, 0, len(release.Assets)) + assets := make([]Asset, 0, len(release.Assets)) for _, a := range release.Assets { - assets = append(assets, a.DownloadURL) + assets = append(assets, Asset{Name: a.Name, DownloadURL: a.URL}) } return assets, nil } -func (f *GithubAssetFinder) FindMatch() ([]string, error) { +func (f *GithubAssetFinder) FindMatch() ([]Asset, error) { tag := f.Tag[len("tags/"):] for page := 1; ; page++ { @@ -160,9 +167,9 @@ func (f *GithubAssetFinder) FindMatch() ([]string, error) { } if strings.Contains(r.Tag, tag) && !r.CreatedAt.Before(f.MinTime) { // we have a winner - assets := make([]string, 0, len(r.Assets)) + assets := make([]Asset, 0, len(r.Assets)) for _, a := range r.Assets { - assets = append(assets, a.DownloadURL) + assets = append(assets, Asset{Name: a.Name, DownloadURL: a.URL}) } return assets, nil } @@ -207,8 +214,12 @@ type DirectAssetFinder struct { URL string } -func (f *DirectAssetFinder) Find() ([]string, error) { - return []string{f.URL}, nil +func (f *DirectAssetFinder) Find() ([]Asset, error) { + asset := Asset{ + Name: f.URL, + DownloadURL: f.URL, + } + return []Asset{asset}, nil } type GithubSourceFinder struct { @@ -217,6 +228,11 @@ type GithubSourceFinder struct { Tag string } -func (f *GithubSourceFinder) Find() ([]string, error) { - return []string{fmt.Sprintf("https://github.com/%s/tarball/%s/%s.tar.gz", f.Repo, f.Tag, f.Tool)}, nil +func (f *GithubSourceFinder) Find() ([]Asset, error) { + name := fmt.Sprintf("%s.tar.gz", f.Tool) + asset := Asset{ + Name: name, + DownloadURL: fmt.Sprintf("https://github.com/%s/tarball/%s/%s", f.Repo, f.Tag, name), + } + return []Asset{asset}, nil }