fix(pagination): loop to get all releases and assets (#22)

Update to 0.1.1 for patch fix.

Previously did not consider pagination at all, so a max of 30 can only
be returned.

Fix assets fetching bug where previously it keeps refreshing rather than
calling "if cached get from cached state instead".

Add last-updated to greatly help preventing the FS API from happy
triggering GitHub HTTP queries.

ghafs.go

@@ -93,7 +93,7 @@ func (t tagDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 }
 
 func (t tagDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
-	assets, err := t.assets.refresh()
+	assets, err := t.assets.get()
 
 	if err != nil {
 		return nil, err

main.go

@@ -27,7 +27,7 @@ func (args) Description() string {
 }
 
 func (args) Version() string {
-	return "ghafs 0.1.0"
+	return "ghafs 0.1.1"
 }
 
 func main() {

releaseAssets.go

@@ -3,10 +3,19 @@ package main
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/google/go-github/v28/github"
 )
 
+// PageLimit GitHub only allows up to 100 per page
+// https://developer.github.com/v3/#pagination
+const PageLimit = 100
+
+// LastUpdatedThreshold sets the period threshold to allow the next update to
+// the items in seconds
+const LastUpdatedThreshold = 30 * time.Second
+
 // GhContext contains the necessary inputs to invoke the Gitub library
 type GhContext struct {
 	ctx    context.Context
@@ -27,6 +36,7 @@ type ReleasesWrap struct {
 	ghc *GhContext
 
 	tagReleases map[string]*Release
+	lastUpdated time.Time
 	m           sync.Mutex
 }
 
@@ -44,8 +54,9 @@ type AssetsWrap struct {
 	ghc     *GhContext
 	release *github.RepositoryRelease
 
-	assets []*github.ReleaseAsset
-	m      sync.Mutex
+	assets      []*github.ReleaseAsset
+	lastUpdated time.Time
+	m           sync.Mutex
 }
 
 func makeGhContext(ctx context.Context, client *github.Client, owner string, repo string) *GhContext {
@@ -60,6 +71,7 @@ func makeReleasesWrap(ghc *GhContext) *ReleasesWrap {
 	w := &ReleasesWrap{}
 	w.ghc = ghc
 	w.tagReleases = make(map[string]*Release)
+	w.lastUpdated = time.Time{} // Zero-ed time
 	return w
 }
 
@@ -72,32 +84,64 @@ func makeAssetsWrap(ghc *GhContext, release *github.RepositoryRelease) *AssetsWr
 	w.ghc = ghc
 	w.release = release
 	w.assets = []*github.ReleaseAsset{}
+	w.lastUpdated = time.Time{} // Zero-ed time
 	return w
 }
 
-// refreshImpl to be only internally only, to be used by mutex wrapping methods
-func (w ReleasesWrap) refreshImpl() (map[string]*Release, error) {
-	releases, _, err := w.ghc.client.Repositories.ListReleases(
-		w.ghc.ctx, w.ghc.owner, w.ghc.repo, nil)
+func loopListReleases(w *ReleasesWrap) ([]*github.RepositoryRelease, error) {
+	var releases []*github.RepositoryRelease
+
+	// Page offset starts from 1
+	for i := 1; ; i++ {
+		partialReleases, rsp, err := w.ghc.client.Repositories.ListReleases(
+			w.ghc.ctx,
+			w.ghc.owner,
+			w.ghc.repo,
+			&github.ListOptions{Page: i, PerPage: PageLimit})
+
+		if err != nil {
+			return nil, err
+		}
+
+		releases = append(releases, partialReleases...)
 
-	if err != nil {
-		return nil, err
+		if i >= rsp.LastPage {
+			break
+		}
 	}
 
-	for _, release := range releases {
-		w.tagReleases[release.GetTagName()] = makeRelease(w.ghc, release)
+	return releases, nil
+}
+
+// refreshImpl to be only internally only, to be used by mutex wrapping methods
+func (w *ReleasesWrap) refreshImpl() (map[string]*Release, error) {
+	timeNow := time.Now()
+
+	// Adhere to the update threshold
+	if w.lastUpdated.Add(LastUpdatedThreshold).Before(timeNow) {
+		releases, err := loopListReleases(w)
+
+		if err != nil {
+			return nil, err
+		}
+
+		for _, release := range releases {
+			w.tagReleases[release.GetTagName()] = makeRelease(w.ghc, release)
+		}
+
+		w.lastUpdated = timeNow
 	}
 
 	return w.tagReleases, nil
 }
 
-func (w ReleasesWrap) refresh() (map[string]*Release, error) {
+func (w *ReleasesWrap) refresh() (map[string]*Release, error) {
 	w.m.Lock()
 	defer w.m.Unlock()
 	return w.refreshImpl()
 }
 
-func (w ReleasesWrap) get() (map[string]*Release, error) {
+func (w *ReleasesWrap) get() (map[string]*Release, error) {
 	w.m.Lock()
 	defer w.m.Unlock()
 
@@ -108,30 +152,58 @@ func (w ReleasesWrap) get() (map[string]*Release, error) {
 	return w.tagReleases, nil
 }
 
+func loopListReleaseAssets(w *AssetsWrap) ([]*github.ReleaseAsset, error) {
+	var assets []*github.ReleaseAsset
+
+	// Page offset starts from 1
+	for i := 1; ; i++ {
+		partialAssets, rsp, err := w.ghc.client.Repositories.ListReleaseAssets(
+			w.ghc.ctx,
+			w.ghc.owner,
+			w.ghc.repo,
+			w.release.GetID(),
+			&github.ListOptions{Page: i, PerPage: PageLimit})
+
+		if err != nil {
+			return nil, err
+		}
+
+		assets = append(assets, partialAssets...)
+
+		if i >= rsp.LastPage {
+			break
+		}
+	}
+
+	return assets, nil
+}
+
 // refreshImpl to be only internally only, to be used by mutex wrapping methods
-func (w AssetsWrap) refreshImpl() ([]*github.ReleaseAsset, error) {
-	assets, _, err := w.ghc.client.Repositories.ListReleaseAssets(
-		w.ghc.ctx,
-		w.ghc.owner,
-		w.ghc.repo,
-		w.release.GetID(),
-		nil)
-
-	if err != nil {
-		return nil, err
+func (w *AssetsWrap) refreshImpl() ([]*github.ReleaseAsset, error) {
+	timeNow := time.Now()
+
+	// Adhere to the update threshold
+	if w.lastUpdated.Add(LastUpdatedThreshold).Before(timeNow) {
+		assets, err := loopListReleaseAssets(w)
+
+		if err != nil {
+			return nil, err
+		}
+
+		w.assets = assets
+		w.lastUpdated = timeNow
 	}
 
-	w.assets = assets
 	return w.assets, nil
 }
 
-func (w AssetsWrap) refresh() ([]*github.ReleaseAsset, error) {
+func (w *AssetsWrap) refresh() ([]*github.ReleaseAsset, error) {
 	w.m.Lock()
 	defer w.m.Unlock()
 	return w.refreshImpl()
 }
 
-func (w AssetsWrap) get() ([]*github.ReleaseAsset, error) {
+func (w *AssetsWrap) get() ([]*github.ReleaseAsset, error) {
 	w.m.Lock()
 	defer w.m.Unlock()