Utilise Strava's Best Effort Estimates (#4)

Strava calculates athlete's best 400m, 1/2 mile, 1K, ... stats from running activities. It only gives limited how those best efforts rank between different activities on the website, but you can fetch all that data in small chunks via Strava API and do the processing on your own.

While we can fetch summary of athlete's activities with decent number of API calls, best effort estimates have to be requested separately for each activity and this quickly becomes problem with Strava's rate limits ( https://developers.strava.com/docs/rate-limits/ ). For this reason, we once again store data as JSON in local files before we start making any sqlite database entries out of it.

On database end, this forced us to create second table into database and so I ended up renaming former `mystats` table into `summary` and introduce new Best Effort table.

On CLI, the best efforts information is available via `mystats best` command. Web UI got new 'Best Running Efforts' tab for web presentation of data.

api/activities.go

@@ -1,8 +1,11 @@
 package api
 
 // Copied from https://github.com/strava/go.strava/blob/99ebe972ba16ef3e1b1e5f62003dae3ac06f3adb/activities.go
-// so that we were able to add WorkoutType into ActivitySummary struct.
-// workout_type is documented attribute in Strava API v3, but for some reason it is missing from go.strava
+// so that we were able to add SportType and WorkoutType into ActivitySummary struct.
+// sport_type and workout_type are documented attributes in Strava API v3,
+// but for some reason it is missing from go.strava
+// sport_type is string value that can be same as type or something newer. Known exceptions
+// - sport_type: TrailRun has type: Run
 // workout_type is integer value that needs separate transformation into string.
 import (
 	"fmt"
@@ -78,3 +81,8 @@ func (as *ActivitySummary) WorkoutType() string {
 	}
 	return fmt.Sprintf("Unknown (%d)", as.WorkoutTypeId)
 }
+
+func NewActivitiesService(client *Client) *strava.ActivitiesService {
+	stravaClient := strava.NewClient(client.token, client.httpClient)
+	return strava.NewActivitiesService(stravaClient)
+}

api/api.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"path/filepath"
 	"time"
+
+	strava "github.com/strava/go.strava"
 )
 
 type Config struct {
@@ -83,7 +85,25 @@ func (cfg *Config) Refresh() (*Config, bool, error) {
 	return &tokens, true, nil
 }
 
-func ReadJSONs(fnames []string) ([]ActivitySummary, error) {
+func ReadBestEffortJSONs(fnames []string) ([]strava.BestEffort, error) {
+	efforts := []strava.BestEffort{}
+	for _, fname := range fnames {
+		body, err := os.ReadFile(filepath.Clean(fname))
+		if err != nil {
+			return efforts, err
+		}
+		activity := strava.ActivityDetailed{}
+		if err = json.Unmarshal(body, &activity); err != nil {
+			return efforts, err
+		}
+		for _, be := range activity.BestEfforts {
+			efforts = append(efforts, *be)
+		}
+	}
+	return efforts, nil
+}
+
+func ReadSummaryJSONs(fnames []string) ([]ActivitySummary, error) {
 	ids := map[int64]string{}
 	activities := []ActivitySummary{}
 	for _, fname := range fnames {

cmd/best.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+
+	_ "github.com/mattn/go-sqlite3"
+	"github.com/spf13/cobra"
+
+	"github.com/jylitalo/mystats/pkg/stats"
+)
+
+// topCmd turns sqlite db into table or csv by week/month/...
+func bestCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "best",
+		Short: "Best Run Efforts based on Strava",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			flags := cmd.Flags()
+			format, _ := flags.GetString("format")
+			limit, _ := flags.GetInt("limit")
+			distance, _ := flags.GetString("distance")
+			update, _ := flags.GetBool("update")
+			formatFn := map[string]func(headers []string, results [][]string){
+				"csv":   printTopCSV,
+				"table": printTopTable,
+			}
+			if _, ok := formatFn[format]; !ok {
+				return fmt.Errorf("unknown format: %s", format)
+			}
+			db, err := makeDB(update)
+			if err != nil {
+				return err
+			}
+			defer db.Close()
+			headers, results, err := stats.Best(db, distance, limit)
+			if err != nil {
+				return err
+			}
+			formatFn[format](headers, results)
+			return nil
+		},
+	}
+	cmd.Flags().String("format", "csv", "output format (csv, table)")
+	cmd.Flags().String("distance", "Marathon", "Best Efforts distance")
+	cmd.Flags().Int("limit", 10, "number of entries")
+	cmd.Flags().Bool("update", true, "update database")
+	return cmd
+}

cmd/cmd.go

@@ -16,7 +16,7 @@ func Execute() error {
 	}
 	rootCmd.AddCommand(
 		configureCmd(), fetchCmd(), makeCmd(),
-		listCmd(types), plotCmd(types), statsCmd(types), topCmd(types),
+		bestCmd(), listCmd(types), plotCmd(types), statsCmd(types), topCmd(types),
 		serverCmd(types),
 	)
 	return rootCmd.Execute()

cmd/fetch.go

@@ -2,15 +2,21 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"log/slog"
 	"os"
 	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
 	"github.com/jylitalo/mystats/api"
 	"github.com/jylitalo/mystats/config"
+	"github.com/jylitalo/mystats/storage"
 )
 
 // fetchCmd fetches activity data from Strava
@@ -19,25 +25,117 @@ func fetchCmd() *cobra.Command {
 		Use:   "fetch",
 		Short: "Fetch activity data from Strava to JSON files",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return fetch()
+			flags := cmd.Flags()
+			be, _ := flags.GetBool("best_efforts")
+			return fetch(be)
 		},
 	}
+	cmd.Flags().Bool("best_efforts", false, "Fetch activities best efforts")
 	return cmd
 }
 
-func fetch() error {
+func fetch(best_efforts bool) error {
 	after, prior, err := getEpoch()
 	if err != nil {
 		return err
 	}
-	call, err := callListActivities(after)
+	client, err := getClient()
+	if err != nil {
+		return err
+	}
+	call, err := callListActivities(client, after)
 	if err != nil {
 		return err
 	}
 	_, err = saveActivities(call, prior)
+	if err == nil && best_efforts {
+		return fetchBestEfforts(client)
+	}
+	return err
+}
+
+func getClient() (*api.Client, error) {
+	cfg, err := config.Get(true)
+	if err != nil {
+		return nil, err
+	}
+	strava := cfg.Strava
+	api.ClientId = strava.ClientID
+	api.ClientSecret = strava.ClientSecret
+	client := api.NewClient(strava.AccessToken)
+	return client, err
+}
+
+func fetchBestEfforts(client *api.Client) error {
+	db := &storage.Sqlite3{}
+	if err := db.Open(); err != nil {
+		return err
+	}
+	rows, err := db.QuerySummary(
+		[]string{"StravaID"},
+		storage.SummaryConditions{Types: []string{"Run"}},
+		&storage.Order{OrderBy: []string{"StravaID desc"}},
+	)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	stravaIDs := []int64{}
+	for rows.Next() {
+		var stravaID int64
+		err = rows.Scan(&stravaID)
+		if err != nil {
+			return err
+		}
+		stravaIDs = append(stravaIDs, stravaID)
+	}
+	if len(stravaIDs) < 1 {
+		return errors.New("no stravaIDs found from database")
+	}
+	fetched := 0
+	_ = os.Mkdir("activities", 0750)
+	actFiles, err := activitiesFiles()
+	if err != nil {
+		return err
+	}
+	alreadyFetched := []int64{}
+	for _, actFile := range actFiles {
+		intStr := strings.Split(strings.Split(actFile, "_")[1], ".")[0]
+		i, _ := strconv.Atoi(intStr)
+		alreadyFetched = append(alreadyFetched, int64(i))
+	}
+	service := api.NewActivitiesService(client)
+	for idx, stravaID := range stravaIDs {
+		if slices.Contains[[]int64, int64](alreadyFetched, stravaID) {
+			continue
+		}
+		call := service.Get(stravaID)
+		activity, err := call.Do()
+		if err != nil {
+			return err
+		}
+		j, err := json.Marshal(activity)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("%d => activities/activity_%d.json ...\n", stravaID, stravaID)
+		if err = os.WriteFile(fmt.Sprintf("activities/activity_%d.json", stravaID), j, 0600); err != nil {
+			return err
+		}
+		fetched++
+		if fetched >= 100 {
+			slog.Info("Already fetched 100 activities", "left", len(stravaIDs)-idx)
+			return nil
+		}
+	}
+	slog.Info("Activity details fetched", "fetched", fetched)
 	return err
 }
 
+func activitiesFiles() ([]string, error) {
+	return filepath.Glob("activities/activity_*.json")
+}
+
 func pageFiles() ([]string, error) {
 	return filepath.Glob("pages/page*.json")
 }
@@ -56,7 +154,7 @@ func getEpoch() (time.Time, int, error) {
 		}
 	}
 	offset := len(fnames)
-	activities, err := api.ReadJSONs(fnames)
+	activities, err := api.ReadSummaryJSONs(fnames)
 	if err != nil {
 		return epoch, offset, err
 	}
@@ -68,15 +166,7 @@ func getEpoch() (time.Time, int, error) {
 	return epoch, offset, nil
 }
 
-func callListActivities(after time.Time) (*api.CurrentAthleteListActivitiesCall, error) {
-	cfg, err := config.Get(true)
-	if err != nil {
-		return nil, err
-	}
-	strava := cfg.Strava
-	api.ClientId = strava.ClientID
-	api.ClientSecret = strava.ClientSecret
-	client := api.NewClient(strava.AccessToken)
+func callListActivities(client *api.Client, after time.Time) (*api.CurrentAthleteListActivitiesCall, error) {
 	current := api.NewCurrentAthleteService(client)
 	call := current.ListActivities()
 	return call.After(int(after.Unix())), nil

cmd/list.go

@@ -17,6 +17,7 @@ func listCmd(types []string) *cobra.Command {
 		Short: "List races or long runs",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			flags := cmd.Flags()
+			limit, _ := flags.GetInt("limit")
 			types, _ := flags.GetStringSlice("type")
 			update, _ := flags.GetBool("update")
 			workouts, _ := flags.GetStringSlice("workout")
@@ -26,7 +27,7 @@ func listCmd(types []string) *cobra.Command {
 			}
 			defer db.Close()
 			table := tablewriter.NewWriter(os.Stdout)
-			headers, results, err := stats.List(db, types, workouts, nil)
+			headers, results, err := stats.List(db, types, workouts, nil, limit)
 			if err != nil {
 				return err
 			}
@@ -36,6 +37,7 @@ func listCmd(types []string) *cobra.Command {
 			return nil
 		},
 	}
+	cmd.Flags().Int("limit", 100, "number of activities")
 	cmd.Flags().StringSlice("type", types, "sport types (run, trail run, ...)")
 	cmd.Flags().Bool("update", true, "update database")
 	cmd.Flags().StringSlice("workout", []string{}, "workout type")

cmd/make.go

@@ -15,10 +15,12 @@ import (
 )
 
 type Storage interface {
-	Query(fields []string, cond storage.Conditions, order *storage.Order) (*sql.Rows, error)
-	QueryTypes(cond storage.Conditions) ([]string, error)
-	QueryWorkoutTypes(cond storage.Conditions) ([]string, error)
-	QueryYears(cond storage.Conditions) ([]int, error)
+	QueryBestEffort(fields []string, name string, order *storage.Order) (*sql.Rows, error)
+	QueryBestEffortDistances() ([]string, error)
+	QuerySummary(fields []string, cond storage.SummaryConditions, order *storage.Order) (*sql.Rows, error)
+	QueryTypes(cond storage.SummaryConditions) ([]string, error)
+	QueryWorkoutTypes(cond storage.SummaryConditions) ([]string, error)
+	QueryYears(cond storage.SummaryConditions) ([]int, error)
 	Close() error
 }
 
@@ -60,7 +62,7 @@ func skipDB(db *storage.Sqlite3, fnames []string) bool {
 func makeDB(update bool) (Storage, error) {
 	slog.Info("Fetch activities from Strava")
 	if update {
-		if err := fetch(); err != nil {
+		if err := fetch(false); err != nil {
 			return nil, err
 		}
 	}
@@ -75,15 +77,15 @@ func makeDB(update bool) (Storage, error) {
 		return db, nil
 	}
 	slog.Info("Making database")
-	activities, err := api.ReadJSONs(fnames)
+	activities, err := api.ReadSummaryJSONs(fnames)
 	if err != nil {
 		return nil, err
 	}
-	dbActivities := []storage.Record{}
+	dbActivities := []storage.SummaryRecord{}
 	for _, activity := range activities {
 		t := activity.StartDateLocal
 		year, week := t.ISOWeek()
-		dbActivities = append(dbActivities, storage.Record{
+		dbActivities = append(dbActivities, storage.SummaryRecord{
 			StravaID:    activity.Id,
 			Year:        year,
 			Month:       int(t.Month()),
@@ -98,9 +100,28 @@ func makeDB(update bool) (Storage, error) {
 			MovingTime:  activity.MovingTime,
 		})
 	}
+	fnames, err = activitiesFiles()
+	if err != nil {
+		return nil, err
+	}
+	detailed, err := api.ReadBestEffortJSONs(fnames)
+	if err != nil {
+		return nil, err
+	}
+	dbEfforts := []storage.BestEffortRecord{}
+	for _, activity := range detailed {
+		dbEfforts = append(dbEfforts, storage.BestEffortRecord{
+			StravaID:    activity.EffortSummary.Activity.Id,
+			Name:        activity.EffortSummary.Name,
+			MovingTime:  activity.EffortSummary.MovingTime,
+			ElapsedTime: activity.EffortSummary.ElapsedTime,
+			Distance:    int(activity.Distance),
+		})
+	}
 	errR := db.Remove()
 	errO := db.Open()
 	errC := db.Create()
-	errI := db.Insert(dbActivities)
-	return db, errors.Join(errR, errO, errC, errI)
+	errI := db.InsertSummary(dbActivities)
+	errBE := db.InsertBestEffort(dbEfforts)
+	return db, errors.Join(errR, errO, errC, errI, errBE)
 }