Optimize GitHub API usage and fix pagination

- Added default 30-day window for fetching activities
- Skip issues and PRs without comments
- Added comments count to issue and PR interfaces
- Fixed pagination by properly handling Link header
- Added logging to help with debugging
- Added integration test that verifies real data

src/services/github.ts

@@ -21,6 +21,7 @@ interface GitHubIssue {
   number: number;
   html_url: string;
   comments_url: string;
+  comments: number;
   pull_request?: {
     url: string;
     html_url: string;
@@ -33,9 +34,16 @@ interface GitHubPR {
   number: number;
   html_url: string;
   comments_url: string;
+  comments: number;
 }
 
-async function fetchWithAuth<T>(url: string): Promise<T> {
+interface GitHubResponse<T> {
+  data: T;
+  hasNextPage: boolean;
+  nextUrl: string | null;
+}
+
+async function fetchWithAuth<T>(url: string): Promise<GitHubResponse<T>> {
   const response = await fetch(url, {
     headers: {
       'Authorization': `Bearer ${GITHUB_TOKEN}`,
@@ -47,7 +55,43 @@ async function fetchWithAuth<T>(url: string): Promise<T> {
     throw new Error(`GitHub API error: ${response.status.toString()} ${response.statusText}`);
   }
 
-  return response.json() as Promise<T>;
+  // Parse Link header for pagination
+  const linkHeader = response.headers.get('Link');
+  let hasNextPage = false;
+  let nextUrl: string | null = null;
+
+  if (linkHeader) {
+    const links = linkHeader.split(',');
+    for (const link of links) {
+      const [url, rel] = link.split(';');
+      if (rel.includes('rel="next"')) {
+        hasNextPage = true;
+        nextUrl = url.trim().slice(1, -1); // Remove < and >
+        break;
+      }
+    }
+  }
+
+  const data = await response.json() as T;
+  return { data, hasNextPage, nextUrl };
+}
+
+async function fetchAllPages<T>(url: string): Promise<T[]> {
+  const allItems: T[] = [];
+  let currentUrl = url;
+  let pageCount = 0;
+
+  while (currentUrl) {
+    pageCount++;
+    console.log(`Fetching page ${pageCount} from ${currentUrl}`);
+    const response = await fetchWithAuth<T[]>(currentUrl);
+    console.log(`Got ${response.data.length} items`);
+    allItems.push(...response.data);
+    currentUrl = response.nextUrl ?? '';
+  }
+
+  console.log(`Total items fetched: ${allItems.length}`);
+  return allItems;
 }
 
 function isBotComment(comment: GitHubComment): boolean {
@@ -108,7 +152,7 @@ function isPRModificationFailureComment(comment: GitHubComment): boolean {
 
 async function processIssueComments(issue: GitHubIssue): Promise<BotActivity[]> {
   const activities: BotActivity[] = [];
-  const comments = await fetchWithAuth<GitHubComment[]>(issue.comments_url);
+  const comments = await fetchAllPages<GitHubComment>(issue.comments_url);
 
   for (let i = 0; i < comments.length; i++) {
     const comment = comments[i];
@@ -137,7 +181,7 @@ async function processIssueComments(issue: GitHubIssue): Promise<BotActivity[]>
 
 async function processPRComments(pr: GitHubPR): Promise<BotActivity[]> {
   const activities: BotActivity[] = [];
-  const comments = await fetchWithAuth<GitHubComment[]>(pr.comments_url);
+  const comments = await fetchAllPages<GitHubComment>(pr.comments_url);
 
   for (let i = 0; i < comments.length; i++) {
     const comment = comments[i];
@@ -179,22 +223,29 @@ export async function fetchBotActivities(since?: string): Promise<BotActivity[]>
 
     if (since !== undefined && since !== '') {
       params.append('since', since);
+    } else {
+      // Default to last 30 days if no since parameter
+      const thirtyDaysAgo = new Date();
+      thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
+      params.append('since', thirtyDaysAgo.toISOString());
     }
 
     // Fetch issues
-    const issues = await fetchWithAuth<GitHubIssue[]>(`${baseUrl}/issues?${params.toString()}`);
+    const issues = await fetchAllPages<GitHubIssue>(`${baseUrl}/issues?${params.toString()}`);
     for (const issue of issues) {
-      if (issue.pull_request === undefined) { // Skip PRs from issues endpoint
+      if (issue.pull_request === undefined && issue.comments > 0) { // Skip PRs and issues without comments
         const issueActivities = await processIssueComments(issue);
         activities.push(...issueActivities);
       }
     }
 
     // Fetch PRs
-    const prs = await fetchWithAuth<GitHubPR[]>(`${baseUrl}/pulls?${params.toString()}`);
+    const prs = await fetchAllPages<GitHubPR>(`${baseUrl}/pulls?${params.toString()}`);
     for (const pr of prs) {
-      const prActivities = await processPRComments(pr);
-      activities.push(...prActivities);
+      if (pr.comments > 0) { // Skip PRs without comments
+        const prActivities = await processPRComments(pr);
+        activities.push(...prActivities);
+      }
     }
 
     // Sort by timestamp in descending order