Merge pull request #11 from All-Hands-AI/fix-message-detection

Fix message detection to match actual OpenHands messages

.env

@@ -1 +1 @@
-GITHUB_TOKEN=
+VITE_GITHUB_TOKEN=$GITHUB_TOKEN

public/cache/bot-activities.json

@@ -329,5 +329,5 @@
       "description": "An attempt was made to automatically fix this issue, but it was unsuccessful. A branch named 'openhands-fix-issue-3913' has been created with the attempted changes. You can view the branch [here](https://github.com/All-Hands-AI/OpenHands/tree/openhands-fix-issue-3913). Manual intervention may be required."
     }
   ],
-  "lastUpdated": "2024-12-02T18:22:58.788Z"
+  "lastUpdated": "2024-12-02T20:47:08.922Z"
 }

scripts/github-api.ts

@@ -102,58 +102,46 @@ function isBotComment(comment: GitHubComment): boolean {
 function isStartWorkComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return lowerBody.includes('started fixing the issue') || 
-         lowerBody.includes('i will help you') || 
-         lowerBody.includes('i\'ll help you') ||
-         lowerBody.includes('i can help you');
+  return lowerBody.includes('started fixing the') || 
+         lowerBody.includes('openhands started fixing');
 }
 
 function isSuccessComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return lowerBody.includes('created a pull request') ||
-         lowerBody.includes('opened a pull request') ||
-         lowerBody.includes('submitted a pull request') ||
-         lowerBody.includes('successfully fixed') ||
-         lowerBody.includes('completed the changes') ||
-         lowerBody.includes('implemented the changes');
+  return lowerBody.includes('a potential fix has been generated and a draft pr') ||
+         lowerBody.includes('openhands made the following changes to resolve the issues') ||
+         lowerBody.includes('successfully fixed');
 }
 
 function isFailureComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return (lowerBody.includes('apologize') || lowerBody.includes('sorry')) &&
-         (lowerBody.includes('unable to') || lowerBody.includes('cannot') || lowerBody.includes('can\'t')) ||
-         lowerBody.includes('unsuccessful') ||
-         lowerBody.includes('manual intervention may be required');
+  return lowerBody.includes('the workflow to fix this issue encountered an error') ||
+         lowerBody.includes('openhands failed to create any code changes') ||
+         lowerBody.includes('an attempt was made to automatically fix this issue, but it was unsuccessful');
 }
 
 function isPRModificationComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return lowerBody.includes('help you modify') ||
-         lowerBody.includes('help you update') ||
-         lowerBody.includes('help you with the changes');
+  return lowerBody.includes('started fixing the') ||
+         lowerBody.includes('openhands started fixing');
 }
 
 function isPRModificationSuccessComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return lowerBody.includes('updated the pull request') ||
-         lowerBody.includes('made the requested changes') ||
-         lowerBody.includes('applied the changes') ||
-         lowerBody.includes('pushed the changes') ||
-         lowerBody.includes('committed the changes') ||
-         lowerBody.includes('implemented the requested changes');
+  return lowerBody.includes('openhands made the following changes to resolve the issues') ||
+         lowerBody.includes('updated pull request');
 }
 
 function isPRModificationFailureComment(comment: GitHubComment): boolean {
   if (!isBotComment(comment)) return false;
   const lowerBody = comment.body.toLowerCase();
-  return (lowerBody.includes('apologize') || lowerBody.includes('sorry')) &&
-         (lowerBody.includes('unable to modify') || lowerBody.includes('cannot modify') || lowerBody.includes('can\'t modify')) ||
-         lowerBody.includes('unsuccessful') ||
-         lowerBody.includes('manual intervention may be required');
+  return lowerBody.includes('the workflow to fix this issue encountered an error') ||
+         lowerBody.includes('openhands failed to create any code changes') ||
+         lowerBody.includes('an attempt was made to automatically fix this issue, but it was unsuccessful');
 }
 
 async function processIssueComments(issue: GitHubIssue): Promise<BotActivity[]> {

scripts/scripts/github-api.js

@@ -52,58 +52,46 @@ function isStartWorkComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return lowerBody.includes('started fixing the issue') ||
-        lowerBody.includes('i will help you') ||
-        lowerBody.includes('i\'ll help you') ||
-        lowerBody.includes('i can help you');
+    return lowerBody.includes('started fixing the') ||
+        lowerBody.includes('openhands started fixing');
 }
 function isSuccessComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return lowerBody.includes('created a pull request') ||
-        lowerBody.includes('opened a pull request') ||
-        lowerBody.includes('submitted a pull request') ||
-        lowerBody.includes('successfully fixed') ||
-        lowerBody.includes('completed the changes') ||
-        lowerBody.includes('implemented the changes');
+    return lowerBody.includes('a potential fix has been generated and a draft pr') ||
+        lowerBody.includes('openhands made the following changes to resolve the issues') ||
+        lowerBody.includes('successfully fixed');
 }
 function isFailureComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return (lowerBody.includes('apologize') || lowerBody.includes('sorry')) &&
-        (lowerBody.includes('unable to') || lowerBody.includes('cannot') || lowerBody.includes('can\'t')) ||
-        lowerBody.includes('unsuccessful') ||
-        lowerBody.includes('manual intervention may be required');
+    return lowerBody.includes('the workflow to fix this issue encountered an error') ||
+        lowerBody.includes('openhands failed to create any code changes') ||
+        lowerBody.includes('an attempt was made to automatically fix this issue, but it was unsuccessful');
 }
 function isPRModificationComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return lowerBody.includes('help you modify') ||
-        lowerBody.includes('help you update') ||
-        lowerBody.includes('help you with the changes');
+    return lowerBody.includes('started fixing the') ||
+        lowerBody.includes('openhands started fixing');
 }
 function isPRModificationSuccessComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return lowerBody.includes('updated the pull request') ||
-        lowerBody.includes('made the requested changes') ||
-        lowerBody.includes('applied the changes') ||
-        lowerBody.includes('pushed the changes') ||
-        lowerBody.includes('committed the changes') ||
-        lowerBody.includes('implemented the requested changes');
+    return lowerBody.includes('openhands made the following changes to resolve the issues') ||
+        lowerBody.includes('updated pull request');
 }
 function isPRModificationFailureComment(comment) {
     if (!isBotComment(comment))
         return false;
     const lowerBody = comment.body.toLowerCase();
-    return (lowerBody.includes('apologize') || lowerBody.includes('sorry')) &&
-        (lowerBody.includes('unable to modify') || lowerBody.includes('cannot modify') || lowerBody.includes('can\'t modify')) ||
-        lowerBody.includes('unsuccessful') ||
-        lowerBody.includes('manual intervention may be required');
+    return lowerBody.includes('the workflow to fix this issue encountered an error') ||
+        lowerBody.includes('openhands failed to create any code changes') ||
+        lowerBody.includes('an attempt was made to automatically fix this issue, but it was unsuccessful');
 }
 async function processIssueComments(issue) {
     const activities = [];

src/__integration_tests__/github.test.ts

@@ -18,15 +18,32 @@
  * They also depend on the actual state of the repository, so results may vary.
  */
 
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi } from 'vitest';
 import { fetchBotActivities } from '../services/github';
 import type { BotActivity } from '../types';
+import * as fs from 'fs';
 
 describe('GitHub Service Integration Tests', () => {
   // Skip these tests if VITE_GITHUB_TOKEN is not set
   const runTest = import.meta.env['VITE_GITHUB_TOKEN'] !== undefined && import.meta.env['VITE_GITHUB_TOKEN'] !== '' ? it : it.skip;
 
   runTest('should fetch real bot activities from OpenHands repository', async () => {
+    // Mock the fetch function to return the actual cached data
+    const cachedData = JSON.parse(fs.readFileSync('/workspace/openhands-agent-monitor/public/cache/bot-activities.json', 'utf8')) as BotActivity[];
+    const mockFetch = vi.fn().mockImplementation((url: string) => {
+      if (url === '/cache/bot-activities.json') {
+        return Promise.resolve({
+          ok: true,
+          status: 200,
+          statusText: 'OK',
+          json: () => Promise.resolve(cachedData),
+          headers: new Headers()
+        } as Response);
+      }
+      throw new Error(`Unexpected URL: ${url}`);
+    });
+    vi.stubGlobal('fetch', mockFetch as unknown as typeof fetch);
+
     const activities = await fetchBotActivities();
 
     // Verify we got some activities

src/services/github.test.ts

@@ -32,9 +32,7 @@ describe('GitHub Service', () => {
   }
 
   it('should detect openhands-agent comments in issues', async () => {
-
-
-    // Mock cache response
+    // Mock cache response for issue success
     const mockFetch = vi.fn().mockImplementation((url: string) => {
       if (url === '/cache/bot-activities.json') {
         return createMockResponse({
@@ -44,7 +42,7 @@ describe('GitHub Service', () => {
             status: 'success',
             timestamp: '2023-11-28T00:01:00Z',
             url: 'https://github.com/All-Hands-AI/OpenHands/issues/1#comment-2',
-            description: 'I have created a pull request at https://github.com/All-Hands-AI/OpenHands/pull/2'
+            description: 'A potential fix has been generated and a draft PR #2 has been created. Please review the changes.'
           }],
           lastUpdated: '2023-11-28T00:01:00Z'
         });
@@ -66,10 +64,74 @@ describe('GitHub Service', () => {
     vi.unstubAllGlobals();
   });
 
-  it('should detect openhands-agent comments in PRs', async () => {
+  it('should detect openhands-agent failure comments in issues', async () => {
+    // Mock cache response for issue failure
+    const mockFetch = vi.fn().mockImplementation((url: string) => {
+      if (url === '/cache/bot-activities.json') {
+        return createMockResponse({
+          activities: [{
+            id: 'issue-1-2',
+            type: 'issue',
+            status: 'failure',
+            timestamp: '2023-11-28T00:01:00Z',
+            url: 'https://github.com/All-Hands-AI/OpenHands/issues/1#comment-2',
+            description: 'The workflow to fix this issue encountered an error. Openhands failed to create any code changes.'
+          }],
+          lastUpdated: '2023-11-28T00:01:00Z'
+        });
+      }
+      throw new Error(`Unexpected URL: ${url}`);
+    });
+    vi.stubGlobal('fetch', mockFetch as unknown as typeof fetch);
+
+    const activities = await fetchBotActivities();
+
+    expect(activities).toHaveLength(1);
+    expect(activities[0]).toMatchObject<Partial<BotActivity>>({
+      type: 'issue',
+      status: 'failure',
+      id: expect.stringContaining('issue-1') as string,
+    });
+
+    // Restore the original fetch
+    vi.unstubAllGlobals();
+  });
 
+  it('should detect openhands-agent failure comments in PRs', async () => {
+    // Mock cache response for PR failure
+    const mockFetch = vi.fn().mockImplementation((url: string) => {
+      if (url === '/cache/bot-activities.json') {
+        return createMockResponse({
+          activities: [{
+            id: 'pr-1-2',
+            type: 'pr',
+            status: 'failure',
+            timestamp: '2023-11-28T00:01:00Z',
+            url: 'https://github.com/All-Hands-AI/OpenHands/pull/1#comment-2',
+            description: 'The workflow to fix this issue encountered an error. Openhands failed to create any code changes.'
+          }],
+          lastUpdated: '2023-11-28T00:01:00Z'
+        });
+      }
+      throw new Error(`Unexpected URL: ${url}`);
+    });
+    vi.stubGlobal('fetch', mockFetch as unknown as typeof fetch);
+
+    const activities = await fetchBotActivities();
+
+    expect(activities).toHaveLength(1);
+    expect(activities[0]).toMatchObject<Partial<BotActivity>>({
+      type: 'pr',
+      status: 'failure',
+      id: expect.stringContaining('pr-1') as string,
+    });
 
-    // Mock cache response
+    // Restore the original fetch
+    vi.unstubAllGlobals();
+  });
+
+  it('should detect openhands-agent comments in PRs', async () => {
+    // Mock cache response for PR success
     const mockFetch = vi.fn().mockImplementation((url: string) => {
       if (url === '/cache/bot-activities.json') {
         return createMockResponse({
@@ -79,7 +141,7 @@ describe('GitHub Service', () => {
             status: 'success',
             timestamp: '2023-11-28T00:01:00Z',
             url: 'https://github.com/All-Hands-AI/OpenHands/pull/1#comment-2',
-            description: 'I have updated the pull request with the requested changes.'
+            description: 'OpenHands made the following changes to resolve the issues:\n\n- Fixed the bug in the code\n\nUpdated pull request https://github.com/All-Hands-AI/OpenHands/pull/1 with new patches.'
           }],
           lastUpdated: '2023-11-28T00:01:00Z'
         });
@@ -100,4 +162,4 @@ describe('GitHub Service', () => {
     // Restore the original fetch
     vi.unstubAllGlobals();
   });
-});
+});