fix #871

server/server.ts

@@ -885,6 +885,7 @@ async function getStats() {
   const vBrowserLaunches = await getRedisCountDay('vBrowserLaunches');
   const vBrowserFails = await getRedisCountDay('vBrowserFails');
   const vBrowserStagingFails = await getRedisCountDay('vBrowserStagingFails');
+  const vBrowserReimages = await getRedisCountDay('vBrowserReimage');
   const vBrowserStopTimeout = await getRedisCountDay(
     'vBrowserTerminateTimeout',
   );
@@ -1008,6 +1009,7 @@ async function getStats() {
     vBrowserLaunches,
     vBrowserFails,
     vBrowserStagingFails,
+    vBrowserReimages,
     vBrowserStopManual,
     vBrowserStopEmpty,
     vBrowserStopTimeout,

server/vm/base.ts

@@ -189,6 +189,7 @@ export abstract class VMManager {
       // Check if VM needs to be reimaged
       if (this.imageId !== vmImageId) {
         await this.reimageVM(vmid);
+        redisCount('vBrowserReimage');
         // Update the vmImageId
         await postgres.query(
           `UPDATE vbrowser SET image = $3 WHERE pool = $1 AND vmid = $2`,
@@ -472,9 +473,10 @@ export abstract class VMManager {
           redisCount('vBrowserStagingFails');
           await redis?.lpush('vBrowserStageFails', vmid);
           await redis?.ltrim('vBrowserStageFails', 0, 24);
+          // VM didn't come up. Try reimaging
+          await this.reimageVM(vmid);
+          redisCount('vBrowserReimage');
           await this.resetVM(vmid);
-          // TODO if we fail this too many times we should probably reimage it
-          // await this.terminateVMWrapper(vmid);
         }
         if (retryCount >= 180) {
           throw new Error('too many attempts on vm ' + vmid);

server/vm/docker.ts

@@ -77,6 +77,8 @@ export class Docker extends VMManager {
       `docker pull ${this.imageId}`,
     );
     console.log(stdout, stderr);
+    // The container is out of date. Delete it
+    this.terminateVMWrapper(id);
     return;
   };