```New server environment variables / .env settings are needed:

PACKRAT_ITOPS_EMAIL=comma-delimited list of email addresses to which Cook connection failure notifications are sent.  The first is also the "FROM" address for all notifications
PACKRAT_SMTP_HOST=SMTP server for output mail
PACKRAT_SMTP_PORT=SMTP server port for output mail
PACKRAT_SMTP_SECURE=SMTP server flag for secure connection for output mail
PACKRAT_SMTP_AUTHUSER=SMTP server username for authentication for output mail
PACKRAT_SMTP_AUTHPASSWORD=SMTP server password for authentication for output mail
```

Build:
* Added nodemailer to server

Migration:
* De-link migration from HTTP request, so that it continues to run even if the launching request is interrupted

System:
* Added Email implementation, used to send IT/OPS messages (such as Cook is down)
* Updated Config to have cookServerUrls[], allowing for rollover to other Cook instances if configured
* Added Config.it, with IT/OPS email addresses (for notifications), as well as SMTP host settings

Job:
* Load array of Cook Server URLs
* Extracted computation of Cook Server URL, based on the "current" server being used
* Detect connection failures; upon every so many, "roll" to the next Cook server, and send an email notification
* Exit the job polling loop if we have too many connection failures

.env.template

@@ -13,6 +13,7 @@ REACT_APP_PACKRAT_SERVER_ENDPOINT=
 # SERVER
 PACKRAT_SERVER_PORT=
 PACKRAT_DATABASE_URL=
+PACKRAT_ITOPS_EMAIL=
 PACKRAT_SESSION_SECRET=
 PACKRAT_EDAN_AUTH_KEY=
 PACKRAT_EDAN_SERVER=
@@ -30,6 +31,11 @@ PACKRAT_LDAP_PASSWORD=
 PACKRAT_LDAP_CN=
 PACKRAT_LDAP_OU=
 PACKRAT_LDAP_DC=
+PACKRAT_SMTP_HOST=
+PACKRAT_SMTP_PORT=
+PACKRAT_SMTP_SECURE=
+PACKRAT_SMTP_AUTHUSER=
+PACKRAT_SMTP_AUTHPASSWORD=
 PACKRAT_LOG_ROOT=
 
 # SOLR

server/config/index.ts

@@ -80,9 +80,17 @@ export type ConfigType = {
     },
     job: {
         type: JOB_TYPE;
-        cookServerUrl: string;
+        cookServerUrls: string[];
         cookClientId: string;
     },
+    it: {
+        itopsEmail: string[];
+        smtpHost?: string | undefined;
+        smtpPort?: number | undefined;
+        smtpSecure?: boolean | undefined;
+        smtpAuthUser?: string | undefined;
+        smtpAuthPassword?: string | undefined;
+    },
     log: {
         root: string;
     },
@@ -143,9 +151,17 @@ export const Config: ConfigType = {
     },
     job: {
         type: JOB_TYPE.NODE_SCHEDULE,
-        cookServerUrl: process.env.PACKRAT_COOK_SERVER_URL ? process.env.PACKRAT_COOK_SERVER_URL : /* istanbul ignore next */ 'http://si-3ddigip01.si.edu:8000/',
+        cookServerUrls: process.env.PACKRAT_COOK_SERVER_URL ? process.env.PACKRAT_COOK_SERVER_URL.split(',') : /* istanbul ignore next */ ['http://si-3ddigip01.si.edu:8000/'],
         cookClientId: '5b258c8e-108c-4990-a088-17ffd6e22852', // Concierge's client ID; taken from C:\Tools\CookDev\server\clients.json on Cook server
     },
+    it: {
+        itopsEmail: process.env.PACKRAT_ITOPS_EMAIL ? process.env.PACKRAT_ITOPS_EMAIL.split(',') : /* istanbul ignore next */ [],
+        smtpHost: process.env.PACKRAT_SMTP_HOST ? process.env.PACKRAT_SMTP_HOST : /* istanbul ignore next */ 'smtp.si.edu',
+        smtpPort: process.env.PACKRAT_SMTP_PORT ? parseInt(process.env.PACKRAT_SMTP_PORT) : /* istanbul ignore next */ undefined,
+        smtpSecure: process.env.PACKRAT_SMTP_SECURE ? convertStringSettingToBoolean(process.env.PACKRAT_SMTP_SECURE) : /* istanbul ignore next */ undefined,
+        smtpAuthUser: process.env.PACKRAT_SMTP_AUTHUSER ? process.env.PACKRAT_SMTP_AUTHUSER : /* istanbul ignore next */ undefined,
+        smtpAuthPassword: process.env.PACKRAT_SMTP_AUTHPASSWORD ? process.env.PACKRAT_SMTP_AUTHPASSWORD : /* istanbul ignore next */ undefined,
+    },
     log: {
         root: process.env.PACKRAT_LOG_ROOT ? process.env.PACKRAT_LOG_ROOT : /* istanbul ignore next */ './var/logs'
     },
@@ -161,3 +177,16 @@ export const Config: ConfigType = {
         type: WORKFLOW_TYPE.PACKRAT,
     }
 };
+
+function convertStringSettingToBoolean(input: string): boolean {
+    switch (input.toLowerCase()) {
+        case '': return false;
+        case '0': return false;
+        case '1': return true;
+        case 'no': return false;
+        case 'yes': return true;
+        case 'false': return false;
+        case 'true': return true;
+        default: return false;
+    }
+}

server/http/routes/migrate.ts

@@ -6,14 +6,17 @@ import { SceneMigrationPackages } from '../../utils/migration/MigrationData';
 import * as UTIL from '../../tests/db/api';
 
 import { Request, Response } from 'express';
+import * as NS from 'node-schedule';
 
 export async function migrate(request: Request, response: Response): Promise<void> {
-    const migrator: Migrator = new Migrator(request, response);
     try {
+        const migrator: Migrator = new Migrator(request, response);
         if (!await migrator.parseArguments())
             return;
-        await migrator.migrate();
-        response.send(`Migration ${migrator.success ? 'Succeeded' : 'Failed'}<br/>\n${migrator.results}`);
+
+        const soon: Date = new Date(new Date().getTime() + 1000); // start 1 second from now
+        NS.scheduleJob(soon, () => Migrator.launcher(migrator));
+        response.send('Migration Scheduled');
     } catch (error) {
         LOG.error('migrate', LOG.LS.eMIG, error);
     }
@@ -34,6 +37,12 @@ class Migrator {
         this.response = response;
     }
 
+    static async launcher(migrator: Migrator): Promise<boolean> {
+        if (!await migrator.parseArguments())
+            return false;
+        return await migrator.migrate();
+    }
+
     /** Returns false if arguments are invalid */
     async parseArguments(): Promise<boolean> {
         // All entry points below perform migration for objects not yet migrated