Refactor adoption service and controller; enhance adoption retrieval …

…with filtering options

backend/src/__tests__/services/calendarClockServiceTests.ts

@@ -10,8 +10,7 @@ import type { MockConfig, SeatsMockConfig } from '../__mock__/types.js';
 import { MetricDailyResponseType } from '../../models/metrics.model.js';
 import Database from '../../database.js';
 import 'dotenv/config';
-import seatsExample from '../__mock__/seats-gen/seatsExampleTest.json'; type: 'json';
-
+import seatsExample from '../__mock__/seats-gen/seatsExampleTest.json';
 
 let members: any[] = [];
 if (!process.env.MONGODB_URI) throw new Error('MONGODB_URI is not defined');

backend/src/__tests__/services/teams.service.spec.ts

@@ -27,7 +27,7 @@ describe('team.service.spec.ts test', () => {
     const bulkOps = members.map((member: any) => ({
       updateOne: {
         filter: { org, id: member.id },
-        update: member,
+        update: { $set: member },
         upsert: true
       }
     }));

backend/src/controllers/adoption.controller.ts

@@ -4,15 +4,23 @@ import adoptionService from '../services/adoption.service.js';
 
 class AdoptionController {
   async getAdoptions(req: Request, res: Response): Promise<void> {
-    const org = req.query.org?.toString()
-    const { daysInactive, precision } = req.query;
-    const _daysInactive = Number(daysInactive);
-    if (!daysInactive || isNaN(_daysInactive)) {
-      res.status(400).json({ error: 'daysInactive query parameter is required' });
-      return;
-    }
+    const org = req.query.org?.toString();
+    const enterprise = req.query.enterprise?.toString();
+    const team = req.query.team?.toString();
+    // const { daysInactive, precision } = req.query;
     try {
-      const adoptions = await adoptionService.getAllAdoptions();
+      const adoptions = await adoptionService.getAllAdoptions2({
+        filter: {
+          org,
+          ...enterprise ? {enterprise: 'enterprise'} : undefined,
+          team,
+        },
+        projection: {
+          seats: 0,
+          _id: 0,
+          __v: 0,
+        }
+      });
       res.status(200).json(adoptions);
     } catch (error) {
       res.status(500).json(error);

backend/src/controllers/setup.controller.ts

@@ -69,7 +69,6 @@ class SetupController {
         isSetup: app.github.app !== undefined,
         installations: app.github.installations.map(i => ({
           installation: i.installation,
-          ...i.queryService.status
         }))
       };
       return res.json(status);

backend/src/controllers/survey.controller.ts

@@ -55,7 +55,6 @@ class SurveyController {
      const Survey = mongoose.model('Survey');
      const survey = await Survey.create(req.body);
       surveyService.createSurvey(req.body);
-      console.log('Creating survey (controller):', survey);
       res.status(201).json(survey);
     } catch (error) {
       res.status(500).json(error);

backend/src/database.ts

@@ -22,7 +22,7 @@ class Database {
         connectTimeoutMS: 30000,          // Connection timeout (e.g., 30 seconds)
         serverSelectionTimeoutMS: 30000,  // Server selection timeout
       });
-    //  this.mongoose.set('debug', false);
+      // this.mongoose.set('debug', false);
       mongoose.set('debug', (collectionName: string, methodName: string, ...methodArgs: unknown[]) => {
         const msgMapper = (m: unknown) => {
           return util.inspect(m, false, 10, true)
@@ -146,7 +146,7 @@ class Database {
         repositories: [RepositorySchema]
       }
     }));
-    // Team Schema 🏢
+
     const teamSchema = new Schema({
       org: { type: String, required: true },
       team: String,
@@ -167,11 +167,10 @@ class Database {
       timestamps: true
     });
 
-    // Member Schema 👥
     const memberSchema = new Schema({
       org: { type: String, required: true },
       login: { type: String, required: true },
-      id: { type: Number, required: true, unique: true }, // renamed from id
+      id: { type: Number, required: true },
       node_id: String,
       avatar_url: String,
       gravatar_id: String,
@@ -195,27 +194,21 @@ class Database {
     }, {
       timestamps: true,
     });
-
+    memberSchema.index({ org: 1, login: 1, id: 1 }, { unique: true });
     memberSchema.virtual('seats', {
       ref: 'Seats',
       localField: '_id',
       foreignField: 'assignee'
     });
-
-    memberSchema.index({ _id: 1, login: 1, id: 1 });
 
-    // TeamMember Association Schema 🤝
     const teamMemberSchema = new Schema({
       team: { type: Schema.Types.ObjectId, ref: 'Team', required: true },
       member: { type: Schema.Types.ObjectId, ref: 'Member', required: true }
     }, {
       timestamps: false
     });
-
-    // Create indexes for faster queries 🔍
     teamMemberSchema.index({ team: 1, member: 1 }, { unique: true });
 
-    // Create models 📦
     mongoose.model('Team', teamSchema);
     mongoose.model('Member', memberSchema);
     mongoose.model('TeamMember', teamMemberSchema);
@@ -229,6 +222,7 @@ class Database {
       last_activity_editor: String,
       queryAt: Date,
       assignee_id: Number,
+      assignee_login: String,
       assignee: {
         type: Schema.Types.ObjectId,
         ref: 'Member'
@@ -243,42 +237,54 @@ class Database {
 
     mongoose.model('Seats', seatsSchema);
 
-    mongoose.model('Survey', new mongoose.Schema({
-      id: Number,
-      userId: String,
-      org: String,
-      repo: String,
-      prNumber: String,
-      usedCopilot: Boolean,
-      percentTimeSaved: Number,
-      reason: String,
-      timeUsedFor: String
-    }, {
-      timestamps: true
-    }));
-
-
     const adoptionSchema = new Schema({
+      enterprise: String,
+      org: String,
+      team: String,
       date: {
-      type: Date,
-      required: true,
-      unique: true
+        type: Date,
+        required: true
       },
       totalSeats: Number,
       totalActive: Number,
       totalInactive: Number,
       seats: [{
-        type: Schema.Types.ObjectId,
-        ref: 'Seats'
+        login: String,
+        last_activity_at: Date,
+        last_activity_editor: String,
+        _assignee: {
+          required: true,
+          type: Schema.Types.ObjectId,
+          ref: 'Member'
+        },
+        _seat: {
+          required: true,
+          type: Schema.Types.ObjectId,
+          ref: 'Seats'
+        }
       }]
     }, {
       timestamps: true
     });
 
     // Create indexes
-    adoptionSchema.index({ date: 1 });
+    adoptionSchema.index({ enterprise: 1, org: 1, team: 1, date: 1 }, { unique: true });
 
     mongoose.model('Adoption', adoptionSchema);
+
+    mongoose.model('Survey', new mongoose.Schema({
+      id: Number,
+      userId: String,
+      org: String,
+      repo: String,
+      prNumber: String,
+      usedCopilot: Boolean,
+      percentTimeSaved: Number,
+      reason: String,
+      timeUsedFor: String
+    }, {
+      timestamps: true
+    }));
   }
 
   async disconnect() {

backend/src/github.ts

@@ -36,13 +36,13 @@ export interface GitHubInput {
 }
 class GitHub {
   app?: App;
+  queryService?: QueryService;
   webhooks?: Express;
   input: GitHubInput;
   expressApp: Express;
   installations = [] as {
     installation: Endpoints["GET /app/installations"]["response"]["data"][0],
     octokit: Octokit
-    queryService: QueryService
   }[];
   status = 'starting';
   cronExpression = '0 * * * * *';
@@ -88,23 +88,22 @@ class GitHub {
       logger.error('Failed to create webhook middleware')
     }
 
+    this.queryService = new QueryService(this.app, {
+      cronTime: this.cronExpression
+    });
+    logger.info(`CRON task ${this.cronExpression} started`);
     for await (const { octokit, installation } of this.app.eachInstallation.iterator()) {
       if (!installation.account?.login) return;
-      const queryService = new QueryService(installation.account.login, octokit, {
-        cronTime: this.cronExpression
-      });
       this.installations.push({
         installation,
-        octokit,
-        queryService
+        octokit
       });
-      logger.info(`${installation.account?.login} cron task ${this.cronExpression} started`);
     }
     return this.app;
   }
 
   disconnect = () => {
-    this.installations.forEach((i) => i.queryService.delete())
+    this.queryService?.delete();
     this.installations = [];
   }
 

backend/src/index.ts

@@ -1,6 +1,7 @@
 import App from './app.js';
 
-const app = new App(Number(process.env.PORT) || 80);
+const DEFAULT_PORT = 80;
+const app = new App(Number(process.env.PORT) || DEFAULT_PORT);
 
 ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach(signal => {
   process.on(signal, async () => {

backend/src/routes/index.ts

@@ -29,7 +29,7 @@ router.get('/metrics/totals', metricsController.getMetricsTotals);
 
 router.get('/seats', SeatsController.getAllSeats);
 router.get('/seats/activity', adoptionController.getAdoptions);
-router.get('/seats/activity/totals', SeatsController.getActivityTotals);
+router.get('/seats/activity/totals', adoptionController.getAdoptionTotals);
 router.get('/seats/:id', SeatsController.getSeat);
 
 router.get('/teams', teamsController.getAllTeams);

backend/src/services/adoption.service.ts

@@ -1,24 +1,16 @@
 import mongoose from 'mongoose';
 import logger from './logger.js';
+import { SeatEntry } from './copilot.seats.service.js';
 
-interface AdoptionType {
-  date: Date;
-  totalSeats: number;
-  totalActive: number;
-  totalInactive: number;
-  seats: mongoose.Schema.Types.ObjectId[];
-}
 
 export class AdoptionService {
   constructor() {
   }
 
-  async createAdoption(adoptionData: Partial<AdoptionType>): Promise<AdoptionType> {
-    const adoptionModel = mongoose.model<AdoptionType>('Adoption');
-    // console.log('adoptionData', adoptionData)
+  async createAdoption(adoptionData: any): Promise<AdoptionType> {
+    const adoptionModel = mongoose.model('Adoption');
     try {
-      const adoption = new adoptionModel(adoptionData);
-      return await adoption.save();
+      await adoptionModel.create(adoptionData);
     } catch (error) {
       logger.error('Error creating adoption:', error);
       throw error;
@@ -71,6 +63,55 @@ export class AdoptionService {
       throw error;
     }
   }
+
+  async getAllAdoptions2(options: {
+    filter?: {
+      org?: string,
+      enterprise?: string,
+      team?: string,
+    },
+    projection?: any
+  }): Promise<AdoptionType[]> {
+    const adoptionModel = mongoose.model<AdoptionType>('Adoption');
+    try {
+      return await adoptionModel
+        .find(
+          options.filter || {},
+          options.projection || { _id: 0, __v: 0 }
+        )
+        .populate('seats')
+        .sort({ date: -1 });
+    } catch (error) {
+      logger.error('Error fetching all adoptions:', error);
+      throw error;
+    }
+  }
+
+  calculateAdoptionTotals(queryAt: Date, data: SeatEntry[], daysInactive = 30) {
+    return data.reduce((acc, activity) => {
+      if (!activity.last_activity_at) {
+        acc.totalInactive++;
+        return acc;
+      }
+      const fromTime = (new Date(activity.last_activity_at)).getTime() || 0;
+      const toTime = queryAt.getTime();
+      const diff = Math.floor((toTime - fromTime) / 86400000);
+      const dateIndex = new Date(queryAt);
+      dateIndex.setUTCMinutes(0, 0, 0);
+      if (activity.last_activity_at && activity.last_activity_editor) {
+        if (diff > daysInactive) {
+          acc.totalActive++;
+        } else {
+          acc.totalInactive++;
+        }
+      }
+      return acc;
+    }, {
+      totalSeats: data.length,
+      totalActive: 0,
+      totalInactive: 0,
+    });
+  }
 }
 
 export default new AdoptionService();

backend/src/services/copilot.seats.service.ts

@@ -4,6 +4,7 @@ import { components } from "@octokit/openapi-types";
 import mongoose from 'mongoose';
 import { MemberActivityType, MemberType } from 'models/teams.model.js';
 import fs from 'fs';
+import adoptionService from './adoption.service.js';
 
 type _Seat = any;// NonNullable<Endpoints["GET /orgs/{org}/copilot"]["response"]["data"]["seats"]>[0];
 export interface SeatEntry extends _Seat {
@@ -34,12 +35,16 @@ class SeatsService {
       .sort({ queryAt: -1 })  // -1 for descending order
       .select('queryAt');
 
+    console.log(`Latest query at: ${latestQuery?.queryAt}`);
+
     const seats = await Seats.find({
       ...(org ? { org } : {}),
       queryAt: latestQuery?.queryAt
     })
       .populate('assignee')
       .sort({ last_activity_at: -1 });  // DESC ordering ⬇️
+
+    console.log(`Found ${seats.length} seats for ${org || 'all orgs'} at ${latestQuery?.queryAt}`);
     return seats;
   }
 
@@ -51,7 +56,7 @@ class SeatsService {
     if (!member) {
       throw `Member with id ${id} not found`
     }
-  
+
     return Seats.find({
       assignee: member._id
     })
@@ -86,82 +91,78 @@ class SeatsService {
   async insertSeats(org: string, queryAt: Date, data: SeatEntry[], team?: string) {
     const Members = mongoose.model('Member');
     const Seats = mongoose.model('Seats');
-    const seatIds = [];
-    for (const seat of data) {
-      const member = await Members.findOneAndUpdate(
-        { org, id: seat.assignee.id },
-        {
-          ...team ? { team } : undefined,
-          org,
-          id: seat.assignee.id,
-          login: seat.assignee.login,
-          node_id: seat.assignee.node_id,
-          avatar_url: seat.assignee.avatar_url,
-          gravatar_id: seat.assignee.gravatar_id || '',
-          url: seat.assignee.url,
-          html_url: seat.assignee.html_url,
-          followers_url: seat.assignee.followers_url,
-          following_url: seat.assignee.following_url,
-          gists_url: seat.assignee.gists_url,
-          starred_url: seat.assignee.starred_url,
-          subscriptions_url: seat.assignee.subscriptions_url,
-          organizations_url: seat.assignee.organizations_url,
-          repos_url: seat.assignee.repos_url,
-          events_url: seat.assignee.events_url,
-          received_events_url: seat.assignee.received_events_url,
-          type: seat.assignee.type,
-          site_admin: seat.assignee.site_admin,
-        },
-        { upsert: false }
-      );
 
-      // const [assigningTeam] = seat.assigning_team ? await Team.findOrCreate({
-      //   where: { id: seat.assigning_team.id },
-      //   defaults: {
-      //     org,
-      //     id: seat.assigning_team.id,
-      //     node_id: seat.assigning_team.node_id,
-      //     url: seat.assigning_team.url,
-      //     html_url: seat.assigning_team.html_url,
-      //     name: seat.assigning_team.name,
-      //     slug: seat.assigning_team.slug,
-      //     description: seat.assigning_team.description,
-      //     privacy: seat.assigning_team.privacy,
-      //     notification_setting: seat.assigning_team.notification_setting,
-      //     permission: seat.assigning_team.permission,
-      //     members_url: seat.assigning_team.members_url,
-      //     repositories_url: seat.assigning_team.repositories_url
-      //   }
-      // }) : [null];
-
-      if (member._id) {
-        const _seat = await Seats.create({
-          queryAt,
-          org,
-          team,
-          ...seat,
-          assignee: member._id,
-          // assigning_team_id: assigningTeam?.id
-        });  
-        seatIds.push(_seat._id);
+    const memberUpdates = data.map(seat => ({
+      updateOne: {
+        filter: { org, id: seat.assignee.id },
+        update: {
+          $set: {
+            ...team ? { team } : undefined,
+            org,
+            id: seat.assignee.id,
+            login: seat.assignee.login,
+            node_id: seat.assignee.node_id,
+            avatar_url: seat.assignee.avatar_url,
+            gravatar_id: seat.assignee.gravatar_id || '',
+            url: seat.assignee.url,
+            html_url: seat.assignee.html_url,
+            followers_url: seat.assignee.followers_url,
+            following_url: seat.assignee.following_url,
+            gists_url: seat.assignee.gists_url,
+            starred_url: seat.assignee.starred_url,
+            subscriptions_url: seat.assignee.subscriptions_url,
+            organizations_url: seat.assignee.organizations_url,
+            repos_url: seat.assignee.repos_url,
+            events_url: seat.assignee.events_url,
+            received_events_url: seat.assignee.received_events_url,
+            type: seat.assignee.type,
+            site_admin: seat.assignee.site_admin,
+          }
+        },
+        upsert: true,
       }
+    }));
+    await Members.bulkWrite(memberUpdates);
+
+    const updatedMembers = await Members.find({
+      org,
+      id: { $in: data.map(seat => seat.assignee.id) }
+    });
+
+    const seatsData = data.map((seat) => ({
+      queryAt,
+      org,
+      team,
+      ...seat,
+      assignee_id: seat.assignee.id,
+      assignee_login: seat.assignee.login,
+      assignee: updatedMembers.find(m => m.id === seat.assignee.id)?._id
+    }));
+    const seatResults = await Seats.insertMany(seatsData);
+    console.log(`Inserted ${seatResults.length} seats for ${org} at ${queryAt}`);
 
-      // await Seat.create({
-      //   queryAt,
-      //   org,
-      //   team,
-      //   created_at: seat.created_at,
-      //   updated_at: seat.updated_at,
-      //   pending_cancellation_date: seat.pending_cancellation_date,
-      //   last_activity_at: seat.last_activity_at,
-      //   last_activity_editor: seat.last_activity_editor,
-      //   plan_type: seat.plan_type,
-      //   assignee_id: assignee.id,
-      //   assigning_team_id: assigningTeam?.id
-      // });
+    const adoptionData = {
+      enterprise: null,
+      org: org,
+      team: null,
+      date: queryAt,
+      ...adoptionService.calculateAdoptionTotals(queryAt, data),
+      seats: seatResults.map(seat => ({
+        login: seat.assignee_login,
+        last_activity_at: seat.last_activity_at,
+        last_activity_editor: seat.last_activity_editor,
+        _assignee: seat.assignee,
+        _seat: seat._id,
+      }))
     }
 
-    return seatIds;
+    await adoptionService.createAdoption(adoptionData);
+
+    return {
+      seats: seatResults,
+      members: updatedMembers,
+      adoption: adoptionData
+    }
   }
 
   async getMembersActivity(params: {
@@ -171,8 +172,11 @@ class SeatsService {
     since?: string;
     until?: string;
   } = {}): Promise<any> { // Promise<MemberDailyActivity> {
-    console.log('getMembersActivity', params);
     const Seats = mongoose.model('Seats');
+    // const seats = await Seats.find({})
+    // return seats.length;
+
+    // return;
     // const Member = mongoose.model('Member');
     const { org, daysInactive = 30, precision = 'day', since, until } = params;
     const assignees: MemberActivityType[] = await Seats.aggregate([
@@ -214,8 +218,6 @@ class SeatsService {
     // .allowDiskUse(true)
     // .explain('executionStats');
 
-    console.log('assignees', assignees.length);
-
     const activityDays: MemberDailyActivity = {};
     assignees.forEach((assignee) => {
       if (!assignee.activity) return;
@@ -261,8 +263,6 @@ class SeatsService {
         .sort(([dateA], [dateB]) => new Date(dateA).getTime() - new Date(dateB).getTime())
     );
 
-    console.log('sortedActivityDays done');
-
     fs.writeFileSync('sortedActivityDays.json', JSON.stringify(sortedActivityDays, null, 2), 'utf-8');
 
     return sortedActivityDays;
@@ -277,18 +277,17 @@ class SeatsService {
     const Member = mongoose.model('Member');
 
     const assignees: MemberType[] = await Member
-    .aggregate([
-      {
-        $lookup: {
-          from: 'seats',          // MongoDB collection name (lowercase)
-          localField: '_id',      // Member model field
-          foreignField: 'assignee', // Seats model field
-          as: 'activity'            // Name for the array of seats
+      .aggregate([
+        {
+          $lookup: {
+            from: 'seats',          // MongoDB collection name (lowercase)
+            localField: '_id',      // Member model field
+            foreignField: 'assignee', // Seats model field
+            as: 'activity'            // Name for the array of seats
+          }
         }
-      }
-    ]);
+      ]);
 
-      console.log('assignees', assignees);
     const activityTotals = assignees.reduce((totals, assignee) => {
       if (assignee.activity) {
         totals[assignee.login] = assignee.activity.reduce((totalMs, activity, index) => {
@@ -309,7 +308,6 @@ class SeatsService {
       return totals;
     }, {} as { [assignee: string]: number });
 
-    console.log('activityTotals done');
     return Object.entries(activityTotals).sort((a: any, b: any) => b[1] - a[1]);
   }
 }

backend/src/services/metrics.service.ts

@@ -305,7 +305,6 @@ class MetricsService {
 
   async insertMetrics(org: string, data: MetricDailyResponseType[], team?: string) {
     const Metrics = mongoose.model('Metrics');
-    console.log('Inserting metrics for org:', org, 'and team:', team);
     for (const day of data) {
       // const parts = day.date.split('-').map(Number);
       // const date = new Date(Date.UTC(parts[0], parts[1] - 1, parts[2] + 1));

backend/src/services/query.service.ts

@@ -2,12 +2,13 @@ import { CronJob, CronJobParams, CronTime } from 'cron';
 import logger from './logger.js';
 import { insertUsage } from '../models/usage.model.js';
 import SeatService, { SeatEntry } from '../services/copilot.seats.service.js';
-import { Octokit } from 'octokit';
+import { App, Octokit } from 'octokit';
 import { MetricDailyResponseType } from '../models/metrics.model.js';
 import mongoose from 'mongoose';
 import metricsService from './metrics.service.js';
 import teamsService from './teams.service.js';
-import AdoptionService from './adoption.service.js';
+import adoptionService from './adoption.service.js';
+import { time } from 'console';
 
 const DEFAULT_CRON_EXPRESSION = '0 * * * *';
 class QueryService {
@@ -19,54 +20,114 @@ class QueryService {
     teamsAndMembers: false,
     dbInitialized: false
   };
+  app: App;
 
   constructor(
-    public org: string,
-    public octokit: Octokit,
+    app: App,
     options?: Partial<CronJobParams>
   ) {
+    this.app = app;
     // Consider Timezone
     const _options: CronJobParams = {
       cronTime: DEFAULT_CRON_EXPRESSION,
       onTick: () => {
-        this.task(this.org);
+        this.task();
       },
       start: true,
       ...options
     }
     this.cronJob = CronJob.from(_options);
-    this.task(this.org);
+    this.task();
   }
 
   delete() {
     this.cronJob.stop();
   }
 
-  private async task(org: string) {
+  private async task() {
     const queryAt = new Date();
-    try {
-      const queries = [
-        this.queryCopilotUsageMetrics(org).then(() => this.status.usage = true),
-        this.queryCopilotUsageMetricsNew(org).then(() => this.status.metrics = true),
-        this.queryCopilotSeatAssignments(org, queryAt).then(() => this.status.copilotSeats = true),
-      ]
+    const tasks = [];
+    for await (const { octokit, installation } of this.app.eachInstallation.iterator()) {
+      if (!installation.account?.login) return;
+      const org = installation.account.login;
+      tasks.push(this.orgTask(octokit, queryAt, org));
+    }
+    const results = await Promise.all(tasks);
+
+    const uniqueSeats = new Map();
+    results.forEach((result) => {
+      if (result?.copilotSeatAssignments) {
+        const { adoption } = result.copilotSeatAssignments;
+        if (adoption) {
+          adoption.seats.forEach((seat) => {
+            if (!uniqueSeats.has(seat.login)) {
+              uniqueSeats.set(seat.login, seat);
+            }
+          });
+        }
+      }
+    });
+
+    const uniqueSeatsArray = Array.from(uniqueSeats.values());
+    const enterpriseAdoptionData = {
+      enterprise: 'enterprise',
+      org: null,
+      team: null,
+      date: queryAt,
+      ...adoptionService.calculateAdoptionTotals(queryAt, uniqueSeatsArray),
+      seats: uniqueSeatsArray
+    }
 
-      this.queryTeamsAndMembers(org).then(() =>
-        this.status.teamsAndMembers = true
-      )
+    adoptionService.createAdoption(enterpriseAdoptionData);
+  }
 
-      await Promise.all(queries).then(() => {
-        this.status.dbInitialized = true;
-      });
+  private async orgTask(octokit: Octokit, queryAt: Date, org: string) {
+    try {
+      let teamsAndMembers = null;
+      const mostRecentEntry = await teamsService.getLastUpdatedAt();
+      const msSinceLastUpdate = new Date().getTime() - new Date(mostRecentEntry).getTime();
+      const hoursSinceLastUpdate = msSinceLastUpdate / 1000 / 60 / 60;
+      logger.info(`Teams & Members updated ${hoursSinceLastUpdate.toFixed(2)} hours ago for ${org}.`);
+      if (mostRecentEntry && hoursSinceLastUpdate > 24) {
+        logger.info(`Updating teams and members for ${org}`);
+        teamsAndMembers = await this.queryTeamsAndMembers(octokit, org).then(() => {
+          this.status.teamsAndMembers = true;
+        });
+      }
+
+      const queries = [
+        this.queryCopilotUsageMetrics(octokit, org).then(result => {
+          this.status.usage = true;
+          return result;
+        }),
+        this.queryCopilotUsageMetricsNew(octokit, org).then(result => {
+          this.status.metrics = true;
+          return result;
+        }),
+        this.queryCopilotSeatAssignments(octokit, org, queryAt).then(result => {
+          this.status.copilotSeats = true;
+          return result;
+        }),
+      ];
+
+      const [usageMetrics, usageMetricsNew, copilotSeatAssignments] = await Promise.all(queries);
+      this.status.dbInitialized = true;
+
+      return {
+        usageMetrics,
+        usageMetricsNew,
+        copilotSeatAssignments,
+        teamsAndMembers
+      }
     } catch (error) {
       logger.error(error);
     }
     logger.info(`${org} finished task`);
   }
 
-  public async queryCopilotUsageMetricsNew(org: string, team?: string) {
+  public async queryCopilotUsageMetricsNew(octokit: Octokit, org: string, team?: string) {
     try {
-      const metricsArray = await this.octokit.paginate<MetricDailyResponseType>(
+      const metricsArray = await octokit.paginate<MetricDailyResponseType>(
         'GET /orgs/{org}/copilot/metrics',
         {
           org: org
@@ -79,22 +140,22 @@ class QueryService {
     }
   }
 
-  public async queryCopilotUsageMetrics(org: string) {
+  public async queryCopilotUsageMetrics(octokit: Octokit, org: string) {
     try {
-      const rsp = await this.octokit.rest.copilot.usageMetricsForOrg({
+      const rsp = await octokit.rest.copilot.usageMetricsForOrg({
         org
       });
 
       insertUsage(org, rsp.data);
-      logger.info(`${this.org} usage metrics updated`);
+      logger.info(`${org} usage metrics updated`);
     } catch (error) {
-      logger.error(`Error updating ${this.org} usage metrics`, error);
+      logger.error(`Error updating ${org} usage metrics`, error);
     }
   }
 
-  public async queryCopilotSeatAssignments(org: string, queryAt: Date) {
+  public async queryCopilotSeatAssignments(octokit: Octokit, org: string, queryAt: Date) {
     try {
-      const rsp = await this.octokit.paginate(this.octokit.rest.copilot.listCopilotSeats, {
+      const rsp = await octokit.paginate(octokit.rest.copilot.listCopilotSeats, {
         org
       }) as { total_seats: number, seats: SeatEntry[] }[];
 
@@ -108,70 +169,26 @@ class QueryService {
         return;
       }
 
-      const seatIds = await SeatService.insertSeats(org, queryAt, seatAssignments.seats);
-
-      const adoptionData = seatAssignments.seats.reduce((acc, activity) => {
-        if (!activity.last_activity_at) {
-          acc.totalInactive++;
-          return acc;
-        }
-        const daysInactive = 30;
-        const fromTime = (new Date(activity.last_activity_at)).getTime() || 0;
-        const toTime = queryAt.getTime();
-        const diff = Math.floor((toTime - fromTime) / 86400000);
-        const dateIndex = new Date(queryAt);
-        dateIndex.setUTCMinutes(0, 0, 0);
-        if (activity.last_activity_at && activity.last_activity_editor) {
-          if (diff > daysInactive) {
-            acc.totalActive++;
-          } else {
-            acc.totalInactive++;
-          }
-        }
-        return acc;
-      }, {
-        date: queryAt,
-        totalSeats: seatAssignments.total_seats,
-        totalActive: 0,
-        totalInactive: 0,
-        seats: seatIds
-      });
-
-      //tmp
-      // const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-      // while (1) {
-      //   const queryAt = new Date();
-      //   console.log('again!');
-      //   await AdoptionService.createAdoption({
-      //     date: queryAt,
-      //     totalSeats: seatAssignments.total_seats,
-      //     totalActive: seatAssignments.seats.filter(seat => seat).length,
-      //     totalInactive: seatAssignments.seats.filter(seat => seat).length,
-      //     seats: seatIds
-      //   });
-      //   await sleep(100);
-      // }
-      //endtmp
-
-      await AdoptionService.createAdoption(adoptionData);
+      const result = await SeatService.insertSeats(org, queryAt, seatAssignments.seats);
 
       logger.info(`${org} seat assignments updated`);
+
+      return result;
     } catch (error) {
-      console.log(error);
       logger.debug(error)
       logger.error('Error querying copilot seat assignments');
     }
   }
 
-  public async queryTeamsAndMembers(org: string) {
+  public async queryTeamsAndMembers(octokit: Octokit, org: string) {
     try {
-      const teams = await this.octokit.paginate(this.octokit.rest.teams.list, {
+      const teams = await octokit.paginate(octokit.rest.teams.list, {
         org
       });
       await teamsService.updateTeams(org, teams);
 
       await Promise.all(
-        teams.map(async (team) => this.octokit.paginate(this.octokit.rest.teams.listMembersInOrg, {
+        teams.map(async (team) => octokit.paginate(octokit.rest.teams.listMembersInOrg, {
           org,
           team_slug: team.slug
         }).then(async (members) =>
@@ -184,7 +201,7 @@ class QueryService {
         )
       )
 
-      const members = await this.octokit.paginate("GET /orgs/{org}/members", {
+      const members = await octokit.paginate("GET /orgs/{org}/members", {
         org
       });
 
@@ -194,14 +211,14 @@ class QueryService {
       const bulkOps = members.map((member) => ({
         updateOne: {
           filter: { org, id: member.id },
-          update: member,
+          update: { $set: member },
           upsert: true
         }
       }));
 
       await Members.bulkWrite(bulkOps, { ordered: false });
 
-      logger.info("Teams & Members successfully updated! 🧑‍🤝‍🧑");
+      logger.info(`${org} teams and members updated`);
     } catch (error) {
       logger.error('Error querying teams', error);
     }

backend/src/services/survey.service.ts

@@ -4,7 +4,6 @@ import mongoose from 'mongoose';
 class SurveyService {
   async createSurvey(survey: SurveyType) {
     const Survey = mongoose.model('Survey');
-    console.log('Creating survey (service):', survey);
     return await Survey.create(survey);
   }
 

backend/src/services/teams.service.ts

@@ -183,9 +183,9 @@ class TeamsService {
     return true;
   }
 
-  async getLastUpdatedAt() {
-    const Team = mongoose.model('Team');
-    const team = await Team.findOne().sort({ updatedAt: -1 });
+  async getLastUpdatedAt(org?: string): Promise<Date> {
+    const Team = mongoose.model('Member');
+    const team = await Team.findOne(org ? { org } : {}).sort({ updatedAt: -1 });
     return team?.updatedAt || new Date(0);
   }
 

frontend/src/app/services/highcharts.service.ts

@@ -430,10 +430,8 @@ export class HighchartsService {
       data: [] as CustomHighchartsPointOptions[]
     };
 
-    console.log(metrics);
     Object.entries(activity).forEach(([date, dateData]) => {
       const currentMetrics = metrics.find(m => m.date.startsWith(date.slice(0, 10)));
-      console.log(date, date.slice(0, 10), currentMetrics);
       if (currentMetrics?.copilot_ide_code_completions) {
         (dailyActiveIdeCompletionsSeries.data).push({
           x: new Date(date).getTime(),
@@ -464,8 +462,6 @@ export class HighchartsService {
       }
     });
 
-    console.log(dailyActiveIdeCompletionsSeries);
-
     return {
       series: [
         dailyActiveIdeCompletionsSeries,