Merge pull request #620 from dzcode-io/index-models-during-cron-job

Indexing models during cron job

api/src/app/index.ts

@@ -41,7 +41,7 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks
 
   // Initialize Search Service
   const searchService = Container.get(SearchService);
-  await searchService.ensureIndexes();
+  await searchService.setupSearch();
 
   // Add crons to DI container
   const CronServices = [DigestCron];

api/src/contributor/table.ts

@@ -8,7 +8,7 @@ export const contributorsTable = pgTable("contributors", {
   recordImportedAt: text("record_imported_at")
     .notNull()
     .default(sql`CURRENT_TIMESTAMP`),
-  runId: text("run_id").notNull().default("initial-run-id"),
+  runId: text("run_id").notNull(),
   name: text("name").notNull(),
   username: text("username").notNull(),
   url: text("url").notNull().unique(),

api/src/digest/cron.ts

@@ -1,12 +1,17 @@
 import { captureException, cron } from "@sentry/node";
-import { CronJob } from "cron";
+
 import { ContributionRepository } from "src/contribution/repository";
+import { ContributionRow } from "src/contribution/table";
 import { ContributorRepository } from "src/contributor/repository";
+import { ContributorRow } from "src/contributor/table";
+import { CronJob } from "cron";
 import { DataService } from "src/data/service";
 import { GithubService } from "src/github/service";
 import { LoggerService } from "src/logger/service";
 import { ProjectRepository } from "src/project/repository";
+import { ProjectRow } from "src/project/table";
 import { RepositoryRepository } from "src/repository/repository";
+import { SearchService } from "src/search/service";
 import { Service } from "typedi";
 
 @Service()
@@ -22,6 +27,7 @@ export class DigestCron {
     private readonly repositoriesRepository: RepositoryRepository,
     private readonly contributionsRepository: ContributionRepository,
     private readonly contributorsRepository: ContributorRepository,
+    private readonly searchService: SearchService,
   ) {
     const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron");
     new SentryCronJob(
@@ -67,11 +73,14 @@ export class DigestCron {
     const projectsFromDataFolder = await this.dataService.listProjects();
 
     for (const project of projectsFromDataFolder) {
-      const [{ id: projectId }] = await this.projectsRepository.upsert({
-        ...project,
+      const projectEntity: ProjectRow = {
         runId,
-        id: project.slug,
-      });
+        id: project.slug.replace(/[.]/g, "-"), // NOTE-OB: MeiliSearch doesn't allow dots in ids
+        name: project.name,
+      };
+      const [{ id: projectId }] =
+        await this.projectsRepository.upsert(projectEntity);
+      await this.searchService.upsert("project", projectEntity);
 
       let addedRepositoryCount = 0;
       try {
@@ -84,15 +93,16 @@ export class DigestCron {
             });
 
             const provider = "github";
-            const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({
-              provider,
-              name: repoInfo.name,
-              owner: repoInfo.owner.login,
-              runId,
-              projectId,
-              stars: repoInfo.stargazers_count,
-              id: `${provider}-${repoInfo.id}`,
-            });
+            const [{ id: repositoryId }] =
+              await this.repositoriesRepository.upsert({
+                provider,
+                name: repoInfo.name,
+                owner: repoInfo.owner.login,
+                runId,
+                projectId,
+                stars: repoInfo.stargazers_count,
+                id: `${provider}-${repoInfo.id}`,
+              });
             addedRepositoryCount++;
 
             const issues = await this.githubService.listRepositoryIssues({
@@ -101,18 +111,24 @@ export class DigestCron {
             });
 
             for (const issue of issues) {
-              const githubUser = await this.githubService.getUser({ username: issue.user.login });
+              const githubUser = await this.githubService.getUser({
+                username: issue.user.login,
+              });
 
               if (githubUser.type !== "User") continue;
 
-              const [{ id: contributorId }] = await this.contributorsRepository.upsert({
+              const contributorEntity: ContributorRow = {
                 name: githubUser.name || githubUser.login,
                 username: githubUser.login,
                 url: githubUser.html_url,
                 avatarUrl: githubUser.avatar_url,
                 runId,
                 id: `${provider}-${githubUser.login}`,
-              });
+              };
+
+              const [{ id: contributorId }] =
+                await this.contributorsRepository.upsert(contributorEntity);
+              await this.searchService.upsert("contributor", contributorEntity);
 
               await this.contributorsRepository.upsertRelationWithRepository({
                 contributorId,
@@ -122,23 +138,32 @@ export class DigestCron {
               });
 
               const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE";
-              await this.contributionsRepository.upsert({
+              const contributionEntity: ContributionRow = {
                 title: issue.title,
                 type,
                 updatedAt: issue.updated_at,
                 activityCount: issue.comments,
                 runId,
-                url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url,
+                url:
+                  type === "PULL_REQUEST"
+                    ? issue.pull_request.html_url
+                    : issue.html_url,
                 repositoryId,
                 contributorId,
                 id: `${provider}-${issue.id}`,
-              });
+              };
+              await this.contributionsRepository.upsert(contributionEntity);
+              await this.searchService.upsert(
+                "contribution",
+                contributionEntity,
+              );
             }
 
-            const repoContributors = await this.githubService.listRepositoryContributors({
-              owner: repository.owner,
-              repository: repository.name,
-            });
+            const repoContributors =
+              await this.githubService.listRepositoryContributors({
+                owner: repository.owner,
+                repository: repository.name,
+              });
 
             const repoContributorsFiltered = repoContributors.filter(
               (contributor) => contributor.type === "User",
@@ -148,14 +173,17 @@ export class DigestCron {
               const contributor = await this.githubService.getUser({
                 username: repoContributor.login,
               });
-              const [{ id: contributorId }] = await this.contributorsRepository.upsert({
+              const contributorEntity: ContributorRow = {
                 name: contributor.name || contributor.login,
                 username: contributor.login,
                 url: contributor.html_url,
                 avatarUrl: contributor.avatar_url,
                 runId,
                 id: `${provider}-${contributor.login}`,
-              });
+              };
+              const [{ id: contributorId }] =
+                await this.contributorsRepository.upsert(contributorEntity);
+              await this.searchService.upsert("contributor", contributorEntity);
 
               await this.contributorsRepository.upsertRelationWithRepository({
                 contributorId,
@@ -179,11 +207,18 @@ export class DigestCron {
     }
 
     try {
-      await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId(runId);
+      await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId(
+        runId,
+      );
       await this.contributionsRepository.deleteAllButWithRunId(runId);
       await this.contributorsRepository.deleteAllButWithRunId(runId);
       await this.repositoriesRepository.deleteAllButWithRunId(runId);
       await this.projectsRepository.deleteAllButWithRunId(runId);
+      await Promise.all([
+        this.searchService.deleteAllButWithRunId("project", runId),
+        this.searchService.deleteAllButWithRunId("contribution", runId),
+        this.searchService.deleteAllButWithRunId("contributor", runId),
+      ]);
     } catch (error) {
       captureException(error, { tags: { type: "CRON" } });
     }

api/src/project/table.ts

@@ -8,7 +8,7 @@ export const projectsTable = pgTable("projects", {
     .notNull()
     .default(sql`CURRENT_TIMESTAMP`),
   name: text("name").notNull(),
-  runId: text("run_id").notNull().default("initial-run-id"),
+  runId: text("run_id").notNull(),
 });
 
 projectsTable.$inferSelect satisfies ProjectEntity;

api/src/search/service.ts

@@ -1,5 +1,6 @@
-import { SearchItem, SearchType } from "./types";
+import { SearchResults, SearchType } from "./types";
 
+import { BaseEntity } from "@dzcode.io/models/dist/_base";
 import { ConfigService } from "src/config/service";
 import { LoggerService } from "src/logger/service";
 import { MeiliSearch } from "meilisearch";
@@ -25,32 +26,71 @@ export class SearchService {
     });
   }
 
-  public search = async (query: string): Promise<SearchItem[]> => {
+  public search = async (query: string): Promise<SearchResults> => {
     this.logger.info({ message: `Searching for ${query}` });
-    return [];
+    return {
+      results: [],
+    };
   };
 
-  public index = async (
+  public upsert = async <T extends BaseEntity>(
     index: SearchType,
-    data: SearchItem[],
+    data: T,
   ): Promise<void> => {
     this.logger.info({
-      message: `Indexing ${data.length} items in ${index}`,
+      message: `Upserting "${data.id}" item to ${index}`,
+    });
+    await this.meilisearch.index(index).updateDocuments([data]);
+    this.logger.info({ message: `Upserted "${data.id}" item to ${index}` });
+  };
+
+  public deleteAllButWithRunId = async (
+    index: SearchType,
+    runId: string,
+  ): Promise<void> => {
+    this.logger.info({
+      message: `Deleting all ${index} but with runId ${runId}`,
+    });
+    await this.meilisearch.index(index).deleteDocuments({
+      filter: `NOT runId=${runId}`,
     });
-    await this.meilisearch.index(index).addDocuments(data);
     this.logger.info({
-      message: `Indexed ${data.length} items in ${index}`,
+      message: `Deleted all ${index} but with runId ${runId}`,
     });
   };
 
-  public ensureIndexes = async (): Promise<void> => {
-    await this.meilisearch.createIndex("project");
-    this.logger.info({ message: "project index created" });
-
-    await this.meilisearch.createIndex("contribution");
-    this.logger.info({ message: "contribution index created" });
+  public setupSearch = async (): Promise<void> => {
+    await this.setupIndexes();
+    await this.updateFilterableAttributes();
+  };
 
-    await this.meilisearch.createIndex("contributor");
-    this.logger.info({ message: "contributor index created" });
+  private setupIndexes = async (): Promise<void> => {
+    await this.upsertIndex("project");
+    await this.upsertIndex("contribution");
+    await this.upsertIndex("contributor");
   };
+
+  private async upsertIndex(index: SearchType): Promise<void> {
+    try {
+      await this.meilisearch.getIndex(index);
+      this.logger.info({ message: `${index} index already exists` });
+    } catch {
+      await this.meilisearch.createIndex(index, {
+        primaryKey: "id",
+      });
+      this.logger.info({ message: `${index} index created` });
+    }
+  }
+
+  private async updateFilterableAttributes(): Promise<void> {
+    await this.meilisearch
+      .index("project")
+      .updateFilterableAttributes(["runId"]);
+    await this.meilisearch
+      .index("contribution")
+      .updateFilterableAttributes(["runId"]);
+    await this.meilisearch
+      .index("contributor")
+      .updateFilterableAttributes(["runId"]);
+  }
 }

api/src/search/types.ts

@@ -1,13 +1,15 @@
+import { ContributionEntity } from "@dzcode.io/models/dist/contribution";
+import { ContributorEntity } from "@dzcode.io/models/dist/contributor";
 import { GeneralResponse } from "src/app/types";
+import { MultiSearchResponse } from "meilisearch";
+import { ProjectEntity } from "@dzcode.io/models/dist/project";
 
 export interface GetSearchResponse extends GeneralResponse {
-  searchResults: Array<SearchItem>;
+  searchResults: SearchResults;
 }
 
-export interface SearchItem {
-  id: string;
-  title: string;
-  type: SearchType;
-}
+export type SearchResults = MultiSearchResponse<
+  ProjectEntity | ContributionEntity | ContributorEntity
+>;
 
 export type SearchType = "project" | "contribution" | "contributor";