diff --git a/api/src/app/index.ts b/api/src/app/index.ts index 5af6505e..dc142be4 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -41,7 +41,7 @@ useContainer(Container); // eslint-disable-line react-hooks/rules-of-hooks // Initialize Search Service const searchService = Container.get(SearchService); - await searchService.ensureIndexes(); + await searchService.setupSearch(); // Add crons to DI container const CronServices = [DigestCron]; diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts index 35b81336..7854fe82 100644 --- a/api/src/contributor/table.ts +++ b/api/src/contributor/table.ts @@ -8,7 +8,7 @@ export const contributorsTable = pgTable("contributors", { recordImportedAt: text("record_imported_at") .notNull() .default(sql`CURRENT_TIMESTAMP`), - runId: text("run_id").notNull().default("initial-run-id"), + runId: text("run_id").notNull(), name: text("name").notNull(), username: text("username").notNull(), url: text("url").notNull().unique(), diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 458b70d8..24bb3ff7 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -1,12 +1,17 @@ import { captureException, cron } from "@sentry/node"; -import { CronJob } from "cron"; + import { ContributionRepository } from "src/contribution/repository"; +import { ContributionRow } from "src/contribution/table"; import { ContributorRepository } from "src/contributor/repository"; +import { ContributorRow } from "src/contributor/table"; +import { CronJob } from "cron"; import { DataService } from "src/data/service"; import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; import { ProjectRepository } from "src/project/repository"; +import { ProjectRow } from "src/project/table"; import { RepositoryRepository } from "src/repository/repository"; +import { SearchService } from "src/search/service"; import { Service } from "typedi"; @Service() @@ -22,6 +27,7 @@ export class DigestCron { private readonly repositoriesRepository: RepositoryRepository, private readonly contributionsRepository: ContributionRepository, private readonly contributorsRepository: ContributorRepository, + private readonly searchService: SearchService, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -67,11 +73,14 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); for (const project of projectsFromDataFolder) { - const [{ id: projectId }] = await this.projectsRepository.upsert({ - ...project, + const projectEntity: ProjectRow = { runId, - id: project.slug, - }); + id: project.slug.replace(/[.]/g, "-"), // NOTE-OB: MeiliSearch doesn't allow dots in ids + name: project.name, + }; + const [{ id: projectId }] = + await this.projectsRepository.upsert(projectEntity); + await this.searchService.upsert("project", projectEntity); let addedRepositoryCount = 0; try { @@ -84,15 +93,16 @@ export class DigestCron { }); const provider = "github"; - const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({ - provider, - name: repoInfo.name, - owner: repoInfo.owner.login, - runId, - projectId, - stars: repoInfo.stargazers_count, - id: `${provider}-${repoInfo.id}`, - }); + const [{ id: repositoryId }] = + await this.repositoriesRepository.upsert({ + provider, + name: repoInfo.name, + owner: repoInfo.owner.login, + runId, + projectId, + stars: repoInfo.stargazers_count, + id: `${provider}-${repoInfo.id}`, + }); addedRepositoryCount++; const issues = await this.githubService.listRepositoryIssues({ @@ -101,18 +111,24 @@ export class DigestCron { }); for (const issue of issues) { - const githubUser = await this.githubService.getUser({ username: issue.user.login }); + const githubUser = await this.githubService.getUser({ + username: issue.user.login, + }); if (githubUser.type !== "User") continue; - const [{ id: contributorId }] = await this.contributorsRepository.upsert({ + const contributorEntity: ContributorRow = { name: githubUser.name || githubUser.login, username: githubUser.login, url: githubUser.html_url, avatarUrl: githubUser.avatar_url, runId, id: `${provider}-${githubUser.login}`, - }); + }; + + const [{ id: contributorId }] = + await this.contributorsRepository.upsert(contributorEntity); + await this.searchService.upsert("contributor", contributorEntity); await this.contributorsRepository.upsertRelationWithRepository({ contributorId, @@ -122,23 +138,32 @@ export class DigestCron { }); const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; - await this.contributionsRepository.upsert({ + const contributionEntity: ContributionRow = { title: issue.title, type, updatedAt: issue.updated_at, activityCount: issue.comments, runId, - url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, + url: + type === "PULL_REQUEST" + ? issue.pull_request.html_url + : issue.html_url, repositoryId, contributorId, id: `${provider}-${issue.id}`, - }); + }; + await this.contributionsRepository.upsert(contributionEntity); + await this.searchService.upsert( + "contribution", + contributionEntity, + ); } - const repoContributors = await this.githubService.listRepositoryContributors({ - owner: repository.owner, - repository: repository.name, - }); + const repoContributors = + await this.githubService.listRepositoryContributors({ + owner: repository.owner, + repository: repository.name, + }); const repoContributorsFiltered = repoContributors.filter( (contributor) => contributor.type === "User", @@ -148,14 +173,17 @@ export class DigestCron { const contributor = await this.githubService.getUser({ username: repoContributor.login, }); - const [{ id: contributorId }] = await this.contributorsRepository.upsert({ + const contributorEntity: ContributorRow = { name: contributor.name || contributor.login, username: contributor.login, url: contributor.html_url, avatarUrl: contributor.avatar_url, runId, id: `${provider}-${contributor.login}`, - }); + }; + const [{ id: contributorId }] = + await this.contributorsRepository.upsert(contributorEntity); + await this.searchService.upsert("contributor", contributorEntity); await this.contributorsRepository.upsertRelationWithRepository({ contributorId, @@ -179,11 +207,18 @@ export class DigestCron { } try { - await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId(runId); + await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId( + runId, + ); await this.contributionsRepository.deleteAllButWithRunId(runId); await this.contributorsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); + await Promise.all([ + this.searchService.deleteAllButWithRunId("project", runId), + this.searchService.deleteAllButWithRunId("contribution", runId), + this.searchService.deleteAllButWithRunId("contributor", runId), + ]); } catch (error) { captureException(error, { tags: { type: "CRON" } }); } diff --git a/api/src/project/table.ts b/api/src/project/table.ts index 01aad9c1..4847c1c7 100644 --- a/api/src/project/table.ts +++ b/api/src/project/table.ts @@ -8,7 +8,7 @@ export const projectsTable = pgTable("projects", { .notNull() .default(sql`CURRENT_TIMESTAMP`), name: text("name").notNull(), - runId: text("run_id").notNull().default("initial-run-id"), + runId: text("run_id").notNull(), }); projectsTable.$inferSelect satisfies ProjectEntity; diff --git a/api/src/search/service.ts b/api/src/search/service.ts index 4ac9436e..2206d641 100644 --- a/api/src/search/service.ts +++ b/api/src/search/service.ts @@ -1,5 +1,6 @@ -import { SearchItem, SearchType } from "./types"; +import { SearchResults, SearchType } from "./types"; +import { BaseEntity } from "@dzcode.io/models/dist/_base"; import { ConfigService } from "src/config/service"; import { LoggerService } from "src/logger/service"; import { MeiliSearch } from "meilisearch"; @@ -25,32 +26,71 @@ export class SearchService { }); } - public search = async (query: string): Promise => { + public search = async (query: string): Promise => { this.logger.info({ message: `Searching for ${query}` }); - return []; + return { + results: [], + }; }; - public index = async ( + public upsert = async ( index: SearchType, - data: SearchItem[], + data: T, ): Promise => { this.logger.info({ - message: `Indexing ${data.length} items in ${index}`, + message: `Upserting "${data.id}" item to ${index}`, + }); + await this.meilisearch.index(index).updateDocuments([data]); + this.logger.info({ message: `Upserted "${data.id}" item to ${index}` }); + }; + + public deleteAllButWithRunId = async ( + index: SearchType, + runId: string, + ): Promise => { + this.logger.info({ + message: `Deleting all ${index} but with runId ${runId}`, + }); + await this.meilisearch.index(index).deleteDocuments({ + filter: `NOT runId=${runId}`, }); - await this.meilisearch.index(index).addDocuments(data); this.logger.info({ - message: `Indexed ${data.length} items in ${index}`, + message: `Deleted all ${index} but with runId ${runId}`, }); }; - public ensureIndexes = async (): Promise => { - await this.meilisearch.createIndex("project"); - this.logger.info({ message: "project index created" }); - - await this.meilisearch.createIndex("contribution"); - this.logger.info({ message: "contribution index created" }); + public setupSearch = async (): Promise => { + await this.setupIndexes(); + await this.updateFilterableAttributes(); + }; - await this.meilisearch.createIndex("contributor"); - this.logger.info({ message: "contributor index created" }); + private setupIndexes = async (): Promise => { + await this.upsertIndex("project"); + await this.upsertIndex("contribution"); + await this.upsertIndex("contributor"); }; + + private async upsertIndex(index: SearchType): Promise { + try { + await this.meilisearch.getIndex(index); + this.logger.info({ message: `${index} index already exists` }); + } catch { + await this.meilisearch.createIndex(index, { + primaryKey: "id", + }); + this.logger.info({ message: `${index} index created` }); + } + } + + private async updateFilterableAttributes(): Promise { + await this.meilisearch + .index("project") + .updateFilterableAttributes(["runId"]); + await this.meilisearch + .index("contribution") + .updateFilterableAttributes(["runId"]); + await this.meilisearch + .index("contributor") + .updateFilterableAttributes(["runId"]); + } } diff --git a/api/src/search/types.ts b/api/src/search/types.ts index 4c34e538..46f846fd 100644 --- a/api/src/search/types.ts +++ b/api/src/search/types.ts @@ -1,13 +1,15 @@ +import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; +import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { GeneralResponse } from "src/app/types"; +import { MultiSearchResponse } from "meilisearch"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; export interface GetSearchResponse extends GeneralResponse { - searchResults: Array; + searchResults: SearchResults; } -export interface SearchItem { - id: string; - title: string; - type: SearchType; -} +export type SearchResults = MultiSearchResponse< + ProjectEntity | ContributionEntity | ContributorEntity +>; export type SearchType = "project" | "contribution" | "contributor";