diff --git a/.env.local.sample b/.env.local.sample index 8bdb669a..f0bf2540 100644 --- a/.env.local.sample +++ b/.env.local.sample @@ -4,6 +4,8 @@ RESEARCH_SERVICE_PORT=4030 UNIFIED_DATABASE_SERVICE_PORT=4040 ENTITY_SERVICE_PORT=4050 +DEPLOY_ENV=local + DB_HOST=localhost DB_PORT=3360 DB_DATABASE=wri_restoration_marketplace_api @@ -16,4 +18,9 @@ REDIS_PORT=6379 JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt # Only needed for the unified database service. Most developers should not have this defined. -AIRTABLE_API_KEY +AIRTABLE_API_KEY= +AIRTABLE_BASE_ID= + +# Should only be defined locally for testing +SLACK_API_KEY= +UDB_SLACK_CHANNEL= diff --git a/.github/workflows/deploy-service.yml b/.github/workflows/deploy-service.yml index 10f85770..21e2763b 100644 --- a/.github/workflows/deploy-service.yml +++ b/.github/workflows/deploy-service.yml @@ -71,7 +71,7 @@ jobs: : # Don't build the base image with NODE_ENV because it'll limit the packages that are installed docker build -t terramatch-microservices-base:nx-base . SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - docker build --build-arg NODE_ENV=production --build-arg BUILD_FLAG='--prod --verbose' -f apps/${{ inputs.service }}/Dockerfile -t $SERVICE_IMAGE . + docker build --build-arg NODE_ENV=production --build-arg DEPLOY_ENV=${{ inputs.env }} --build-arg SENTRY_DSN="${{ vars.SENTRY_DSN }}" --build-arg BUILD_FLAG='--prod --verbose' -f apps/${{ inputs.service }}/Dockerfile -t $SERVICE_IMAGE . docker push $SERVICE_IMAGE echo "image=$SERVICE_IMAGE" diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e968eb17..06729c2f 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -46,7 +46,13 @@ jobs: # in a clean way. For some reason, the `run-many` is necessary here. If this line simply uses # nx test database, the connection to the DB gets cut off before the sync is complete. - name: Sync DB Schema - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database --no-cache + run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database --skip-nx-cache + + # Run the UDB service tests in isolation because they require clearing out DB tables and ensuring + # they know exactly what's in them, which is not conducive to parallel runs with other test + # suites. + - name: Unified DB Test + run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test unified-database-service --coverage - name: Test all - run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests + run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests --exclude unified-database-service database diff --git a/README.md b/README.md index 1a571bbb..8be1d257 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ Repository for the Microservices API backend of the TerraMatch service - Node v20.11.1. Using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file) is recommended. - [Docker](https://www.docker.com/) - [CDK CLI](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) (install globally) -- [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html) - [NX](https://nx.dev/getting-started/installation#installing-nx-globally) (install globally) - [NestJS](https://docs.nestjs.com/) (install globally, useful for development) diff --git a/apps/entity-service/Dockerfile b/apps/entity-service/Dockerfile index e7a06cfa..9ce4376e 100644 --- a/apps/entity-service/Dockerfile +++ b/apps/entity-service/Dockerfile @@ -8,8 +8,12 @@ RUN npx nx build entity-service ${BUILD_FLAG} FROM terramatch-microservices-base:nx-base ARG NODE_ENV +ARG SENTRY_DSN +ARG DEPLOY_ENV WORKDIR /app COPY --from=builder /app/builder ./ ENV NODE_ENV=${NODE_ENV} +ENV SENTRY_DSN=${SENTRY_DSN} +ENV DEPLOY_ENV=${DEPLOY_ENV} CMD ["node", "./dist/apps/entity-service/main.js"] diff --git a/apps/entity-service/src/trees/dto/establishment-trees.dto.ts b/apps/entity-service/src/trees/dto/establishment-trees.dto.ts index 29ccfb6c..20b0f2f2 100644 --- a/apps/entity-service/src/trees/dto/establishment-trees.dto.ts +++ b/apps/entity-service/src/trees/dto/establishment-trees.dto.ts @@ -20,7 +20,9 @@ export class EstablishmentsTreesDto extends JsonApiAttributes; @@ -33,7 +35,9 @@ export class EstablishmentsTreesDto extends JsonApiAttributes { ) .map(({ name }) => name) .sort(); + // hidden trees are ignored + await TreeSpeciesFactory.forProjectTreePlanted.create({ + speciesableId: project.id, + hidden: true + }); const siteTreesPlanted = (await TreeSpeciesFactory.forSiteTreePlanted.createMany(2, { speciesableId: site.id })) .map(({ name }) => name) .sort(); + await TreeSpeciesFactory.forSiteTreePlanted.create({ speciesableId: site.id, hidden: true }); const siteNonTrees = (await TreeSpeciesFactory.forSiteNonTree.createMany(3, { speciesableId: site.id })) .map(({ name }) => name) .sort(); + const siteSeedings = (await SeedingFactory.forSite.createMany(3, { seedableId: site.id })) + .map(({ name }) => name) + .sort(); + await SeedingFactory.forSite.create({ seedableId: site.id, hidden: true }); const nurserySeedlings = ( await TreeSpeciesFactory.forNurserySeedling.createMany(4, { speciesableId: nursery.id }) ) .map(({ name }) => name) .sort(); + await TreeSpeciesFactory.forNurserySeedling.create({ + speciesableId: nursery.id, + hidden: true + }); let result = await service.getEstablishmentTrees("project-reports", projectReport.uuid); - console.log("result", result); expect(Object.keys(result).length).toBe(1); expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted); result = await service.getEstablishmentTrees("sites", site.uuid); @@ -103,9 +117,11 @@ describe("TreeService", () => { expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted); result = await service.getEstablishmentTrees("site-reports", siteReport.uuid); - expect(Object.keys(result).length).toBe(2); + expect(Object.keys(result).length).toBe(3); expect(result["tree-planted"].sort()).toEqual(uniq([...siteTreesPlanted, ...projectTreesPlanted]).sort()); expect(result["non-tree"].sort()).toEqual(siteNonTrees); + expect(result["seeds"].sort()).toEqual(siteSeedings); + result = await service.getEstablishmentTrees("nursery-reports", nurseryReport.uuid); expect(Object.keys(result).length).toBe(2); expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted); @@ -145,7 +161,7 @@ describe("TreeService", () => { dueAt: DateTime.fromJSDate(nurseryReport1.dueAt).plus({ months: 3 }).toJSDate() }); - const reduceTreeCounts = (counts: Record, tree: TreeSpecies) => ({ + const reduceTreeCounts = (counts: Record, tree: TreeSpecies | Seeding) => ({ ...counts, [tree.name]: { taxonId: counts[tree.name]?.taxonId ?? tree.taxonId, @@ -161,9 +177,16 @@ describe("TreeService", () => { taxonId: "wfo-projectreporttree" }) ); + // hidden trees should be ignored + let hidden = await TreeSpeciesFactory.forProjectReportTreePlanted.create({ + speciesableId: projectReport1.id, + hidden: true + }); + let result = await service.getPreviousPlanting("project-reports", projectReport2.uuid); expect(Object.keys(result)).toMatchObject(["tree-planted"]); expect(result).toMatchObject({ "tree-planted": projectReportTreesPlanted.reduce(reduceTreeCounts, {}) }); + expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name); const siteReport1TreesPlanted = await TreeSpeciesFactory.forSiteReportTreePlanted.createMany(3, { speciesableId: siteReport1.id @@ -174,33 +197,51 @@ describe("TreeService", () => { taxonId: "wfo-sitereporttree" }) ); + // hidden trees should be ignored + hidden = await TreeSpeciesFactory.forSiteReportTreePlanted.create({ + speciesableId: siteReport1.id, + hidden: true + }); const siteReport2TreesPlanted = await TreeSpeciesFactory.forSiteReportTreePlanted.createMany(3, { speciesableId: siteReport2.id }); const siteReport2NonTrees = await TreeSpeciesFactory.forSiteReportNonTree.createMany(2, { speciesableId: siteReport2.id }); + const siteReport2Seedings = await SeedingFactory.forSiteReport.createMany(2, { seedableId: siteReport2.id }); + await SeedingFactory.forSiteReport.create({ seedableId: siteReport2.id, hidden: true }); + result = await service.getPreviousPlanting("site-reports", siteReport1.uuid); expect(result).toMatchObject({}); result = await service.getPreviousPlanting("site-reports", siteReport2.uuid); const siteReport1TreesPlantedReduced = siteReport1TreesPlanted.reduce(reduceTreeCounts, {}); - expect(Object.keys(result)).toMatchObject(["tree-planted"]); - expect(result).toMatchObject({ "tree-planted": siteReport1TreesPlantedReduced }); + expect(Object.keys(result).sort()).toMatchObject(["seeds", "tree-planted"]); + expect(result).toMatchObject({ "tree-planted": siteReport1TreesPlantedReduced, seeds: {} }); + expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name); result = await service.getPreviousPlanting("site-reports", siteReport3.uuid); - expect(Object.keys(result).sort()).toMatchObject(["non-tree", "tree-planted"]); + expect(Object.keys(result).sort()).toMatchObject(["non-tree", "seeds", "tree-planted"]); expect(result).toMatchObject({ "tree-planted": siteReport2TreesPlanted.reduce(reduceTreeCounts, siteReport1TreesPlantedReduced), - "non-tree": siteReport2NonTrees.reduce(reduceTreeCounts, {}) + "non-tree": siteReport2NonTrees.reduce(reduceTreeCounts, {}), + seeds: siteReport2Seedings.reduce(reduceTreeCounts, {}) }); + expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name); result = await service.getPreviousPlanting("nursery-reports", nurseryReport2.uuid); expect(result).toMatchObject({}); const nurseryReportSeedlings = await TreeSpeciesFactory.forNurseryReportSeedling.createMany(5, { speciesableId: nurseryReport1.id }); + // hidden trees should be ignored + hidden = await TreeSpeciesFactory.forNurseryReportSeedling.create({ + speciesableId: nurseryReport1.id, + hidden: true + }); + result = await service.getPreviousPlanting("nursery-reports", nurseryReport2.uuid); expect(Object.keys(result)).toMatchObject(["nursery-seedling"]); expect(result).toMatchObject({ "nursery-seedling": nurseryReportSeedlings.reduce(reduceTreeCounts, {}) }); + expect(Object.keys(result["nursery-seedling"])).not.toContain(hidden.name); }); it("handles bad input to get previous planting with undefined or an exception", async () => { diff --git a/apps/entity-service/src/trees/tree.service.ts b/apps/entity-service/src/trees/tree.service.ts index 9e2a1f37..f12d891c 100644 --- a/apps/entity-service/src/trees/tree.service.ts +++ b/apps/entity-service/src/trees/tree.service.ts @@ -8,7 +8,7 @@ import { TreeSpecies, TreeSpeciesResearch } from "@terramatch-microservices/database/entities"; -import { Op, WhereOptions } from "sequelize"; +import { Includeable, Op, WhereOptions } from "sequelize"; import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; import { Dictionary, filter, flatten, flattenDeep, groupBy, uniq } from "lodash"; import { PreviousPlantingCountDto } from "./dto/establishment-trees.dto"; @@ -25,7 +25,12 @@ type TreeModelType = TreeReportModelType | typeof Project | typeof Site | typeof const isReport = (type: EstablishmentEntity): type is EstablishmentReport => type.endsWith("-reports"); const treeAssociations = (model: TreeModelType, attributes: string[], where?: WhereOptions) => - model.TREE_ASSOCIATIONS.map(association => ({ required: false, association, attributes, where })); + model.TREE_ASSOCIATIONS.map(association => ({ + required: false, + association, + attributes, + where: { ...where, hidden: false } + })); const uniqueTreeNames = (trees: Dictionary) => Object.keys(trees).reduce( @@ -60,29 +65,38 @@ export class TreeService { // For site and nursery reports, we fetch both the establishment species on the parent entity // and on the Project const parentModel = entity === "site-reports" ? Site : Nursery; - const whereOptions = { - where: { uuid }, - attributes: [], + const include = { + model: parentModel, + // This id isn't necessary for the data we want to fetch, but sequelize requires it for + // the nested includes + attributes: ["id"], include: [ + ...treeAssociations(parentModel, ["name", "collection"]), { - model: parentModel, + model: Project, // This id isn't necessary for the data we want to fetch, but sequelize requires it for // the nested includes attributes: ["id"], - include: [ - ...treeAssociations(parentModel, ["name", "collection"]), - { - model: Project, - // This id isn't necessary for the data we want to fetch, but sequelize requires it for - // the nested includes - attributes: ["id"], - include: treeAssociations(Project, ["name", "collection"]) - } - ] + include: treeAssociations(Project, ["name", "collection"]) } ] }; + if (entity === "site-reports") { + include.include.push({ + required: false, + association: "seedsPlanted", + attributes: ["name"], + where: { hidden: false } + }); + } + + const whereOptions = { + where: { uuid }, + attributes: [], + include: [include] + }; + const report = await (entity === "site-reports" ? SiteReport.findOne(whereOptions) : NurseryReport.findOne(whereOptions)); @@ -97,7 +111,11 @@ export class TreeService { "collection" ); - return uniqueTreeNames(trees); + const treeNames = uniqueTreeNames(trees); + if (entity === "site-reports") { + treeNames["seeds"] = uniq(((parent as Site).seedsPlanted ?? []).map(({ name }) => name)); + } + return treeNames; } else if (["sites", "nurseries", "project-reports"].includes(entity)) { // for these we simply pull the project's trees const whereOptions = { @@ -160,6 +178,18 @@ export class TreeService { }); if (report == null) throw new NotFoundException(); + const modelIncludes: Includeable[] = treeAssociations(model, ["taxonId", "name", "collection", "amount"], { + amount: { [Op.gt]: 0 } + }); + if (entity === "site-reports") { + modelIncludes.push({ + required: false, + association: "seedsPlanted", + attributes: ["name", "taxonId", "amount"], + where: { hidden: false, amount: { [Op.gt]: 0 } } + }); + } + // @ts-expect-error Can't narrow the union TreeReportModelType automatically const records: InstanceType[] = await model.findAll({ attributes: [], @@ -167,7 +197,7 @@ export class TreeService { [model.PARENT_ID]: report[model.PARENT_ID], dueAt: { [Op.lt]: report.dueAt } }, - include: treeAssociations(model, ["taxonId", "name", "collection", "amount"], { amount: { [Op.gt]: 0 } }) + include: modelIncludes }); const trees = groupBy( @@ -177,7 +207,7 @@ export class TreeService { "collection" ); - return Object.keys(trees).reduce( + const planting = Object.keys(trees).reduce( (dict, collection) => ({ ...dict, [collection]: trees[collection].reduce( @@ -193,5 +223,20 @@ export class TreeService { }), {} as Dictionary> ); + + if (entity === "site-reports") { + planting["seeds"] = flatten((records as SiteReport[]).map(({ seedsPlanted }) => seedsPlanted)).reduce( + (counts, seeding) => ({ + ...counts, + [seeding.name]: { + taxonId: undefined, + amount: (counts[seeding.name]?.amount ?? 0) + (seeding.amount ?? 0) + } + }), + {} as Dictionary + ); + } + + return planting; } } diff --git a/apps/job-service/Dockerfile b/apps/job-service/Dockerfile index 234cdd30..5ef6a5d6 100644 --- a/apps/job-service/Dockerfile +++ b/apps/job-service/Dockerfile @@ -8,8 +8,12 @@ RUN npx nx build job-service ${BUILD_FLAG} FROM terramatch-microservices-base:nx-base ARG NODE_ENV +ARG SENTRY_DSN +ARG DEPLOY_ENV WORKDIR /app COPY --from=builder /app/builder ./ ENV NODE_ENV=${NODE_ENV} +ENV SENTRY_DSN=${SENTRY_DSN} +ENV DEPLOY_ENV=${DEPLOY_ENV} CMD ["node", "./dist/apps/job-service/main.js"] diff --git a/apps/research-service/Dockerfile b/apps/research-service/Dockerfile index 721783bf..39f272b8 100644 --- a/apps/research-service/Dockerfile +++ b/apps/research-service/Dockerfile @@ -8,8 +8,12 @@ RUN npx nx build research-service ${BUILD_FLAG} FROM terramatch-microservices-base:nx-base ARG NODE_ENV +ARG SENTRY_DSN +ARG DEPLOY_ENV WORKDIR /app COPY --from=builder /app/builder ./ ENV NODE_ENV=${NODE_ENV} +ENV SENTRY_DSN=${SENTRY_DSN} +ENV DEPLOY_ENV=${DEPLOY_ENV} CMD ["node", "./dist/apps/research-service/main.js"] diff --git a/apps/unified-database-service/Dockerfile b/apps/unified-database-service/Dockerfile index e070d2c9..cf05693b 100644 --- a/apps/unified-database-service/Dockerfile +++ b/apps/unified-database-service/Dockerfile @@ -8,8 +8,12 @@ RUN npx nx build unified-database-service ${BUILD_FLAG} FROM terramatch-microservices-base:nx-base ARG NODE_ENV +ARG SENTRY_DSN +ARG DEPLOY_ENV WORKDIR /app COPY --from=builder /app/builder ./ ENV NODE_ENV=${NODE_ENV} +ENV SENTRY_DSN=${SENTRY_DSN} +ENV DEPLOY_ENV=${DEPLOY_ENV} CMD ["node", "./dist/apps/unified-database-service/main.js"] diff --git a/apps/unified-database-service/src/airtable/airtable.module.ts b/apps/unified-database-service/src/airtable/airtable.module.ts index 03c910ec..5d213170 100644 --- a/apps/unified-database-service/src/airtable/airtable.module.ts +++ b/apps/unified-database-service/src/airtable/airtable.module.ts @@ -5,6 +5,8 @@ import { BullModule } from "@nestjs/bullmq"; import { Module } from "@nestjs/common"; import { AirtableService } from "./airtable.service"; import { AirtableProcessor } from "./airtable.processor"; +import { QueueHealthService } from "./queue-health.service"; +import { SlackModule } from "nestjs-slack"; @Module({ imports: [ @@ -18,13 +20,23 @@ import { AirtableProcessor } from "./airtable.processor"; connection: { host: configService.get("REDIS_HOST"), port: configService.get("REDIS_PORT"), - prefix: "unified-database-service" + prefix: "unified-database-service", + // Use TLS in AWS + ...(process.env.NODE_ENV !== "development" ? { tls: {} } : null) } }) }), - BullModule.registerQueue({ name: "airtable" }) + BullModule.registerQueue({ name: "airtable" }), + SlackModule.forRootAsync({ + imports: [ConfigModule], + inject: [ConfigService], + useFactory: (configService: ConfigService) => ({ + type: "api", + token: configService.get("SLACK_API_KEY") + }) + }) ], - providers: [AirtableService, AirtableProcessor], - exports: [AirtableService] + providers: [AirtableService, AirtableProcessor, QueueHealthService], + exports: [AirtableService, QueueHealthService] }) export class AirtableModule {} diff --git a/apps/unified-database-service/src/airtable/airtable.processor.spec.ts b/apps/unified-database-service/src/airtable/airtable.processor.spec.ts new file mode 100644 index 00000000..2d3e7598 --- /dev/null +++ b/apps/unified-database-service/src/airtable/airtable.processor.spec.ts @@ -0,0 +1,97 @@ +import { AIRTABLE_ENTITIES, AirtableProcessor, ENTITY_TYPES } from "./airtable.processor"; +import { Test } from "@nestjs/testing"; +import { ConfigService } from "@nestjs/config"; +import { createMock } from "@golevelup/ts-jest"; +import { InternalServerErrorException, NotImplementedException } from "@nestjs/common"; +import { Job } from "bullmq"; +import { SlackService } from "nestjs-slack"; + +jest.mock("airtable", () => + jest.fn(() => ({ + base: () => jest.fn() + })) +); + +describe("AirtableProcessor", () => { + let processor: AirtableProcessor; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + AirtableProcessor, + { provide: ConfigService, useValue: createMock() }, + { provide: SlackService, useValue: createMock() } + ] + }).compile(); + + processor = await module.resolve(AirtableProcessor); + }); + + it("throws an error with an unknown job name", async () => { + await expect(processor.process({ name: "unknown" } as Job)).rejects.toThrow(NotImplementedException); + }); + + describe("updateEntities", () => { + it("throws an error with an unknown entity type", async () => { + await expect(processor.process({ name: "updateEntities", data: { entityType: "foo" } } as Job)).rejects.toThrow( + InternalServerErrorException + ); + }); + + it("calls updateBase on the entity", async () => { + const updateBase = jest.fn(() => Promise.resolve()); + // @ts-expect-error faking the SiteEntity + AIRTABLE_ENTITIES.site = class { + updateBase = updateBase; + }; + const updatedSince = new Date(); + await processor.process({ + name: "updateEntities", + data: { entityType: "site", startPage: 2, updatedSince } + } as Job); + expect(updateBase).toHaveBeenCalledWith(expect.anything(), { startPage: 2, updatedSince }); + }); + }); + + describe("deleteEntities", () => { + it("throws an error with an unknown entity type", async () => { + await expect(processor.process({ name: "deleteEntities", data: { entityType: "foo" } } as Job)).rejects.toThrow( + InternalServerErrorException + ); + }); + + it("calls deleteStaleRecords on the entity", async () => { + const deleteStaleRecords = jest.fn(() => Promise.resolve()); + // @ts-expect-error faking the SiteEntity + AIRTABLE_ENTITIES.site = class { + deleteStaleRecords = deleteStaleRecords; + }; + const deletedSince = new Date(); + await processor.process({ name: "deleteEntities", data: { entityType: "site", deletedSince } } as Job); + expect(deleteStaleRecords).toHaveBeenCalledWith(expect.anything(), deletedSince); + }); + }); + + describe("updateAll", () => { + it("calls updateEntities and deleteEntities with all types", async () => { + const updateSpy = ( + jest.spyOn(processor as never, "updateEntities") as jest.SpyInstance> + ).mockImplementation(() => Promise.resolve()); + const deleteSpy = ( + jest.spyOn(processor as never, "deleteEntities") as jest.SpyInstance> + ).mockImplementation(() => Promise.resolve()); + + const updatedSince = new Date(); + await processor.process({ name: "updateAll", data: { updatedSince } } as Job); + + for (let ii = 0; ii < ENTITY_TYPES.length; ii++) { + const entityType = ENTITY_TYPES[ii]; + expect(updateSpy).toHaveBeenNthCalledWith(ii + 1, { entityType, updatedSince }); + expect(deleteSpy).toHaveBeenNthCalledWith(ii + 1, { entityType, deletedSince: updatedSince }); + } + + updateSpy.mockRestore(); + deleteSpy.mockRestore(); + }); + }); +}); diff --git a/apps/unified-database-service/src/airtable/airtable.processor.ts b/apps/unified-database-service/src/airtable/airtable.processor.ts index 6535c02b..75994844 100644 --- a/apps/unified-database-service/src/airtable/airtable.processor.ts +++ b/apps/unified-database-service/src/airtable/airtable.processor.ts @@ -1,71 +1,150 @@ -import { Processor, WorkerHost } from "@nestjs/bullmq"; -import { InternalServerErrorException, LoggerService, NotImplementedException, Scope } from "@nestjs/common"; +import { OnWorkerEvent, Processor, WorkerHost } from "@nestjs/bullmq"; +import { InternalServerErrorException, LoggerService, NotImplementedException } from "@nestjs/common"; import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; import { Job } from "bullmq"; -import { UpdateEntitiesData } from "./airtable.service"; import { ConfigService } from "@nestjs/config"; import Airtable from "airtable"; -import { ProjectEntity } from "./entities"; +import { + ApplicationEntity, + DemographicEntity, + NurseryEntity, + NurseryReportEntity, + OrganisationEntity, + ProjectEntity, + ProjectReportEntity, + RestorationPartnerEntity, + SiteEntity, + SiteReportEntity, + TreeSpeciesEntity, + WorkdayEntity +} from "./entities"; +import * as Sentry from "@sentry/node"; +import { SlackService } from "nestjs-slack"; -const AIRTABLE_ENTITIES = { - project: ProjectEntity +export const AIRTABLE_ENTITIES = { + application: ApplicationEntity, + demographic: DemographicEntity, + nursery: NurseryEntity, + "nursery-report": NurseryReportEntity, + organisation: OrganisationEntity, + project: ProjectEntity, + "project-report": ProjectReportEntity, + "restoration-partner": RestorationPartnerEntity, + site: SiteEntity, + "site-report": SiteReportEntity, + "tree-species": TreeSpeciesEntity, + workday: WorkdayEntity +}; + +export type EntityType = keyof typeof AIRTABLE_ENTITIES; +export const ENTITY_TYPES = Object.keys(AIRTABLE_ENTITIES) as EntityType[]; + +export type UpdateEntitiesData = { + entityType: EntityType; + startPage?: number; + updatedSince?: Date; +}; + +export type DeleteEntitiesData = { + entityType: EntityType; + deletedSince: Date; +}; + +export type UpdateAllData = { + updatedSince: Date; }; /** * Processes jobs in the airtable queue. Note that if we see problems with this crashing or * consuming too many resources, we have the option to run this in a forked process, although * it will involve some additional setup: https://docs.nestjs.com/techniques/queues#separate-processes - * - * Scope.REQUEST causes this processor to get created fresh for each event in the Queue, which means - * that it will be fully garbage collected after its work is done. */ -@Processor({ name: "airtable", scope: Scope.REQUEST }) +@Processor("airtable") export class AirtableProcessor extends WorkerHost { private readonly logger: LoggerService = new TMLogService(AirtableProcessor.name); private readonly base: Airtable.Base; - constructor(configService: ConfigService) { + constructor(private readonly config: ConfigService, private readonly slack: SlackService) { super(); - this.base = new Airtable({ apiKey: configService.get("AIRTABLE_API_KEY") }).base( - configService.get("AIRTABLE_BASE_ID") - ); + this.base = new Airtable({ apiKey: this.config.get("AIRTABLE_API_KEY") }).base(this.config.get("AIRTABLE_BASE_ID")); } async process(job: Job) { - switch (job.name) { + const { name, data } = job; + await this.sendSlackUpdate(`:construction_worker: Beginning job: ${JSON.stringify({ name, data })}`); + switch (name) { case "updateEntities": - return await this.updateEntities(job.data as UpdateEntitiesData); + return await this.updateEntities(data as UpdateEntitiesData); + + case "deleteEntities": + return await this.deleteEntities(data as DeleteEntitiesData); + + case "updateAll": + return await this.updateAll(data as UpdateAllData); default: - throw new NotImplementedException(`Unknown job type: ${job.name}`); + throw new NotImplementedException(`Unknown job type: ${name}`); } } - private async updateEntities({ entityType }: UpdateEntitiesData) { - this.logger.log(`Beginning entity update: ${JSON.stringify({ entityType })}`); + @OnWorkerEvent("failed") + async onFailed(job: Job, error: Error) { + Sentry.captureException(error); + this.logger.error(`Worker event failed: ${JSON.stringify(job)}`, error.stack); + await this.sendSlackUpdate(`:warning: ERROR: Job processing failed: ${JSON.stringify(job)}`); + } - const airtableEntity = AIRTABLE_ENTITIES[entityType]; - if (airtableEntity == null) { + private async updateEntities({ entityType, startPage, updatedSince }: UpdateEntitiesData) { + this.logger.log(`Beginning entity update: ${JSON.stringify({ entityType, updatedSince })}`); + + const entityClass = AIRTABLE_ENTITIES[entityType]; + if (entityClass == null) { throw new InternalServerErrorException(`Entity mapping not found for entity type ${entityType}`); } - // bogus offset and page size. The big early PPC records are blowing up the JS memory heap. Need - // to make some performance improvements to how the project airtable entity fetches records, pulling - // in fewer included associations. That will be a refactor for the next ticket. - const records = await airtableEntity.findMany(3, 200); - const airtableRecords = await Promise.all( - records.map(async record => ({ fields: await airtableEntity.mapDbEntity(record) })) + const entity = new entityClass(); + await entity.updateBase(this.base, { startPage, updatedSince }); + + this.logger.log(`Completed entity update: ${JSON.stringify({ entityType, updatedSince })}`); + await this.sendSlackUpdate(`Completed updating table "${entity.TABLE_NAME}" [updatedSince: ${updatedSince}]`); + } + + private async deleteEntities({ entityType, deletedSince }: DeleteEntitiesData) { + this.logger.log(`Beginning entity delete: ${JSON.stringify({ entityType, deletedSince })}`); + + const entityClass = AIRTABLE_ENTITIES[entityType]; + if (entityClass == null) { + throw new InternalServerErrorException(`Entity mapping not found for entity type ${entityType}`); + } + + const entity = new entityClass(); + await entity.deleteStaleRecords(this.base, deletedSince); + + this.logger.log(`Completed entity delete: ${JSON.stringify({ entityType, deletedSince })}`); + await this.sendSlackUpdate( + `Completed deleting rows from table "${entity.TABLE_NAME}" [deletedSince: ${deletedSince}]` ); - try { - // @ts-expect-error The types for this lib haven't caught up with its support for upserts - // https://github.com/Airtable/airtable.js/issues/348 - await this.base(airtableEntity.TABLE_NAME).update(airtableRecords, { - performUpsert: { fieldsToMergeOn: ["uuid"] } - }); - } catch (error) { - this.logger.error(`Entity update failed: ${JSON.stringify({ entityType, error, airtableRecords }, null, 2)}`); - throw error; + } + + private async updateAll({ updatedSince }: UpdateAllData) { + await this.sendSlackUpdate(`:white_check_mark: Beginning sync of all data [changedSince: ${updatedSince}]`); + for (const entityType of ENTITY_TYPES) { + await this.updateEntities({ entityType, updatedSince }); + await this.deleteEntities({ entityType, deletedSince: updatedSince }); } - this.logger.log(`Entity update complete: ${JSON.stringify({ entityType })}`); + await this.sendSlackUpdate(`:100: Completed sync of all data [changedSince: ${updatedSince}]`); + } + + private async sendSlackUpdate(message: string) { + const channel = this.config.get("UDB_SLACK_CHANNEL"); + if (channel == null) return; + + await this.slack + .sendText(`[${process.env.DEPLOY_ENV}]: ${message}`, { channel }) + // Don't allow a failure in slack sending to hose our process, but do log it and send it to Sentry + .catch(error => { + Sentry.captureException(error); + this.logger.error("Send to slack failed", error.stack); + }); } } diff --git a/apps/unified-database-service/src/airtable/airtable.service.spec.ts b/apps/unified-database-service/src/airtable/airtable.service.spec.ts new file mode 100644 index 00000000..3aecdd2d --- /dev/null +++ b/apps/unified-database-service/src/airtable/airtable.service.spec.ts @@ -0,0 +1,55 @@ +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { AirtableService } from "./airtable.service"; +import { Queue } from "bullmq"; +import { Test } from "@nestjs/testing"; +import { getQueueToken } from "@nestjs/bullmq"; + +describe("AirtableService", () => { + let service: AirtableService; + let queue: DeepMocked; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [ + AirtableService, + { + provide: getQueueToken("airtable"), + useValue: (queue = createMock()) + } + ] + }).compile(); + + service = module.get(AirtableService); + }); + + describe("updateAirtableJob", () => { + it("adds the job to the queue", async () => { + const updatedSince = new Date(); + await service.updateAirtable("nursery", 10, updatedSince); + expect(queue.add).toHaveBeenCalledWith("updateEntities", { entityType: "nursery", startPage: 10, updatedSince }); + }); + }); + + describe("deleteAirtableJob", () => { + it("adds the job to the queue", async () => { + const deletedSince = new Date(); + await service.deleteFromAirtable("project", deletedSince); + expect(queue.add).toHaveBeenCalledWith("deleteEntities", { entityType: "project", deletedSince }); + }); + }); + + describe("updateAll", () => { + it("adds the job to the queue", async () => { + const updatedSince = new Date(); + await service.updateAll(updatedSince); + expect(queue.add).toHaveBeenCalledWith("updateAll", { updatedSince }); + }); + }); + + describe("handleDailyUpdate", () => { + it("adds the job to the queue", async () => { + await service.handleDailyUpdate(); + expect(queue.add).toHaveBeenCalledWith("updateAll", { updatedSince: expect.any(Date) }); + }); + }); +}); diff --git a/apps/unified-database-service/src/airtable/airtable.service.ts b/apps/unified-database-service/src/airtable/airtable.service.ts index db3596bd..a87d2a5d 100644 --- a/apps/unified-database-service/src/airtable/airtable.service.ts +++ b/apps/unified-database-service/src/airtable/airtable.service.ts @@ -2,13 +2,9 @@ import { Injectable, LoggerService } from "@nestjs/common"; import { InjectQueue } from "@nestjs/bullmq"; import { Queue } from "bullmq"; import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; - -export const ENTITY_TYPES = ["project"] as const; -export type EntityType = (typeof ENTITY_TYPES)[number]; - -export type UpdateEntitiesData = { - entityType: EntityType; -}; +import { DeleteEntitiesData, EntityType, UpdateAllData, UpdateEntitiesData } from "./airtable.processor"; +import { Cron, CronExpression } from "@nestjs/schedule"; +import { DateTime } from "luxon"; @Injectable() export class AirtableService { @@ -16,11 +12,30 @@ export class AirtableService { constructor(@InjectQueue("airtable") private readonly airtableQueue: Queue) {} - // TODO (NJC) This method will probably go away entirely, or at least change drastically after this POC - async updateAirtableJob(entityType: EntityType) { - const data: UpdateEntitiesData = { entityType }; + async updateAirtable(entityType: EntityType, startPage?: number, updatedSince?: Date) { + const data: UpdateEntitiesData = { entityType, startPage, updatedSince }; this.logger.log(`Adding entity update to queue: ${JSON.stringify(data)}`); await this.airtableQueue.add("updateEntities", data); } + + async deleteFromAirtable(entityType: EntityType, deletedSince: Date) { + const data: DeleteEntitiesData = { entityType, deletedSince }; + + this.logger.log(`Adding entity delete to queue: ${JSON.stringify(data)}`); + await this.airtableQueue.add("deleteEntities", data); + } + + async updateAll(updatedSince: Date) { + const data: UpdateAllData = { updatedSince }; + + this.logger.log(`Adding update all to queue: ${JSON.stringify(data)}`); + await this.airtableQueue.add("updateAll", data); + } + + @Cron(CronExpression.EVERY_DAY_AT_8PM) + async handleDailyUpdate() { + this.logger.log("Triggering daily update"); + await this.updateAll(DateTime.now().minus({ days: 2 }).toJSDate()); + } } diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts new file mode 100644 index 00000000..a16b0e6d --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts @@ -0,0 +1,866 @@ +import { airtableColumnName, AirtableEntity, ColumnMapping } from "./airtable-entity"; +import { faker } from "@faker-js/faker"; +import { + Application, + Demographic, + Nursery, + NurseryReport, + Organisation, + Project, + ProjectReport, + RestorationPartner, + Site, + SiteReport, + TreeSpecies, + Workday +} from "@terramatch-microservices/database/entities"; +import { + ApplicationFactory, + DemographicFactory, + FormSubmissionFactory, + FundingProgrammeFactory, + NurseryFactory, + NurseryReportFactory, + OrganisationFactory, + ProjectFactory, + ProjectReportFactory, + RestorationPartnerFactory, + SeedingFactory, + SiteFactory, + SitePolygonFactory, + SiteReportFactory, + TreeSpeciesFactory, + WorkdayFactory +} from "@terramatch-microservices/database/factories"; +import Airtable from "airtable"; +import { + ApplicationEntity, + DemographicEntity, + NurseryEntity, + NurseryReportEntity, + OrganisationEntity, + ProjectEntity, + ProjectReportEntity, + RestorationPartnerEntity, + SiteEntity, + SiteReportEntity, + TreeSpeciesEntity, + WorkdayEntity +} from "./"; +import { orderBy, sortBy } from "lodash"; +import { Model } from "sequelize-typescript"; +import { FRAMEWORK_NAMES, FrameworkKey } from "@terramatch-microservices/database/constants/framework"; +import { FindOptions, Op } from "sequelize"; +import { DateTime } from "luxon"; + +const airtableUpdate = jest.fn, [{ fields: object }[], object]>(() => Promise.resolve()); +const airtableSelectFirstPage = jest.fn, never>(() => Promise.resolve([])); +const airtableSelect = jest.fn(() => ({ firstPage: airtableSelectFirstPage })); +const airtableDestroy = jest.fn(() => Promise.resolve()); +const Base = jest.fn(() => ({ + update: airtableUpdate, + select: airtableSelect, + destroy: airtableDestroy +})) as unknown as Airtable.Base; + +const mapEntityColumns = jest.fn(() => Promise.resolve({})); +export class StubEntity extends AirtableEntity { + readonly TABLE_NAME = "stubs"; + readonly COLUMNS = ["id"] as ColumnMapping[]; + readonly MODEL = Site; + + protected getUpdatePageFindOptions = (page: number, updatedSince?: Date) => ({ + ...super.getUpdatePageFindOptions(page, updatedSince), + limit: 1 + }); + + protected mapEntityColumns = mapEntityColumns; +} + +async function testAirtableUpdates, A>( + entity: AirtableEntity, + records: M[], + spotCheckFields: (record: M) => { fields: object } +) { + await entity.updateBase(Base); + + const batches = []; + for (let ii = 0; ii < Math.ceil(records.length / 10); ii++) { + batches.push(sortBy(records.slice(ii * 10, (ii + 1) * 10), ["uuid"])); + } + + expect(airtableUpdate).toHaveBeenCalledTimes(batches.length); + + const columnsExpected = entity.COLUMNS.map(airtableColumnName).sort(); + for (let ii = 0; ii < batches.length; ii++) { + const batch = batches[ii]; + const updates = sortBy(airtableUpdate.mock.calls[ii][0], ["fields.uuid"]); + expect(updates).toMatchObject(batch.map(spotCheckFields)); + + for (const update of updates) { + expect(Object.keys(update.fields).sort()).toEqual(columnsExpected); + } + } +} + +describe("AirtableEntity", () => { + afterEach(async () => { + airtableUpdate.mockClear(); + airtableSelect.mockClear(); + airtableSelectFirstPage.mockClear(); + airtableDestroy.mockClear(); + }); + + describe("BaseClass", () => { + describe("updateBase", () => { + beforeAll(async () => { + // Ensure there's at least one site so the mapping happens + await SiteFactory.create(); + }); + + it("re-raises mapping errors", async () => { + mapEntityColumns.mockRejectedValue(new Error("mapping error")); + await expect(new StubEntity().updateBase(null, { startPage: 0 })).rejects.toThrow("mapping error"); + mapEntityColumns.mockReset(); + }); + + it("re-raises airtable errors", async () => { + airtableUpdate.mockRejectedValue(new Error("airtable error")); + await expect(new StubEntity().updateBase(Base)).rejects.toThrow("airtable error"); + airtableUpdate.mockReset(); + }); + + it("includes the updatedSince timestamp in the query", async () => { + const entity = new StubEntity(); + const spy = jest.spyOn(entity as never, "getUpdatePageFindOptions") as jest.SpyInstance>; + const updatedSince = new Date(); + await entity.updateBase(Base, { updatedSince }); + expect(spy.mock.results[0].value.where).toMatchObject({ + updatedAt: { [Op.gte]: updatedSince } + }); + spy.mockReset(); + }); + + it("skips the updatedSince timestamp if the model doesn't support it", async () => { + const entity = new StubEntity(); + // @ts-expect-error overriding readonly property for test. + (entity as never).SUPPORTS_UPDATED_SINCE = false; + const spy = jest.spyOn(entity as never, "getUpdatePageFindOptions") as jest.SpyInstance>; + const updatedSince = new Date(); + await entity.updateBase(Base, { updatedSince }); + expect(spy.mock.results[0].value.where).not.toMatchObject({ + updatedAt: expect.anything() + }); + }); + }); + + describe("deleteStaleRecords", () => { + let deletedSites: Site[]; + const deletedSince = DateTime.fromJSDate(new Date()).minus({ days: 2 }).toJSDate(); + + beforeAll(async () => { + // Truncate on a paranoid model soft deletes instead of removing the row. There is a force: true + // option, but it doesn't work on tables that have foreign key constraints from other tables + // so we're working around it here by making sure all rows in the table were "deleted" before + // our deletedSince timestamp before creating new rows + await Site.truncate(); + await Site.update( + { deletedAt: DateTime.fromJSDate(deletedSince).minus({ days: 1 }).toJSDate() }, + { where: {}, paranoid: false } + ); + + const sites = await SiteFactory.createMany(25); + for (const site of faker.helpers.uniqueArray(sites, 9)) { + await site.destroy(); + } + + deletedSites = sites.filter(site => site.isSoftDeleted()); + }); + + it("re-raises search errors", async () => { + airtableSelectFirstPage.mockRejectedValue(new Error("select error")); + await expect(new SiteEntity().deleteStaleRecords(Base, deletedSince)).rejects.toThrow("select error"); + }); + + it("re-raises delete errors", async () => { + airtableSelectFirstPage.mockResolvedValue([{ id: "fakeid", fields: { uuid: "fakeuuid" } }]); + airtableDestroy.mockRejectedValue(new Error("delete error")); + await expect(new SiteEntity().deleteStaleRecords(Base, deletedSince)).rejects.toThrow("delete error"); + airtableDestroy.mockReset(); + }); + + it("calls delete with all records found", async () => { + const searchResult = deletedSites.map(({ uuid }) => ({ + id: String(faker.number.int({ min: 1000, max: 9999 })), + fields: { uuid } + })); + airtableSelectFirstPage.mockResolvedValue(searchResult); + await new SiteEntity().deleteStaleRecords(Base, deletedSince); + expect(airtableSelect).toHaveBeenCalledTimes(1); + expect(airtableSelect).toHaveBeenCalledWith( + expect.objectContaining({ + fields: ["uuid"], + filterByFormula: expect.stringMatching( + `(${deletedSites.map(({ uuid }) => `{uuid}='${uuid}'`).join("|")}{${deletedSites.length + 1})` + ) + }) + ); + expect(airtableDestroy).toHaveBeenCalledTimes(1); + expect(airtableDestroy).toHaveBeenCalledWith(expect.arrayContaining(searchResult.map(({ id }) => id))); + }); + }); + }); + + describe("ApplicationEntity", () => { + let fundingProgrammeNames: Record; + let applications: Application[]; + let submissionStatuses: Record; + + beforeAll(async () => { + await Application.truncate(); + + const org = await OrganisationFactory.create({}); + const fundingProgrammes = await FundingProgrammeFactory.createMany(3); + fundingProgrammeNames = fundingProgrammes.reduce((names, { uuid, name }) => ({ ...names, [uuid]: name }), {}); + const allApplications = []; + for (let ii = 0; ii < 15; ii++) { + allApplications.push( + await ApplicationFactory.create({ + organisationUuid: org.uuid, + // test one not having an attached funding programme + fundingProgrammeUuid: ii === 4 ? null : faker.helpers.arrayElement(Object.keys(fundingProgrammeNames)) + }) + ); + } + + await allApplications[3].destroy(); + await allApplications[11].destroy(); + applications = allApplications.filter(application => !application.isSoftDeleted()); + + let first = true; + submissionStatuses = {}; + for (const { id, uuid } of applications) { + // skip for the first one so we test an export that's missing a submission + if (first) { + first = false; + continue; + } + + const submissions = await FormSubmissionFactory.createMany(faker.number.int({ min: 1, max: 5 }), { + applicationId: id + }); + submissionStatuses[uuid] = orderBy(submissions, ["id"], ["desc"])[0].status; + } + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new ApplicationEntity(), + applications, + ({ uuid, organisationUuid, fundingProgrammeUuid }) => ({ + fields: { + uuid, + organisationUuid, + fundingProgrammeName: fundingProgrammeNames[fundingProgrammeUuid], + status: submissionStatuses[uuid] + } + }) + ); + }); + }); + + describe("DemographicEntity", () => { + let associationUuids: Record; + let demographics: Demographic[]; + + beforeAll(async () => { + await Demographic.truncate(); + + associationUuids = {}; + const workday = await WorkdayFactory.forProjectReport.create(); + associationUuids[Workday.LARAVEL_TYPE] = workday.uuid; + const partner = await RestorationPartnerFactory.forProjectReport.create(); + associationUuids[RestorationPartner.LARAVEL_TYPE] = partner.uuid; + + const factories = [ + () => DemographicFactory.forWorkday.create({ demographicalId: workday.id }), + () => DemographicFactory.forRestorationPartner.create({ demographicalId: partner.id }) + ]; + + const allDemographics = []; + for (const factory of factories) { + allDemographics.push(await factory()); + } + for (let ii = 0; ii < 35; ii++) { + allDemographics.push(await faker.helpers.arrayElement(factories)()); + } + + const toDelete = faker.helpers.uniqueArray(() => faker.number.int(allDemographics.length - 1), 10); + for (const ii of toDelete) { + await allDemographics[ii].destroy(); + } + + demographics = allDemographics.filter(demographic => !demographic.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new DemographicEntity(), + demographics, + ({ id, type, subtype, name, amount, demographicalType }) => ({ + fields: { + id, + type, + subtype, + name, + amount, + workdayUuid: demographicalType === Workday.LARAVEL_TYPE ? associationUuids[demographicalType] : undefined, + restorationPartnerUuid: + demographicalType === RestorationPartner.LARAVEL_TYPE ? associationUuids[demographicalType] : undefined + } + }) + ); + }); + }); + + describe("NurseryEntity", () => { + let projectUuids: Record; + let nurseries: Nursery[]; + + beforeAll(async () => { + await Nursery.truncate(); + + const projects = await ProjectFactory.createMany(2); + projectUuids = projects.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const projectIds = projects.map(({ id }) => id); + const allNurseries = []; + for (let ii = 0; ii < 15; ii++) { + allNurseries.push(await NurseryFactory.create({ projectId: faker.helpers.arrayElement(projectIds) })); + } + allNurseries.push(await NurseryFactory.create({ projectId: null })); + + await allNurseries[2].destroy(); + nurseries = allNurseries.filter(nursery => !nursery.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new NurseryEntity(), nurseries, ({ uuid, name, projectId, status }) => ({ + fields: { + uuid, + name, + projectUuid: projectUuids[projectId], + status + } + })); + }); + }); + + describe("NurseryReportEntity", () => { + let nurseryUuids: Record; + let reports: NurseryReport[]; + + beforeAll(async () => { + await NurseryReport.truncate(); + + const nurseries = await NurseryFactory.createMany(2); + nurseryUuids = nurseries.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const nurseryIds = nurseries.reduce((ids, { id }) => [...ids, id], [] as number[]); + const allReports = []; + for (let ii = 0; ii < 15; ii++) { + allReports.push(await NurseryReportFactory.create({ nurseryId: faker.helpers.arrayElement(nurseryIds) })); + } + allReports.push(await NurseryReportFactory.create({ nurseryId: null })); + + await allReports[6].destroy(); + reports = allReports.filter(report => !report.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new NurseryReportEntity(), reports, ({ uuid, nurseryId, status, dueAt }) => ({ + fields: { + uuid, + nurseryUuid: nurseryUuids[nurseryId], + status, + dueAt + } + })); + }); + }); + + describe("OrganisationEntity", () => { + let organisations: Organisation[]; + + beforeAll(async () => { + await Organisation.truncate(); + + const allOrgs = await OrganisationFactory.createMany(16); + await allOrgs[5].destroy(); + await allOrgs[12].destroy(); + + organisations = allOrgs.filter(org => !org.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new OrganisationEntity(), organisations, ({ uuid, name, status }) => ({ + fields: { + uuid, + name, + status + } + })); + }); + }); + + describe("ProjectEntity", () => { + let organisationUuids: Record; + let applicationUuids: Record; + let projects: Project[]; + let calculatedValues: Record>; + + beforeAll(async () => { + await Project.truncate(); + + const orgs = await OrganisationFactory.createMany(3); + organisationUuids = orgs.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const orgIds = orgs.reduce((ids, { id }) => [...ids, id], [] as number[]); + const orgId = () => faker.helpers.arrayElement(orgIds); + + const allProjects = [] as Project[]; + for (let ii = 0; ii < 15; ii++) { + allProjects.push(await ProjectFactory.create({ organisationId: orgId() })); + } + + for (const ii of faker.helpers.uniqueArray(() => allProjects.length - 1, 2)) { + await allProjects[ii].destroy(); + } + + // include some slightly broken fields for testing. + allProjects.push( + await ProjectFactory.create({ + organisationId: null, + continent: null, + applicationId: null, + frameworkKey: "foo" as FrameworkKey + }) + ); + + projects = allProjects.filter(project => !project.isSoftDeleted()); + applicationUuids = ( + await Application.findAll({ + where: { id: projects.map(({ applicationId }) => applicationId) }, + attributes: ["id", "uuid"] + }) + ).reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + + await NurseryFactory.create({ projectId: projects[0].id, status: "approved" }); + + const { uuid: startedSiteUuid } = await SiteFactory.create({ projectId: projects[0].id, status: "started" }); + const { uuid: site1Uuid } = await SiteFactory.create({ + projectId: projects[0].id, + status: "approved" + }); + const { uuid: site2Uuid } = await SiteFactory.create({ + projectId: projects[0].id, + status: "approved" + }); + await SiteFactory.create({ projectId: projects[0].id, status: "approved" }); + + // won't count because siteReport1 is not approved + await SitePolygonFactory.create({ siteUuid: startedSiteUuid }); + let hectaresRestoredToDate = (await SitePolygonFactory.create({ siteUuid: site1Uuid })).calcArea; + // won't count because it's not active + await SitePolygonFactory.create({ siteUuid: site2Uuid, isActive: false }); + hectaresRestoredToDate += (await SitePolygonFactory.create({ siteUuid: site2Uuid })).calcArea; + + calculatedValues = { + [projects[0].uuid]: { + hectaresRestoredToDate: Math.round(hectaresRestoredToDate) + } + }; + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new ProjectEntity(), + projects, + ({ uuid, name, frameworkKey, organisationId, applicationId }) => ({ + fields: { + uuid, + name, + cohort: FRAMEWORK_NAMES[frameworkKey] ?? frameworkKey, + organisationUuid: organisationUuids[organisationId], + applicationUuid: applicationUuids[applicationId], + hectaresRestoredToDate: calculatedValues[uuid]?.hectaresRestoredToDate ?? 0 + } + }) + ); + }); + }); + + describe("ProjectReportEntity", () => { + let projectUuids: Record; + let reports: ProjectReport[]; + let calculatedValues: Record>; + + beforeAll(async () => { + await ProjectReport.truncate(); + + const projects = await ProjectFactory.createMany(2); + projectUuids = projects.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const projectIds = projects.reduce((ids, { id }) => [...ids, id], [] as number[]); + const allReports = []; + for (let ii = 0; ii < 15; ii++) { + allReports.push(await ProjectReportFactory.create({ projectId: faker.helpers.arrayElement(projectIds) })); + } + allReports.push(await ProjectReportFactory.create({ projectId: null })); + + const ppcReport = await ProjectReportFactory.create({ + projectId: faker.helpers.arrayElement(projectIds), + frameworkKey: "ppc" + }); + allReports.push(ppcReport); + const ppcSeedlings = ( + await TreeSpeciesFactory.forProjectReportTreePlanted.createMany(3, { speciesableId: ppcReport.id }) + ).reduce((total, { amount }) => total + amount, 0); + // make sure hidden is ignored + await TreeSpeciesFactory.forProjectReportTreePlanted.create({ speciesableId: ppcReport.id, hidden: true }); + + // TODO this might start causing problems when Task is implemented in this codebase and we have a factory + // that's generating real records + const terrafundReport = await ProjectReportFactory.create({ + projectId: faker.helpers.arrayElement(projectIds), + frameworkKey: "terrafund", + taskId: 123 + }); + allReports.push(terrafundReport); + const terrafundSeedlings = ( + await NurseryReportFactory.createMany(2, { + taskId: terrafundReport.taskId, + seedlingsYoungTrees: faker.number.int({ min: 10, max: 100 }), + status: "approved" + }) + ).reduce((total, { seedlingsYoungTrees }) => total + seedlingsYoungTrees, 0); + // make sure non-approved reports are ignored + await NurseryReportFactory.create({ + taskId: terrafundReport.taskId, + seedlingsYoungTrees: faker.number.int({ min: 10, max: 100 }), + status: "due" + }); + + await allReports[6].destroy(); + reports = allReports.filter(report => !report.isSoftDeleted()); + + calculatedValues = { + [ppcReport.uuid]: { + totalSeedlingsGrown: ppcSeedlings + }, + [terrafundReport.uuid]: { + totalSeedlingsGrown: terrafundSeedlings + } + }; + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new ProjectReportEntity(), reports, ({ uuid, projectId, status, dueAt }) => ({ + fields: { + uuid, + projectUuid: projectUuids[projectId], + status, + dueAt, + totalSeedlingsGrown: calculatedValues[uuid]?.totalSeedlingsGrown ?? 0 + } + })); + }); + }); + + describe("RestorationPartnerEntity", () => { + let associationUuids: Record; + let partners: RestorationPartner[]; + + beforeAll(async () => { + await RestorationPartner.truncate(); + + associationUuids = {}; + const projectReport = await ProjectReportFactory.create(); + associationUuids[ProjectReport.LARAVEL_TYPE] = projectReport.uuid; + + const factories = [() => RestorationPartnerFactory.forProjectReport.create({ partnerableId: projectReport.id })]; + + const allPartners: RestorationPartner[] = []; + for (const factory of factories) { + // make sure we have at least one of each type + allPartners.push(await factory()); + } + for (let ii = 0; ii < 35; ii++) { + // create a whole bunch mor at random + allPartners.push(await faker.helpers.arrayElement(factories)()); + } + const toDeleteOrHide = faker.helpers.uniqueArray(() => faker.number.int(allPartners.length - 1), 10); + let hide = true; + for (const ii of toDeleteOrHide) { + if (hide) { + await allPartners[ii].update({ hidden: true }); + } else { + await allPartners[ii].destroy(); + } + hide = !hide; + } + + // create one with a bogus association type for testing + allPartners.push( + await RestorationPartnerFactory.forProjectReport.create({ partnerableType: "foo", partnerableId: 1 }) + ); + allPartners.push(await RestorationPartnerFactory.forProjectReport.create({ partnerableId: 0 })); + + partners = allPartners.filter(partner => !partner.isSoftDeleted() && partner.hidden === false); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new RestorationPartnerEntity(), + partners, + ({ uuid, collection, partnerableType, partnerableId }) => ({ + fields: { + uuid, + collection, + projectReportUuid: + partnerableType === ProjectReport.LARAVEL_TYPE && partnerableId > 0 + ? associationUuids[partnerableType] + : undefined + } + }) + ); + }); + }); + + describe("SiteEntity", () => { + let projectUuids: Record; + let sites: Site[]; + + beforeAll(async () => { + await Site.truncate(); + + const projects = await ProjectFactory.createMany(2); + projectUuids = projects.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const sites1 = await SiteFactory.createMany(7, { projectId: projects[0].id }); + await sites1[2].destroy(); + const sites2 = await SiteFactory.createMany(8, { projectId: projects[1].id }); + await sites2[1].destroy(); + const siteWithoutProject = await SiteFactory.create({ projectId: null }); + sites = [...sites1, ...sites2, siteWithoutProject].filter(site => !site.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new SiteEntity(), sites, ({ uuid, name, projectId, status }) => ({ + fields: { + uuid, + name, + projectUuid: projectUuids[projectId], + status + } + })); + }); + }); + + describe("SiteReportEntity", () => { + let siteUuids: Record; + let totalSeedsPlanted: Record; + let reports: SiteReport[]; + + beforeAll(async () => { + await SiteReport.truncate(); + + const sites = await SiteFactory.createMany(2); + siteUuids = sites.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {}); + const siteIds = sites.reduce((ids, { id }) => [...ids, id], [] as number[]); + const allReports = []; + for (let ii = 0; ii < 15; ii++) { + allReports.push(await SiteReportFactory.create({ siteId: faker.helpers.arrayElement(siteIds) })); + } + allReports.push(await SiteReportFactory.create({ siteId: null })); + + const seedings = await SeedingFactory.forSiteReport.createMany(3, { seedableId: allReports[0].id }); + totalSeedsPlanted = { [allReports[0].id]: seedings.reduce((total, { amount }) => total + amount, 0) }; + + await allReports[6].destroy(); + reports = allReports.filter(report => !report.isSoftDeleted()); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates(new SiteReportEntity(), reports, ({ id, uuid, siteId, status, dueAt }) => ({ + fields: { + uuid, + siteUuid: siteUuids[siteId], + status, + dueAt, + totalSeedsPlanted: totalSeedsPlanted[id] ?? 0 + } + })); + }); + }); + + describe("TreeSpeciesEntity", () => { + let associationUuids: Record; + let trees: TreeSpecies[]; + + beforeAll(async () => { + await TreeSpecies.truncate(); + + associationUuids = {}; + const nursery = await NurseryFactory.create(); + associationUuids[Nursery.LARAVEL_TYPE] = nursery.uuid; + const nurseryReport = await NurseryReportFactory.create(); + associationUuids[NurseryReport.LARAVEL_TYPE] = nurseryReport.uuid; + const organisation = await OrganisationFactory.create(); + associationUuids[Organisation.LARAVEL_TYPE] = organisation.uuid; + const project = await ProjectFactory.create(); + associationUuids[Project.LARAVEL_TYPE] = project.uuid; + const projectReport = await ProjectReportFactory.create(); + associationUuids[ProjectReport.LARAVEL_TYPE] = projectReport.uuid; + const site = await SiteFactory.create(); + associationUuids[Site.LARAVEL_TYPE] = site.uuid; + const siteReport = await SiteReportFactory.create(); + associationUuids[SiteReport.LARAVEL_TYPE] = siteReport.uuid; + + const factories = [ + () => TreeSpeciesFactory.forNurserySeedling.create({ speciesableId: nursery.id }), + () => TreeSpeciesFactory.forNurseryReportSeedling.create({ speciesableId: nurseryReport.id }), + () => TreeSpeciesFactory.forProjectTreePlanted.create({ speciesableId: project.id }), + () => TreeSpeciesFactory.forProjectReportTreePlanted.create({ speciesableId: projectReport.id }), + () => TreeSpeciesFactory.forSiteTreePlanted.create({ speciesableId: site.id }), + () => TreeSpeciesFactory.forSiteNonTree.create({ speciesableId: site.id }), + () => TreeSpeciesFactory.forSiteReportTreePlanted.create({ speciesableId: siteReport.id }), + () => TreeSpeciesFactory.forSiteReportNonTree.create({ speciesableId: siteReport.id }) + ]; + + const allTrees: TreeSpecies[] = []; + for (const factory of factories) { + // make sure we have at least one of each type + allTrees.push(await factory()); + } + for (let ii = 0; ii < 35; ii++) { + // create a whole bunch more at random + allTrees.push(await faker.helpers.arrayElement(factories)()); + } + const toDeleteOrHide = faker.helpers.uniqueArray(() => faker.number.int(allTrees.length - 1), 10); + let hide = true; + for (const ii of toDeleteOrHide) { + if (hide) { + await allTrees[ii].update({ hidden: true }); + } else { + await allTrees[ii].destroy(); + } + hide = !hide; + } + + // create one with a bogus association type for testing + allTrees.push(await TreeSpeciesFactory.forNurserySeedling.create({ speciesableType: "foo", speciesableId: 3 })); + // create one with a bad association id for testing + allTrees.push(await TreeSpeciesFactory.forNurseryReportSeedling.create({ speciesableId: 0 })); + + trees = allTrees.filter(tree => !tree.isSoftDeleted() && tree.hidden === false); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new TreeSpeciesEntity(), + trees, + ({ uuid, name, amount, collection, speciesableType, speciesableId }) => ({ + fields: { + uuid, + name, + amount, + collection, + nurseryUuid: speciesableType === Nursery.LARAVEL_TYPE ? associationUuids[speciesableType] : undefined, + nurseryReportUuid: + speciesableType === NurseryReport.LARAVEL_TYPE && speciesableId > 0 + ? associationUuids[speciesableType] + : undefined, + projectUuid: speciesableType === Project.LARAVEL_TYPE ? associationUuids[speciesableType] : undefined, + projectReportUuid: + speciesableType === ProjectReport.LARAVEL_TYPE ? associationUuids[speciesableType] : undefined, + siteUuid: speciesableType === Site.LARAVEL_TYPE ? associationUuids[speciesableType] : undefined, + siteReportUuid: speciesableType === SiteReport.LARAVEL_TYPE ? associationUuids[speciesableType] : undefined + } + }) + ); + }); + + it("customizes the where clause for deleting records", async () => { + class Test extends TreeSpeciesEntity { + // make method accessible + public getDeletePageFindOptions = (deletedSince: Date, page: number) => + super.getDeletePageFindOptions(deletedSince, page); + } + const deletedSince = new Date(); + const result = new Test().getDeletePageFindOptions(deletedSince, 0); + expect(result.where[Op.or]).not.toBeNull(); + expect(result.where[Op.or]?.[Op.and]?.updatedAt?.[Op.gte]).toBe(deletedSince); + }); + }); + + describe("WorkdayEntity", () => { + let associationUuids: Record; + let workdays: Workday[]; + + beforeAll(async () => { + await Workday.truncate(); + + associationUuids = {}; + const projectReport = await ProjectReportFactory.create(); + associationUuids[ProjectReport.LARAVEL_TYPE] = projectReport.uuid; + const siteReport = await SiteReportFactory.create(); + associationUuids[SiteReport.LARAVEL_TYPE] = siteReport.uuid; + + const factories = [ + () => WorkdayFactory.forProjectReport.create({ workdayableId: projectReport.id }), + () => WorkdayFactory.forSiteReport.create({ workdayableId: siteReport.id }) + ]; + + const allWorkdays: Workday[] = []; + for (const factory of factories) { + // make sure we have at least one of each type + allWorkdays.push(await factory()); + } + for (let ii = 0; ii < 35; ii++) { + // create a whole bunch mor at random + allWorkdays.push(await faker.helpers.arrayElement(factories)()); + } + const toDeleteOrHide = faker.helpers.uniqueArray(() => faker.number.int(allWorkdays.length - 1), 10); + let hide = true; + for (const ii of toDeleteOrHide) { + if (hide) { + await allWorkdays[ii].update({ hidden: true }); + } else { + await allWorkdays[ii].destroy(); + } + hide = !hide; + } + + // create one with a bogus assocation type for testing + allWorkdays.push(await WorkdayFactory.forProjectReport.create({ workdayableType: "foo", workdayableId: 1 })); + allWorkdays.push(await WorkdayFactory.forSiteReport.create({ workdayableId: 0 })); + + workdays = allWorkdays.filter(workday => !workday.isSoftDeleted() && workday.hidden === false); + }); + + it("sends all records to airtable", async () => { + await testAirtableUpdates( + new WorkdayEntity(), + workdays, + ({ uuid, collection, workdayableType, workdayableId }) => ({ + fields: { + uuid, + collection, + projectReportUuid: + workdayableType === ProjectReport.LARAVEL_TYPE ? associationUuids[workdayableType] : undefined, + siteReportUuid: + workdayableType === SiteReport.LARAVEL_TYPE && workdayableId > 0 + ? associationUuids[workdayableType] + : undefined + } + }) + ); + }); + }); +}); diff --git a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts index 34996f2f..19392179 100644 --- a/apps/unified-database-service/src/airtable/entities/airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/airtable-entity.ts @@ -1,46 +1,288 @@ -import { Model, ModelType } from "sequelize-typescript"; -import { cloneDeep, isArray, isObject, uniq } from "lodash"; -import { Attributes } from "sequelize"; - -export type AirtableEntity> = { - TABLE_NAME: string; - UUID_COLUMN: string; - mapDbEntity: (entity: T) => Promise; - findMany: (pageSize: number, offset: number) => Promise; -}; +import { Model, ModelCtor, ModelType } from "sequelize-typescript"; +import { cloneDeep, flatten, groupBy, isObject, uniq } from "lodash"; +import { Attributes, FindOptions, Op, WhereOptions } from "sequelize"; +import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service"; +import { LoggerService } from "@nestjs/common"; +import Airtable from "airtable"; + +// The Airtable API only supports bulk updates of up to 10 rows. +const AIRTABLE_PAGE_SIZE = 10; + +type UpdateBaseOptions = { startPage?: number; updatedSince?: Date }; + +export abstract class AirtableEntity, AssociationType = Record> { + abstract readonly TABLE_NAME: string; + abstract readonly COLUMNS: ColumnMapping[]; + abstract readonly MODEL: ModelCtor; + readonly IDENTITY_COLUMN: string = "uuid"; + readonly SUPPORTS_UPDATED_SINCE: boolean = true; + readonly HAS_HIDDEN_FLAG: boolean = false; + + protected readonly logger: LoggerService = new TMLogService(AirtableEntity.name); + + /** + * If an airtable entity provides a concrete type for Associations, this method should be overridden + * to execute the necessary DB queries and provide a mapping of record number to concrete instance + * of the association type. + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + protected async loadAssociations(entities: ModelType[]): Promise> { + // The default implementation returns an empty mapping. + return {}; + } + + async updateBase(base: Airtable.Base, { startPage, updatedSince }: UpdateBaseOptions = {}) { + // Get any find options that might have been provided by a subclass to issue this query + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { offset, limit, attributes, include, ...countOptions } = this.getUpdatePageFindOptions(0, updatedSince); + const count = await this.MODEL.count(countOptions); + if (count === 0) { + this.logger.log(`No updates to process, skipping: ${JSON.stringify({ table: this.TABLE_NAME, updatedSince })}`); + return; + } + + const expectedPages = Math.floor(count / AIRTABLE_PAGE_SIZE); + for (let page = startPage ?? 0; await this.processUpdatePage(base, page, updatedSince); page++) { + this.logger.log(`Processed update page: ${JSON.stringify({ table: this.TABLE_NAME, page, expectedPages })}`); + } + } + + async deleteStaleRecords(base: Airtable.Base, deletedSince: Date) { + // Use the delete page find options except limit and offset to get an accurate count + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { offset, limit, attributes, ...countOptions } = this.getDeletePageFindOptions(deletedSince, 0); + const count = await this.MODEL.count(countOptions); + if (count === 0) { + this.logger.log(`No deletes to process, skipping: ${JSON.stringify({ table: this.TABLE_NAME, deletedSince })})`); + return; + } + + const expectedPages = Math.floor(count / AIRTABLE_PAGE_SIZE); + for (let page = 0; await this.processDeletePage(base, deletedSince, page); page++) { + this.logger.log(`Processed delete page: ${JSON.stringify({ table: this.TABLE_NAME, page, expectedPages })}`); + } + } + + protected getUpdatePageFindOptions(page: number, updatedSince?: Date) { + const where = {} as WhereOptions; + + if (this.SUPPORTS_UPDATED_SINCE && updatedSince != null) { + where["updatedAt"] = { [Op.gte]: updatedSince }; + } + if (this.HAS_HIDDEN_FLAG) { + where["hidden"] = false; + } + + return { + attributes: selectAttributes(this.COLUMNS), + include: selectIncludes(this.COLUMNS), + limit: AIRTABLE_PAGE_SIZE, + offset: page * AIRTABLE_PAGE_SIZE, + where + } as FindOptions; + } + + protected getDeletePageFindOptions(deletedSince: Date, page: number) { + const where = {} as WhereOptions; + + const deletedAtCondition = { [Op.gte]: deletedSince }; + if (this.HAS_HIDDEN_FLAG) { + where[Op.or] = { + deletedAt: deletedAtCondition, + // include records that have been hidden since the timestamp as well + [Op.and]: { + updatedAt: { ...deletedAtCondition }, + hidden: true + } + }; + } else { + where["deletedAt"] = deletedAtCondition; + } + + return { + attributes: [this.IDENTITY_COLUMN], + paranoid: false, + where, + limit: AIRTABLE_PAGE_SIZE, + offset: page * AIRTABLE_PAGE_SIZE + } as FindOptions; + } + + private async processUpdatePage(base: Airtable.Base, page: number, updatedSince?: Date) { + let airtableRecords: { fields: object }[]; + try { + const records = await this.MODEL.findAll(this.getUpdatePageFindOptions(page, updatedSince)); + + // Page had no records, halt processing. + if (records.length === 0) return false; + + const associations = await this.loadAssociations(records); + + airtableRecords = await Promise.all( + records.map(async record => ({ fields: await this.mapEntityColumns(record, associations[record.id]) })) + ); + } catch (error) { + this.logger.error(`Airtable mapping failed: ${JSON.stringify({ entity: this.TABLE_NAME, page })}`, error.stack); + throw error; + } + + try { + // @ts-expect-error The types for this lib haven't caught up with its support for upserts + // https://github.com/Airtable/airtable.js/issues/348 + await base(this.TABLE_NAME).update(airtableRecords, { + performUpsert: { fieldsToMergeOn: [this.IDENTITY_COLUMN] }, + // Enables new multi/single selection options to be populated by this upsert. + typecast: true + }); + } catch (error) { + this.logger.error( + `Entity update failed: ${JSON.stringify({ entity: this.TABLE_NAME, page, airtableRecords }, null, 2)}`, + error.stack + ); + throw error; + } -export type MergeableInclude = { + // True signals that processing succeeded and the next page should begin + return true; + } + + private async processDeletePage(base: Airtable.Base, deletedSince: Date, page: number) { + let idMapping: Record; + + try { + const records = (await this.MODEL.findAll( + this.getDeletePageFindOptions(deletedSince, page) + )) as unknown as UuidModel[]; + + // Page had no records, halt processing. + if (records.length === 0) return false; + + const formula = `OR(${records + .map(record => `{${this.IDENTITY_COLUMN}}='${record[this.IDENTITY_COLUMN]}'`) + .join(",")})`; + const result = await base(this.TABLE_NAME) + .select({ filterByFormula: formula, fields: [this.IDENTITY_COLUMN] }) + .firstPage(); + + idMapping = result.reduce( + (idMapping, { id, fields }) => ({ + ...idMapping, + [id]: fields[this.IDENTITY_COLUMN] + }), + {} + ); + } catch (error) { + this.logger.error( + `Fetching Airtable records failed: ${JSON.stringify({ entity: this.TABLE_NAME, page })}`, + error.stack + ); + throw error; + } + + const recordIds = Object.keys(idMapping); + // None of these records in our DB currently exist on AirTable. On to the next page. + if (recordIds.length == 0) return true; + + try { + await base(this.TABLE_NAME).destroy(recordIds); + this.logger.log(`Deleted records from Airtable: ${JSON.stringify({ entity: this.TABLE_NAME, idMapping })}`); + } catch (error) { + this.logger.error( + `Airtable record delete failed: ${JSON.stringify({ entity: this.TABLE_NAME, page, idMapping })}`, + error.stack + ); + throw error; + } + + // True signals that processing succeeded and the next page should begin + return true; + } + + protected async mapEntityColumns(record: ModelType, associations: AssociationType) { + const airtableObject = {}; + for (const mapping of this.COLUMNS) { + airtableObject[airtableColumnName(mapping)] = isObject(mapping) + ? await mapping.valueMap(record, associations) + : record[mapping]; + } + + return airtableObject; + } + + protected async loadPolymorphicUuidAssociations( + typeMappings: Record>, + typeColumn: keyof Attributes, + idColumn: keyof Attributes, + entities: ModelType[] + ) { + const byType = groupBy(entities, typeColumn); + const associations = {} as Record; + + // This loop takes the polymorphic types that have been grouped from this set of entities, queries + // the appropriate models to find their UUIDs, and then associates that UUID with the correct + // member of the association type for that entity. Each entity will only have one of these + // UUIDs set. + for (const type of Object.keys(byType)) { + if (typeMappings[type] == null) { + this.logger.error(`Speciesable type not recognized, ignoring [${type}]`); + continue; + } + + const { model, association } = typeMappings[type]; + const entitiesForType = byType[type]; + const ids = uniq(entitiesForType.map(entity => entity[idColumn])) as number[]; + const models = await model.findAll({ where: { id: ids }, attributes: ["id", "uuid"] }); + for (const entity of entitiesForType) { + const { uuid } = (models.find(({ id }) => id === entity[idColumn]) as unknown as { uuid: string }) ?? {}; + associations[entity.id as number] = { [association]: uuid } as AssociationType; + } + } + + return associations; + } +} + +export type Include = { model?: ModelType; association?: string; attributes?: string[]; - include?: MergeableInclude[]; }; /** - * A ColumnMapping is either a tuple of [dbColumn, airtableColumn], or a more descriptive object + * A ColumnMapping is either a string (airtableColumn and dbColumn are the same), or a more descriptive object */ -export type ColumnMapping> = +export type ColumnMapping, A = Record> = | keyof Attributes - | [keyof Attributes, string] | { airtableColumn: string; // Include if this mapping should include a particular DB column in the DB query - dbColumn?: keyof Attributes; + dbColumn?: keyof Attributes | (keyof Attributes)[]; // Include if this mapping should eager load an association on the DB query - include?: MergeableInclude[]; - valueMap: (entity: T) => Promise; + include?: Include[]; + valueMap: (entity: T, associations: A) => Promise; }; -export const selectAttributes = >(columns: ColumnMapping[]) => - columns - .map(mapping => (isArray(mapping) ? mapping[0] : isObject(mapping) ? mapping.dbColumn : mapping)) - .filter(dbColumn => dbColumn != null); +export type PolymorphicUuidAssociation = { + model: ModelCtor; + association: keyof AssociationType; +}; + +// used in the test suite +export const airtableColumnName = >(mapping: ColumnMapping) => + isObject(mapping) ? mapping.airtableColumn : (mapping as string); + +const selectAttributes = , A>(columns: ColumnMapping[]) => + uniq([ + "id", + ...flatten( + columns.map(mapping => (isObject(mapping) ? mapping.dbColumn : mapping)).filter(dbColumn => dbColumn != null) + ) + ]); /** - * Recursively merges MergeableIncludes to arrive at a cohesive set of IncludeOptions for a Sequelize find - * query. + * Merges Includes to arrive at a cohesive set of IncludeOptions for a Sequelize find query. */ -const mergeInclude = (includes: MergeableInclude[], include: MergeableInclude) => { +const mergeInclude = (includes: Include[], include: Include) => { const existing = includes.find( ({ model, association }) => (model != null && model === include.model) || (association != null && association === include.association) @@ -60,42 +302,38 @@ const mergeInclude = (includes: MergeableInclude[], include: MergeableInclude) = existing.attributes = uniq([...existing.attributes, ...include.attributes]); } } - - if (include.include != null) { - // Use clone deep here so that if this include gets modified in the future, it doesn't mutate the - // original definition. - if (existing.include == null) existing.include = cloneDeep(include.include); - else { - existing.include = include.include.reduce(mergeInclude, existing.include); - } - } } return includes; }; -export const selectIncludes = >(columns: ColumnMapping[]) => +const selectIncludes = , A>(columns: ColumnMapping[]) => columns.reduce((includes, mapping) => { - if (isArray(mapping) || !isObject(mapping)) return includes; + if (!isObject(mapping)) return includes; if (mapping.include == null) return includes; return mapping.include.reduce(mergeInclude, includes); - }, [] as MergeableInclude[]); - -export const mapEntityColumns = async >(entity: T, columns: ColumnMapping[]) => { - const airtableObject = {}; - for (const mapping of columns) { - const airtableColumn = isArray(mapping) - ? mapping[1] - : isObject(mapping) - ? mapping.airtableColumn - : (mapping as string); - airtableObject[airtableColumn] = isArray(mapping) - ? entity[mapping[0]] - : isObject(mapping) - ? await mapping.valueMap(entity) - : entity[mapping]; - } + }, [] as Include[]); - return airtableObject; -}; +type UuidModel = Model & { uuid: string }; +export const commonEntityColumns = , A = Record>(adminSiteType: string) => + [ + "uuid", + "createdAt", + "updatedAt", + { + airtableColumn: "linkToTerramatch", + dbColumn: "uuid", + valueMap: ({ uuid }) => `https://www.terramatch.org/admin#/${adminSiteType}/${uuid}/show` + } + ] as ColumnMapping[]; + +export const associatedValueColumn = , A>( + valueName: keyof A, + dbColumn: keyof Attributes | (keyof Attributes)[] +) => + ({ + airtableColumn: valueName, + dbColumn, + valueMap: async (_, associations: A) => associations?.[valueName] + } as ColumnMapping); diff --git a/apps/unified-database-service/src/airtable/entities/application.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/application.airtable-entity.ts new file mode 100644 index 00000000..d9b22f7a --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/application.airtable-entity.ts @@ -0,0 +1,55 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { Application, FormSubmission, FundingProgramme } from "@terramatch-microservices/database/entities"; +import { groupBy, orderBy, uniq } from "lodash"; + +const loadFormSubmissions = async (applicationIds: number[]) => + groupBy( + await FormSubmission.findAll({ + where: { applicationId: applicationIds }, + attributes: ["applicationId", "id", "status"] + }), + "applicationId" + ); + +type ApplicationAssociations = { + fundingProgrammeName?: string; + formSubmissions: FormSubmission[]; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("application"), + "organisationUuid", + associatedValueColumn("fundingProgrammeName", "fundingProgrammeUuid"), + { + airtableColumn: "status", + valueMap: async (_, { formSubmissions }) => orderBy(formSubmissions, ["id"], ["desc"])[0]?.status + } +]; + +export class ApplicationEntity extends AirtableEntity { + readonly TABLE_NAME = "Applications"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Application; + readonly SUPPORTS_UPDATED_SINCE = false; + + protected async loadAssociations(applications: Application[]) { + const applicationIds = applications.map(({ id }) => id); + const fundingProgrammeUuids = uniq(applications.map(({ fundingProgrammeUuid }) => fundingProgrammeUuid)); + const fundingProgrammes = await FundingProgramme.findAll({ + where: { uuid: fundingProgrammeUuids }, + attributes: ["uuid", "name"] + }); + const formSubmissions = await loadFormSubmissions(applicationIds); + + return applications.reduce( + (associations, { id, fundingProgrammeUuid }) => ({ + ...associations, + [id]: { + fundingProgrammeName: fundingProgrammes.find(({ uuid }) => uuid === fundingProgrammeUuid)?.name, + formSubmissions: formSubmissions[id] ?? [] + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/demographic.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/demographic.airtable-entity.ts new file mode 100644 index 00000000..94108366 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/demographic.airtable-entity.ts @@ -0,0 +1,44 @@ +import { Demographic, RestorationPartner, Workday } from "@terramatch-microservices/database/entities"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, PolymorphicUuidAssociation } from "./airtable-entity"; + +const LARAVEL_TYPE_MAPPINGS: Record> = { + [Workday.LARAVEL_TYPE]: { + association: "workdayUuid", + model: Workday + }, + [RestorationPartner.LARAVEL_TYPE]: { + association: "restorationPartnerUuid", + model: RestorationPartner + } +}; + +type DemographicAssociations = { + workdayUuid?: string; + restorationPartnerUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + "id", + "type", + "subtype", + "name", + "amount", + associatedValueColumn("workdayUuid", ["demographicalType", "demographicalId"]), + associatedValueColumn("restorationPartnerUuid", ["demographicalType", "demographicalId"]) +]; + +export class DemographicEntity extends AirtableEntity { + readonly TABLE_NAME = "Demographics"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Demographic; + readonly IDENTITY_COLUMN = "id"; + + protected async loadAssociations(demographics: Demographic[]) { + return this.loadPolymorphicUuidAssociations( + LARAVEL_TYPE_MAPPINGS, + "demographicalType", + "demographicalId", + demographics + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/index.ts b/apps/unified-database-service/src/airtable/entities/index.ts index 574c0efc..39a4d18f 100644 --- a/apps/unified-database-service/src/airtable/entities/index.ts +++ b/apps/unified-database-service/src/airtable/entities/index.ts @@ -1 +1,12 @@ -export { ProjectEntity } from "./project.airtable-entity"; +export * from "./application.airtable-entity"; +export * from "./demographic.airtable-entity"; +export * from "./nursery.airtable-entity"; +export * from "./nursery-report.airtable-entity"; +export * from "./organisation.airtable-entity"; +export * from "./project.airtable-entity"; +export * from "./project-report.airtable-entity"; +export * from "./restoration-partner.airtable-entity"; +export * from "./site.airtable-entity"; +export * from "./site-report.airtable-entity"; +export * from "./tree-species.airtable-entity"; +export * from "./workday.airtable-entity"; diff --git a/apps/unified-database-service/src/airtable/entities/nursery-report.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/nursery-report.airtable-entity.ts new file mode 100644 index 00000000..7c447406 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/nursery-report.airtable-entity.ts @@ -0,0 +1,40 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { Nursery, NurseryReport } from "@terramatch-microservices/database/entities"; +import { uniq } from "lodash"; + +type NurseryReportAssociations = { + nurseryUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("nurseryReport"), + associatedValueColumn("nurseryUuid", "nurseryId"), + "status", + "updateRequestStatus", + "dueAt", + "seedlingsYoungTrees" +]; + +export class NurseryReportEntity extends AirtableEntity { + readonly TABLE_NAME = "Nursery Reports"; + readonly COLUMNS = COLUMNS; + readonly MODEL = NurseryReport; + + protected async loadAssociations(nurseryReports: NurseryReport[]) { + const nurseryIds = uniq(nurseryReports.map(({ nurseryId }) => nurseryId)); + const nurseries = await Nursery.findAll({ + where: { id: nurseryIds }, + attributes: ["id", "uuid"] + }); + + return nurseryReports.reduce( + (associations, { id, nurseryId }) => ({ + ...associations, + [id]: { + nurseryUuid: nurseries.find(({ id }) => id === nurseryId)?.uuid + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/nursery.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/nursery.airtable-entity.ts new file mode 100644 index 00000000..869dd991 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/nursery.airtable-entity.ts @@ -0,0 +1,39 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { Nursery, Project } from "@terramatch-microservices/database/entities"; +import { uniq } from "lodash"; + +type NurseryAssociations = { + projectUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("nursery"), + "name", + associatedValueColumn("projectUuid", "projectId"), + "status", + "updateRequestStatus" +]; + +export class NurseryEntity extends AirtableEntity { + readonly TABLE_NAME = "Nurseries"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Nursery; + + protected async loadAssociations(nurseries: Nursery[]) { + const projectIds = uniq(nurseries.map(({ projectId }) => projectId)); + const projects = await Project.findAll({ + where: { id: projectIds }, + attributes: ["id", "uuid"] + }); + + return nurseries.reduce( + (associations, { id, projectId }) => ({ + ...associations, + [id]: { + projectUuid: projects.find(({ id }) => id === projectId)?.uuid + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts new file mode 100644 index 00000000..5373fc8d --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/organisation.airtable-entity.ts @@ -0,0 +1,116 @@ +import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { Organisation } from "@terramatch-microservices/database/entities"; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("organisation"), + "status", + "type", + "private", + "isTest", + "name", + "phone", + "hqStreet1", + "hqStreet2", + "hqCity", + "hqState", + "hqZipcode", + "hqCountry", + "leadershipTeamTxt", + "foundingDate", + "description", + "countries", + "languages", + "treeCareApproach", + "relevantExperienceYears", + "treesGrown3Year", + "treesGrownTotal", + "haRestored3Year", + "haRestoredTotal", + "finStartMonth", + "finBudgetCurrentYear", + "finBudget1Year", + "finBudget2Year", + "finBudget3Year", + "webUrl", + "facebookUrl", + "instagramUrl", + "linkedinUrl", + "twitterUrl", + "ftPermanentEmployees", + "ptPermanentEmployees", + "tempEmployees", + "femaleEmployees", + "maleEmployees", + "youngEmployees", + "over35Employees", + "additionalFundingDetails", + "communityExperience", + "totalEngagedCommunityMembers3Yr", + "percentEngagedWomen3Yr", + "percentEngagedMen3Yr", + "percentEngagedUnder353Yr", + "percentEngagedOver353Yr", + "percentEngagedSmallholder3Yr", + "totalTreesGrown", + "avgTreeSurvivalRate", + "treeMaintenanceAftercareApproach", + "restoredAreasDescription", + "restorationTypesImplemented", + "historicMonitoringGeojson", + "monitoringEvaluationExperience", + "fundingHistory", + "engagementFarmers", + "engagementWomen", + "engagementYouth", + "currency", + "states", + "district", + "accountNumber1", + "accountNumber2", + "loanStatusAmount", + "loanStatusTypes", + "approachOfMarginalizedCommunities", + "communityEngagementNumbersMarginalized", + "landSystems", + "fundUtilisation", + "detailedInterventionTypes", + "communityMembersEngaged3yr", + "communityMembersEngaged3yrWomen", + "communityMembersEngaged3yrMen", + "communityMembersEngaged3yrYouth", + "communityMembersEngaged3yrNonYouth", + "communityMembersEngaged3yrSmallholder", + "communityMembersEngaged3YrBackwardClass", + "totalBoardMembers", + "pctBoardWomen", + "pctBoardMen", + "pctBoardYouth", + "pctBoardNonYouth", + "engagementNonYouth", + "treeRestorationPractices", + "businessModel", + "subtype", + "organisationRevenueThisYear", + "fieldStaffSkills", + "fpcCompany", + "numOfFarmersOnBoard", + "numOfMarginalisedEmployees", + "benefactorsFpcCompany", + "boardRemunerationFpcCompany", + "boardEngagementFpcCompany", + "biodiversityFocus", + "globalPlanningFrameworks", + "pastGovCollaboration", + "engagementLandless", + "socioeconomicImpact", + "environmentalImpact", + "growthStage", + "totalEmployees", + "additionalComments" +]; + +export class OrganisationEntity extends AirtableEntity { + readonly TABLE_NAME = "Organisations"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Organisation; +} diff --git a/apps/unified-database-service/src/airtable/entities/project-report.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/project-report.airtable-entity.ts new file mode 100644 index 00000000..3ee5807b --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/project-report.airtable-entity.ts @@ -0,0 +1,137 @@ +import { NurseryReport, Project, ProjectReport, TreeSpecies } from "@terramatch-microservices/database/entities"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { groupBy, uniq } from "lodash"; + +type ProjectReportAssociations = { + projectUuid?: string; + associatedNurseryReports: NurseryReport[]; + trees: TreeSpecies[]; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("projectReport"), + associatedValueColumn("projectUuid", "projectId"), + "status", + "updateRequestStatus", + "dueAt", + "landscapeCommunityContribution", + "topThreeSuccesses", + "challengesFaced", + "lessonsLearned", + "maintenanceAndMonitoringActivities", + "significantChange", + "pctSurvivalToDate", + "survivalCalculation", + "survivalComparison", + "newJobsDescription", + "ftWomen", + "ftTotal", + "ftNonYouth", + "ftYouth", + "ftMen", + "ftOther", + "ptWomen", + "ptMen", + "ptYouth", + "ptNonYouth", + "ptTotal", + "ptOther", + "volunteerWomen", + "volunteerTotal", + "volunteerNonYouth", + "volunteerYouth", + "volunteerMen", + "volunteersWorkDescription", + "volunteerOther", + "beneficiaries", + "beneficiariesDescription", + "beneficiariesWomen", + "beneficiariesLargeScale", + "beneficiariesSmallholder", + "beneficiariesNonYouth", + "beneficiariesYouth", + "beneficiariesMen", + "beneficiariesOther", + "beneficiariesTrainingWomen", + "beneficiariesTrainingMen", + "beneficiariesTrainingOther", + "beneficiariesTrainingYouth", + "beneficiariesTrainingNonYouth", + "beneficiariesIncomeIncrease", + "beneficiariesIncomeIncreaseDescription", + "beneficiariesSkillsKnowledgeIncrease", + "beneficiariesSkillsKnowledgeIncreaseDescription", + "sharedDriveLink", + "communityProgress", + "localEngagementDescription", + "equitableOpportunities", + "indirectBeneficiaries", + "indirectBeneficiariesDescription", + "resilienceProgress", + "localGovernance", + "adaptiveManagement", + "scalabilityReplicability", + "convergenceJobsDescription", + "convergenceSchemes", + "volunteerScstobc", + "beneficiariesScstobc", + "beneficiariesScstobcFarmers", + "communityPartnersAssetsDescription", + "peopleKnowledgeSkillsIncreased", + { + airtableColumn: "totalSeedlingsGrown", + dbColumn: ["frameworkKey", "taskId"], + valueMap: async ({ frameworkKey }, { trees, associatedNurseryReports }) => + frameworkKey === "ppc" + ? trees.reduce((total, { amount }) => total + amount, 0) + : frameworkKey === "terrafund" + ? associatedNurseryReports.reduce((total, { seedlingsYoungTrees }) => total + seedlingsYoungTrees, 0) + : 0 + }, + "technicalNarrative", + "publicNarrative", + "totalUniqueRestorationPartners" +]; + +export class ProjectReportEntity extends AirtableEntity { + readonly TABLE_NAME = "Project Reports"; + readonly COLUMNS = COLUMNS; + readonly MODEL = ProjectReport; + readonly SUPPORTS_UPDATED_SINCE = false; + + protected async loadAssociations(projectReports: ProjectReport[]) { + const reportIds = projectReports.map(({ id }) => id); + const projectIds = uniq(projectReports.map(({ projectId }) => projectId)); + const projects = await Project.findAll({ + where: { id: projectIds }, + attributes: ["id", "uuid"] + }); + const taskIds = projectReports.map(({ taskId }) => taskId); + const nurseryReportsByTaskId = groupBy( + await NurseryReport.findAll({ + where: { taskId: taskIds, status: NurseryReport.APPROVED_STATUSES }, + attributes: ["id", "taskId", "seedlingsYoungTrees"] + }), + "taskId" + ); + const treesByReportId = groupBy( + await TreeSpecies.findAll({ + where: { speciesableType: ProjectReport.LARAVEL_TYPE, speciesableId: reportIds, hidden: false }, + attributes: ["id", "speciesableId", "amount"] + }), + "speciesableId" + ); + + return projectReports.reduce( + (associations, { id, projectId, taskId }) => ({ + ...associations, + [id]: { + projectUuid: projects.find(({ id }) => id === projectId)?.uuid, + associatedNurseryReports: nurseryReportsByTaskId[taskId] ?? [], + trees: treesByReportId[id] ?? [] + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts index b3f10442..39468bdb 100644 --- a/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts +++ b/apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts @@ -1,32 +1,37 @@ -import { - Application, - Demographic, - Nursery, - Organisation, - Project, - ProjectReport, - Site, - SitePolygon, - SiteReport, - Workday -} from "@terramatch-microservices/database/entities"; -import { AirtableEntity, ColumnMapping, mapEntityColumns, selectAttributes, selectIncludes } from "./airtable-entity"; -import { flatten, flattenDeep } from "lodash"; -import { literal, Op } from "sequelize"; +import { Application, Organisation, Project, Site, SitePolygon } from "@terramatch-microservices/database/entities"; +import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { flatten, groupBy } from "lodash"; +import { FRAMEWORK_NAMES } from "@terramatch-microservices/database/constants/framework"; -const COHORTS = { - terrafund: "TerraFund Top 100", - "terrafund-landscapes": "TerraFund Landscapes", - ppc: "Priceless Planet Coalition (PPC)" +const loadApprovedSites = async (projectIds: number[]) => + groupBy( + await Site.findAll({ + where: { projectId: projectIds, status: Site.APPROVED_STATUSES }, + attributes: ["id", "uuid", "projectId"] + }), + "projectId" + ); + +const loadSitePolygons = async (siteUuids: string[]) => + groupBy( + await SitePolygon.findAll({ + where: { siteUuid: siteUuids, isActive: true }, + attributes: ["siteUuid", "calcArea"] + }), + "siteUuid" + ); + +type ProjectAssociations = { + sitePolygons: SitePolygon[]; }; -const COLUMNS: ColumnMapping[] = [ - "uuid", +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("project"), "name", { dbColumn: "frameworkKey", airtableColumn: "cohort", - valueMap: async ({ frameworkKey }) => COHORTS[frameworkKey] ?? frameworkKey + valueMap: async ({ frameworkKey }) => FRAMEWORK_NAMES[frameworkKey] ?? frameworkKey }, { airtableColumn: "applicationUuid", @@ -38,11 +43,6 @@ const COLUMNS: ColumnMapping[] = [ include: [{ model: Organisation, attributes: ["uuid"] }], valueMap: async ({ organisation }) => organisation?.uuid }, - { - airtableColumn: "organisationName", - include: [{ model: Organisation, attributes: ["name"] }], - valueMap: async ({ organisation }) => organisation?.name - }, "status", "country", "description", @@ -54,16 +54,6 @@ const COLUMNS: ColumnMapping[] = [ "sitingStrategy", "sitingStrategyDescription", "history", - { - airtableColumn: "treeSpecies", - include: [{ association: "treesPlanted", attributes: ["name"] }], - valueMap: async ({ treesPlanted }) => treesPlanted?.map(({ name }) => name)?.join(", ") - }, - { - airtableColumn: "treeSpeciesCount", - include: [{ association: "treesPlanted", attributes: ["name"] }], - valueMap: async ({ treesPlanted }) => treesPlanted?.length ?? 0 - }, "treesGrownGoal", "totalHectaresRestoredGoal", "environmentalGoals", @@ -80,281 +70,56 @@ const COLUMNS: ColumnMapping[] = [ "goalTreesRestoredPlanting", "goalTreesRestoredAnr", "goalTreesRestoredDirectSeeding", - { - airtableColumn: "treesPlantedToDate", - include: [ - { - model: Site, - attributes: ["status"], - include: [ - { - model: SiteReport, - attributes: ["status"], - include: [{ association: "treesPlanted", attributes: ["amount", "hidden"] }] - } - ] - } - ], - // We could potentially limit the number of rows in the query by filtering for these statuses - // in a where clause, but the mergeable include system is complicated enough without it trying - // to understand how to merge where clauses, so doing this filtering in memory is fine. - valueMap: async ({ sites }) => - flattenDeep( - (sites ?? []) - .filter(({ status }) => Site.APPROVED_STATUSES.includes(status)) - .map(({ reports }) => - (reports ?? []) - .filter(({ status }) => SiteReport.APPROVED_STATUSES.includes(status)) - .map(({ treesPlanted }) => treesPlanted?.map(({ amount }) => amount)) - ) - ).reduce((sum, amount) => sum + (amount ?? 0), 0) - }, - { - airtableColumn: "seedsPlantedToDate", - include: [ - { - model: Site, - attributes: ["status"], - include: [ - { - model: SiteReport, - attributes: ["status"], - include: [{ association: "seedsPlanted", attributes: ["amount", "hidden"] }] - } - ] - } - ], - // We could potentially limit the number of rows in the query by filtering for these statuses - // in a where clause, but the mergeable include system is complicated enough without it trying - // to understand how to merge where clauses, so doing this filtering in memory is fine. - valueMap: async ({ sites }) => - flattenDeep( - (sites ?? []) - .filter(({ status }) => Site.APPROVED_STATUSES.includes(status)) - .map(({ reports }) => - (reports ?? []) - .filter(({ status }) => SiteReport.APPROVED_STATUSES.includes(status)) - .map(({ seedsPlanted }) => seedsPlanted?.map(({ amount }) => amount)) - ) - ).reduce((sum, amount) => sum + (amount ?? 0), 0) - }, - { - airtableColumn: "jobsCreatedToDate", - include: [ - { - model: ProjectReport, - attributes: ["status", "ftTotal", "ptTotal"] - } - ], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ftTotal, ptTotal }) => sum + (ftTotal ?? 0) + (ptTotal ?? 0), 0) - }, { airtableColumn: "hectaresRestoredToDate", - include: [ - { - model: Site, - attributes: ["uuid"] - } - ], - // A given project can end up with _a lot_ of site polygons, which blows up the - // first SQL query into too many rows, so doing this one as its own query is more - // efficient. - valueMap: async ({ sites }) => { - if (sites == null || sites.length === 0) return 0; - const sitePolygons = await SitePolygon.findAll({ - where: { siteUuid: { [Op.in]: sites.map(({ uuid }) => uuid) }, isActive: true }, - attributes: ["calcArea"] - }); - return Math.round(sitePolygons.reduce((total, { calcArea }) => total + calcArea, 0)); - } - }, - { - airtableColumn: "numberOfSites", - include: [{ model: Site, attributes: ["id"] }], - valueMap: async ({ sites }) => (sites ?? []).length - }, - { - airtableColumn: "numberOfNurseries", - include: [{ model: Nursery, attributes: ["id"] }], - valueMap: async ({ nurseries }) => (nurseries ?? []).length - }, - "continent", - { - airtableColumn: "ftWomen", - include: [{ model: ProjectReport, attributes: ["ftWomen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ftWomen }) => sum + ftWomen, 0) - }, - { - airtableColumn: "ftMen", - include: [{ model: ProjectReport, attributes: ["ftMen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ftMen }) => sum + ftMen, 0) - }, - { - airtableColumn: "ftYouth", - include: [{ model: ProjectReport, attributes: ["ftYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ftYouth }) => sum + ftYouth, 0) - }, - { - airtableColumn: "ftNonYouth", - include: [{ model: ProjectReport, attributes: ["ftNonYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ftNonYouth }) => sum + ftNonYouth, 0) - }, - { - airtableColumn: "ptWomen", - include: [{ model: ProjectReport, attributes: ["ptWomen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ptWomen }) => sum + ptWomen, 0) - }, - { - airtableColumn: "ptMen", - include: [{ model: ProjectReport, attributes: ["ptMen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ptMen }) => sum + ptMen, 0) + valueMap: async (_, { sitePolygons }) => + Math.round(sitePolygons.reduce((total, { calcArea }) => total + calcArea, 0)) }, { - airtableColumn: "ptYouth", - include: [{ model: ProjectReport, attributes: ["ptYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ptYouth }) => sum + ptYouth, 0) - }, - { - airtableColumn: "ptNonYouth", - include: [{ model: ProjectReport, attributes: ["ptNonYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { ptNonYouth }) => sum + ptNonYouth, 0) - }, - { - airtableColumn: "volunteerTotal", - include: [{ model: ProjectReport, attributes: ["volunteerTotal"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { volunteerTotal }) => sum + volunteerTotal, 0) - }, - { - airtableColumn: "volunteerWomen", - include: [{ model: ProjectReport, attributes: ["volunteerWomen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { volunteerWomen }) => sum + volunteerWomen, 0) - }, - { - airtableColumn: "volunteerMen", - include: [{ model: ProjectReport, attributes: ["volunteerMen"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { volunteerMen }) => sum + volunteerMen, 0) - }, - { - airtableColumn: "volunteerYouth", - include: [{ model: ProjectReport, attributes: ["volunteerYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { volunteerYouth }) => sum + volunteerYouth, 0) - }, - { - airtableColumn: "volunteerNonYouth", - include: [{ model: ProjectReport, attributes: ["volunteerNonYouth"] }], - valueMap: async ({ reports }) => - (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .reduce((sum, { volunteerNonYouth }) => sum + volunteerNonYouth, 0) - }, - { - airtableColumn: "workdaysCount", - include: [ - { model: ProjectReport, attributes: ["id", "status"] }, - { - model: Site, - attributes: ["status"], - include: [ - { - model: SiteReport, - attributes: ["id", "status"] - } - ] - } - ], - valueMap: async ({ reports, sites }) => { - const siteReportIds = flatten( - (sites ?? []).filter(({ status }) => Site.APPROVED_STATUSES.includes(status)).map(({ reports }) => reports) - ) - .filter(({ status }) => SiteReport.APPROVED_STATUSES.includes(status)) - .map(({ id }) => id); - const siteReportWorkdays = literal(`( - select id - from v2_workdays - where workdayable_type = '${SiteReport.LARAVEL_TYPE.replace(/\\/g, "\\\\")}' - and workdayable_id in (${siteReportIds.join(",")}) - and hidden = false - )`); - - const projectReportIds = (reports ?? []) - .filter(({ status }) => ProjectReport.APPROVED_STATUSES.includes(status)) - .map(({ id }) => id); - const projectReportWorkdays = literal(`( - select id - from v2_workdays - where workdayable_type = '${ProjectReport.LARAVEL_TYPE.replace(/\\/g, "\\\\")}' - and workdayable_id in (${projectReportIds.join(",")}) - and hidden = false - )`); - - return ( - await Demographic.findAll({ - attributes: ["amount"], - where: { - demographicalType: Workday.LARAVEL_TYPE, - demographicalId: { - [Op.or]: [{ [Op.in]: siteReportWorkdays }, { [Op.in]: projectReportWorkdays }] - } - } - }) - ).reduce((count, { amount }) => count + amount, 0); - } + airtableColumn: "continent", + dbColumn: "continent", + valueMap: async ({ continent }) => continent?.replace("_", "-") }, "survivalRate", "descriptionOfProjectTimeline", - "landholderCommEngage" + "landholderCommEngage", + "states", + "detailedInterventionTypes", + "waterSource", + "baselineBiodiversity", + "projImpactFoodsec", + "projImpactBiodiv", + "pctEmployeesMarginalised", + "pctBeneficiariesMarginalised", + "proposedGovPartners", + "yearFiveCrownCover", + "directSeedingSurvivalRate" ]; -export const ProjectEntity: AirtableEntity = { - TABLE_NAME: "Projects", - UUID_COLUMN: "uuid", +export class ProjectEntity extends AirtableEntity { + readonly TABLE_NAME = "Projects"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Project; + readonly SUPPORTS_UPDATED_SINCE = false; - findMany: async (pageSize: number, offset: number) => - await Project.findAll({ - attributes: selectAttributes(COLUMNS), - include: selectIncludes(COLUMNS), - limit: pageSize, - offset - }), + async loadAssociations(projects: Project[]) { + const projectIds = projects.map(({ id }) => id); + const approvedSites = await loadApprovedSites(projectIds); + const allSiteUuids = flatten(Object.values(approvedSites).map(sites => sites.map(({ uuid }) => uuid))); + const sitePolygons = await loadSitePolygons(allSiteUuids); - mapDbEntity: async (project: Project) => await mapEntityColumns(project, COLUMNS) -}; + return projectIds.reduce((associations, projectId) => { + const sites = approvedSites[projectId] ?? []; + + return { + ...associations, + [projectId]: { + sitePolygons: sites.reduce( + (polygons, { uuid }) => [...polygons, ...(sitePolygons[uuid] ?? [])], + [] as SitePolygon[] + ) + } + }; + }, {} as Record); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/restoration-partner.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/restoration-partner.airtable-entity.ts new file mode 100644 index 00000000..892d3ce2 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/restoration-partner.airtable-entity.ts @@ -0,0 +1,31 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, PolymorphicUuidAssociation } from "./airtable-entity"; +import { ProjectReport, RestorationPartner } from "@terramatch-microservices/database/entities"; + +const LARAVEL_TYPE_MAPPINGS: Record> = { + [ProjectReport.LARAVEL_TYPE]: { + association: "projectReportUuid", + model: ProjectReport + } +}; + +type RestorationPartnerAssociations = { + projectReportUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + "uuid", + "collection", + "description", + associatedValueColumn("projectReportUuid", ["partnerableId", "partnerableType"]) +]; + +export class RestorationPartnerEntity extends AirtableEntity { + readonly TABLE_NAME = "Restoration Partners"; + readonly COLUMNS = COLUMNS; + readonly MODEL = RestorationPartner; + readonly HAS_HIDDEN_FLAG = true; + + protected async loadAssociations(partners: RestorationPartner[]) { + return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPINGS, "partnerableType", "partnerableId", partners); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/site-report.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/site-report.airtable-entity.ts new file mode 100644 index 00000000..e9c9c870 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/site-report.airtable-entity.ts @@ -0,0 +1,64 @@ +import { Seeding, Site, SiteReport } from "@terramatch-microservices/database/entities"; +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { groupBy, uniq } from "lodash"; + +type SiteReportAssociations = { + siteUuid?: string; + totalSeedsPlanted: number; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("siteReport"), + associatedValueColumn("siteUuid", "siteId"), + "status", + "updateRequestStatus", + "dueAt", + "pctSurvivalToDate", + "survivalCalculation", + "survivalDescription", + "maintenanceActivities", + "regenerationDescription", + "technicalNarrative", + "publicNarrative", + associatedValueColumn("totalSeedsPlanted", "id"), + "numTreesRegenerating", + "soilWaterRestorationDescription", + "waterStructures" +]; + +export class SiteReportEntity extends AirtableEntity { + readonly TABLE_NAME = "Site Reports"; + readonly COLUMNS = COLUMNS; + readonly MODEL = SiteReport; + + protected async loadAssociations(siteReports: SiteReport[]) { + const reportIds = siteReports.map(({ id }) => id); + const siteIds = uniq(siteReports.map(({ siteId }) => siteId)); + const sites = await Site.findAll({ + where: { id: siteIds }, + attributes: ["id", "uuid"] + }); + const seedsPlanted = groupBy( + await Seeding.findAll({ + where: { + seedableId: reportIds, + seedableType: SiteReport.LARAVEL_TYPE, + hidden: false + }, + attributes: ["seedableId", "name", "amount"] + }), + "seedableId" + ); + + return siteReports.reduce( + (associations, { id, siteId }) => ({ + ...associations, + [id]: { + siteUuid: sites.find(({ id }) => id === siteId)?.uuid, + totalSeedsPlanted: (seedsPlanted[id] ?? []).reduce((sum, { amount }) => sum + (amount ?? 0), 0) + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/site.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/site.airtable-entity.ts new file mode 100644 index 00000000..24a42a1c --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/site.airtable-entity.ts @@ -0,0 +1,41 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, commonEntityColumns } from "./airtable-entity"; +import { Project, Site } from "@terramatch-microservices/database/entities"; +import { uniq } from "lodash"; + +type SiteAssociations = { + projectUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + ...commonEntityColumns("site"), + "name", + associatedValueColumn("projectUuid", "projectId"), + "status", + "updateRequestStatus", + "sitingStrategy", + "descriptionSitingStrategy" +]; + +export class SiteEntity extends AirtableEntity { + readonly TABLE_NAME = "Sites"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Site; + + protected async loadAssociations(sites: Site[]) { + const projectIds = uniq(sites.map(({ projectId }) => projectId)); + const projects = await Project.findAll({ + where: { id: projectIds }, + attributes: ["id", "uuid"] + }); + + return sites.reduce( + (associations, { id, projectId }) => ({ + ...associations, + [id]: { + projectUuid: projects.find(({ id }) => id === projectId)?.uuid + } + }), + {} as Record + ); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/tree-species.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/tree-species.airtable-entity.ts new file mode 100644 index 00000000..b5390e2d --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/tree-species.airtable-entity.ts @@ -0,0 +1,85 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, PolymorphicUuidAssociation } from "./airtable-entity"; +import { + Nursery, + NurseryReport, + Organisation, + Project, + ProjectPitch, + ProjectReport, + Site, + SiteReport, + TreeSpecies +} from "@terramatch-microservices/database/entities"; + +const LARAVEL_TYPE_MAPPING: Record> = { + [Nursery.LARAVEL_TYPE]: { + association: "nurseryUuid", + model: Nursery + }, + [NurseryReport.LARAVEL_TYPE]: { + association: "nurseryReportUuid", + model: NurseryReport + }, + [Organisation.LARAVEL_TYPE]: { + association: "organisationUuid", + model: Organisation + }, + [ProjectPitch.LARAVEL_TYPE]: { + association: "projectPitchUuid", + model: ProjectPitch + }, + [Project.LARAVEL_TYPE]: { + association: "projectUuid", + model: Project + }, + [ProjectReport.LARAVEL_TYPE]: { + association: "projectReportUuid", + model: ProjectReport + }, + [Site.LARAVEL_TYPE]: { + association: "siteUuid", + model: Site + }, + [SiteReport.LARAVEL_TYPE]: { + association: "siteReportUuid", + model: SiteReport + } +}; + +type TreeSpeciesAssociations = { + nurseryUuid?: string; + nurseryReportUuid?: string; + organisationUuid?: string; + projectPitchUuid?: string; + projectUuid?: string; + projectReportUuid?: string; + siteUuid?: string; + siteReportUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + "uuid", + "name", + "taxonId", + "amount", + "collection", + associatedValueColumn("nurseryUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("nurseryReportUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("organisationUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("projectPitchUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("projectUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("projectReportUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("siteUuid", ["speciesableId", "speciesableType"]), + associatedValueColumn("siteReportUuid", ["speciesableId", "speciesableType"]) +]; + +export class TreeSpeciesEntity extends AirtableEntity { + readonly TABLE_NAME = "Tree Species"; + readonly COLUMNS = COLUMNS; + readonly MODEL = TreeSpecies; + readonly HAS_HIDDEN_FLAG = true; + + protected async loadAssociations(treeSpecies: TreeSpecies[]) { + return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPING, "speciesableType", "speciesableId", treeSpecies); + } +} diff --git a/apps/unified-database-service/src/airtable/entities/workday.airtable-entity.ts b/apps/unified-database-service/src/airtable/entities/workday.airtable-entity.ts new file mode 100644 index 00000000..dfb487a1 --- /dev/null +++ b/apps/unified-database-service/src/airtable/entities/workday.airtable-entity.ts @@ -0,0 +1,37 @@ +import { AirtableEntity, associatedValueColumn, ColumnMapping, PolymorphicUuidAssociation } from "./airtable-entity"; +import { ProjectReport, SiteReport, Workday } from "@terramatch-microservices/database/entities"; + +const LARAVEL_TYPE_MAPPINGS: Record> = { + [ProjectReport.LARAVEL_TYPE]: { + association: "projectReportUuid", + model: ProjectReport + }, + [SiteReport.LARAVEL_TYPE]: { + association: "siteReportUuid", + model: SiteReport + } +}; + +type WorkdayAssociations = { + projectReportUuid?: string; + siteReportUuid?: string; +}; + +const COLUMNS: ColumnMapping[] = [ + "uuid", + "collection", + "description", + associatedValueColumn("projectReportUuid", ["workdayableId", "workdayableType"]), + associatedValueColumn("siteReportUuid", ["workdayableId", "workdayableType"]) +]; + +export class WorkdayEntity extends AirtableEntity { + readonly TABLE_NAME = "Workdays"; + readonly COLUMNS = COLUMNS; + readonly MODEL = Workday; + readonly HAS_HIDDEN_FLAG = true; + + protected async loadAssociations(workdays: Workday[]) { + return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPINGS, "workdayableType", "workdayableId", workdays); + } +} diff --git a/apps/unified-database-service/src/airtable/queue-health.service.ts b/apps/unified-database-service/src/airtable/queue-health.service.ts new file mode 100644 index 00000000..fe70ebfa --- /dev/null +++ b/apps/unified-database-service/src/airtable/queue-health.service.ts @@ -0,0 +1,29 @@ +import { InjectQueue } from "@nestjs/bullmq"; +import { Injectable } from "@nestjs/common"; +import { HealthCheckError, HealthIndicator, HealthIndicatorResult } from "@nestjs/terminus"; +import { Queue } from "bullmq"; + +@Injectable() +export class QueueHealthService extends HealthIndicator { + constructor(@InjectQueue("airtable") private readonly airtableQueue: Queue) { + super(); + } + + public async queueHealthCheck(): Promise { + const isHealthy = (await (await this.airtableQueue.client).ping()) === "PONG"; + if (!isHealthy) { + throw new HealthCheckError("Redis connection for Airtable queue unavailable", {}); + } + + try { + const data = { + waitingJobs: await this.airtableQueue.getWaitingCount(), + totalJobs: await this.airtableQueue.count() + }; + + return this.getStatus("redis-queue:airtable", isHealthy, data); + } catch (error) { + throw new HealthCheckError("Error fetching Airtable queue stats", error); + } + } +} diff --git a/apps/unified-database-service/src/app.module.ts b/apps/unified-database-service/src/app.module.ts index c1c839f6..7e9ae861 100644 --- a/apps/unified-database-service/src/app.module.ts +++ b/apps/unified-database-service/src/app.module.ts @@ -5,9 +5,10 @@ import { AirtableModule } from "./airtable/airtable.module"; import { SentryGlobalFilter, SentryModule } from "@sentry/nestjs/setup"; import { DatabaseModule } from "@terramatch-microservices/database"; import { APP_FILTER } from "@nestjs/core"; +import { ScheduleModule } from "@nestjs/schedule"; @Module({ - imports: [SentryModule.forRoot(), DatabaseModule, HealthModule, AirtableModule], + imports: [SentryModule.forRoot(), ScheduleModule.forRoot(), DatabaseModule, HealthModule, AirtableModule], controllers: [WebhookController], providers: [ { diff --git a/apps/unified-database-service/src/health/health.controller.ts b/apps/unified-database-service/src/health/health.controller.ts index 12ffbbbb..0d57e4c8 100644 --- a/apps/unified-database-service/src/health/health.controller.ts +++ b/apps/unified-database-service/src/health/health.controller.ts @@ -1,29 +1,28 @@ -import { Controller, Get } from '@nestjs/common'; -import { - HealthCheck, - HealthCheckService, - SequelizeHealthIndicator, -} from '@nestjs/terminus'; -import { NoBearerAuth } from '@terramatch-microservices/common/guards'; -import { ApiExcludeController } from '@nestjs/swagger'; -import { User } from '@terramatch-microservices/database/entities'; +import { Controller, Get } from "@nestjs/common"; +import { HealthCheck, HealthCheckService, SequelizeHealthIndicator } from "@nestjs/terminus"; +import { NoBearerAuth } from "@terramatch-microservices/common/guards"; +import { ApiExcludeController } from "@nestjs/swagger"; +import { User } from "@terramatch-microservices/database/entities"; +import { QueueHealthService } from "../airtable/queue-health.service"; -@Controller('health') +@Controller("health") @ApiExcludeController() export class HealthController { constructor( private readonly health: HealthCheckService, - private readonly db: SequelizeHealthIndicator + private readonly db: SequelizeHealthIndicator, + private readonly queue: QueueHealthService ) {} @Get() @HealthCheck() @NoBearerAuth async check() { - const connection = await User.sequelize.connectionManager.getConnection({ type: 'read' }); + const connection = await User.sequelize.connectionManager.getConnection({ type: "read" }); try { return this.health.check([ - () => this.db.pingCheck('database', { connection }), + () => this.db.pingCheck("database", { connection }), + () => this.queue.queueHealthCheck() ]); } finally { User.sequelize.connectionManager.releaseConnection(connection); diff --git a/apps/unified-database-service/src/health/health.module.ts b/apps/unified-database-service/src/health/health.module.ts index 0208ef74..1b7de1b3 100644 --- a/apps/unified-database-service/src/health/health.module.ts +++ b/apps/unified-database-service/src/health/health.module.ts @@ -1,9 +1,10 @@ -import { Module } from '@nestjs/common'; -import { TerminusModule } from '@nestjs/terminus'; -import { HealthController } from './health.controller'; +import { Module } from "@nestjs/common"; +import { TerminusModule } from "@nestjs/terminus"; +import { HealthController } from "./health.controller"; +import { AirtableModule } from "../airtable/airtable.module"; @Module({ - imports: [TerminusModule], - controllers: [HealthController], + imports: [TerminusModule, AirtableModule], + controllers: [HealthController] }) export class HealthModule {} diff --git a/apps/unified-database-service/src/webhook/dto/delete-records-query.dto.ts b/apps/unified-database-service/src/webhook/dto/delete-records-query.dto.ts new file mode 100644 index 00000000..24c446f2 --- /dev/null +++ b/apps/unified-database-service/src/webhook/dto/delete-records-query.dto.ts @@ -0,0 +1,17 @@ +import { ENTITY_TYPES, EntityType } from "../../airtable/airtable.processor"; +import { IsDate, IsIn } from "class-validator"; +import { ApiProperty } from "@nestjs/swagger"; + +export class DeleteRecordsQueryDto { + @IsIn(ENTITY_TYPES) + @ApiProperty({ + enum: ENTITY_TYPES, + description: "Entity type to update in Airtable", + example: "project" + }) + entityType: EntityType; + + @IsDate() + @ApiProperty({ description: "The timestamp from which to look for deleted records" }) + deletedSince: Date; +} diff --git a/apps/unified-database-service/src/webhook/dto/update-all-query.dto.ts b/apps/unified-database-service/src/webhook/dto/update-all-query.dto.ts new file mode 100644 index 00000000..a3997fea --- /dev/null +++ b/apps/unified-database-service/src/webhook/dto/update-all-query.dto.ts @@ -0,0 +1,8 @@ +import { IsDate } from "class-validator"; +import { ApiProperty } from "@nestjs/swagger"; + +export class UpdateAllQueryDto { + @IsDate() + @ApiProperty({ description: "The timestamp from which to look for updated records" }) + updatedSince: Date; +} diff --git a/apps/unified-database-service/src/webhook/dto/update-records-query.dto.ts b/apps/unified-database-service/src/webhook/dto/update-records-query.dto.ts new file mode 100644 index 00000000..0d057d94 --- /dev/null +++ b/apps/unified-database-service/src/webhook/dto/update-records-query.dto.ts @@ -0,0 +1,23 @@ +import { ENTITY_TYPES, EntityType } from "../../airtable/airtable.processor"; +import { IsDate, IsIn, IsInt, IsOptional } from "class-validator"; +import { ApiProperty } from "@nestjs/swagger"; + +export class UpdateRecordsQueryDto { + @IsIn(ENTITY_TYPES) + @ApiProperty({ + enum: ENTITY_TYPES, + description: "Entity type to update in Airtable", + example: "project" + }) + entityType: EntityType; + + @IsInt() + @IsOptional() + @ApiProperty({ description: "The page to start processing on.", required: false }) + startPage?: number; + + @IsDate() + @IsOptional() + @ApiProperty({ description: "The timestamp from which to look for updated records", required: false }) + updatedSince?: Date; +} diff --git a/apps/unified-database-service/src/webhook/webhook.controller.spec.ts b/apps/unified-database-service/src/webhook/webhook.controller.spec.ts new file mode 100644 index 00000000..489f2667 --- /dev/null +++ b/apps/unified-database-service/src/webhook/webhook.controller.spec.ts @@ -0,0 +1,85 @@ +import { WebhookController } from "./webhook.controller"; +import { Test } from "@nestjs/testing"; +import { AirtableService } from "../airtable/airtable.service"; +import { createMock, DeepMocked } from "@golevelup/ts-jest"; +import { UnauthorizedException } from "@nestjs/common"; +import { Permission } from "@terramatch-microservices/database/entities"; + +describe("WebhookController", () => { + let controller: WebhookController; + let service: DeepMocked; + let permissionSpy: jest.SpyInstance, [userId: number]>; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + controllers: [WebhookController], + providers: [{ provide: AirtableService, useValue: (service = createMock()) }] + }).compile(); + + controller = module.get(WebhookController); + permissionSpy = jest.spyOn(Permission, "getUserPermissionNames"); + permissionSpy.mockResolvedValue(["reports-manage"]); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + describe("updateRecords", () => { + it("should throw an error if the user doesn't have the correct permissions", async () => { + permissionSpy.mockResolvedValue([]); + await expect(controller.updateRecords({ entityType: "project" }, { authenticatedUserId: 1 })).rejects.toThrow( + UnauthorizedException + ); + }); + + it("should call into the service with query params", async () => { + const updatedSince = new Date(); + let result = await controller.updateRecords( + { entityType: "project", startPage: 2, updatedSince }, + { authenticatedUserId: 1 } + ); + expect(result).toEqual({ status: "OK" }); + expect(service.updateAirtable).toHaveBeenCalledWith("project", 2, updatedSince); + + result = await controller.updateRecords({ entityType: "site-report" }, { authenticatedUserId: 1 }); + expect(result).toEqual({ status: "OK" }); + expect(service.updateAirtable).toHaveBeenCalledWith("site-report", undefined, undefined); + }); + }); + + describe("removeDeletedRecords", () => { + it("should throw an error if the user doesn't have the correct permissions", async () => { + permissionSpy.mockResolvedValue([]); + await expect( + controller.removeDeletedRecords({ entityType: "project", deletedSince: new Date() }, { authenticatedUserId: 1 }) + ).rejects.toThrow(UnauthorizedException); + }); + + it("should call into the service with query params", async () => { + const deletedSince = new Date(); + const result = await controller.removeDeletedRecords( + { entityType: "project", deletedSince }, + { authenticatedUserId: 1 } + ); + expect(result).toEqual({ status: "OK" }); + expect(service.deleteFromAirtable).toHaveBeenCalledWith("project", deletedSince); + }); + }); + + describe("updateAll", () => { + it("should throw an error if the user doesn't have the correct permissions", async () => { + permissionSpy.mockResolvedValue([]); + await expect(controller.updateAll({ updatedSince: new Date() }, { authenticatedUserId: 1 })).rejects.toThrow( + UnauthorizedException + ); + }); + + it("should call into the service with query params", async () => { + const updatedSince = new Date(); + const result = await controller.updateAll({ updatedSince: updatedSince }, { authenticatedUserId: 1 }); + expect(result).toEqual({ status: "OK" }); + expect(service.updateAll).toHaveBeenCalledWith(updatedSince); + }); + }); +}); diff --git a/apps/unified-database-service/src/webhook/webhook.controller.ts b/apps/unified-database-service/src/webhook/webhook.controller.ts index b7846c6d..e402a5bd 100644 --- a/apps/unified-database-service/src/webhook/webhook.controller.ts +++ b/apps/unified-database-service/src/webhook/webhook.controller.ts @@ -1,24 +1,93 @@ -import { BadRequestException, Controller, Get, Query } from "@nestjs/common"; -import { AirtableService, EntityType } from "../airtable/airtable.service"; -import { NoBearerAuth } from "@terramatch-microservices/common/guards"; +import { + BadRequestException, + Controller, + Get, + HttpStatus, + Query, + Request, + UnauthorizedException +} from "@nestjs/common"; +import { AirtableService } from "../airtable/airtable.service"; +import { ApiOperation, ApiResponse } from "@nestjs/swagger"; +import { ApiException } from "@nanogiants/nestjs-swagger-api-exception-decorator"; +import { UpdateRecordsQueryDto } from "./dto/update-records-query.dto"; +import { Permission } from "@terramatch-microservices/database/entities"; +import { DeleteRecordsQueryDto } from "./dto/delete-records-query.dto"; +import { UpdateAllQueryDto } from "./dto/update-all-query.dto"; @Controller("unified-database/v3/webhook") export class WebhookController { constructor(private readonly airtableService: AirtableService) {} - @Get() - @NoBearerAuth - // TODO (NJC): Documentation if we end up keeping this webhook. - async triggerWebhook(@Query("entityType") entityType: EntityType) { - if (entityType == null) { - throw new BadRequestException("Missing query params"); + private async authorize(userId: number) { + const permissions = await Permission.getUserPermissionNames(userId); + // This isn't a perfect match for what this controller does, but it is close, and all admins have + // this permission, so it's a reasonable way for now to restrict this controller to logged in + // admins. + if (!permissions.includes("reports-manage")) { + throw new UnauthorizedException(); } + } - if (!["project"].includes(entityType)) { - throw new BadRequestException("entityType invalid"); - } + @Get("updateRecords") + @ApiOperation({ + operationId: "triggerAirtableUpdate", + description: "trigger an update of a specific set of records to Airtable" + }) + // This endpoint is not to be consumed by the TM FE and does not conform to our usual JSON API structure + @ApiResponse({ + status: HttpStatus.OK, + schema: { type: "object", properties: { status: { type: "string", example: "OK" } } } + }) + @ApiException(() => UnauthorizedException, { description: "Authorization failed" }) + @ApiException(() => BadRequestException, { description: "Query params were invalid" }) + async updateRecords( + @Query() { entityType, startPage, updatedSince }: UpdateRecordsQueryDto, + @Request() { authenticatedUserId } + ) { + await this.authorize(authenticatedUserId); + await this.airtableService.updateAirtable(entityType, startPage, updatedSince); + + return { status: "OK" }; + } + + @Get("deleteRecords") + @ApiOperation({ + operationId: "triggerAirtableDelete", + description: "trigger a delete of a specific set of soft-deleted records from Airtable" + }) + // This endpoint is not to be consumed by the TM FE and does not conform to our usual JSON API structure + @ApiResponse({ + status: HttpStatus.OK, + schema: { type: "object", properties: { status: { type: "string", example: "OK" } } } + }) + @ApiException(() => UnauthorizedException, { description: "Authorization failed" }) + @ApiException(() => BadRequestException, { description: "Query params were invalid" }) + async removeDeletedRecords( + @Query() { entityType, deletedSince }: DeleteRecordsQueryDto, + @Request() { authenticatedUserId } + ) { + await this.authorize(authenticatedUserId); + await this.airtableService.deleteFromAirtable(entityType, deletedSince); + + return { status: "OK" }; + } - await this.airtableService.updateAirtableJob(entityType); + @Get("updateAll") + @ApiOperation({ + operationId: "triggerAirtableUpdateAll", + description: "trigger a complete update of airtable (changes and deletions for all records)" + }) + // This endpoint is not to be consumed by the TM FE and does not conform to our usual JSON API structure + @ApiResponse({ + status: HttpStatus.OK, + schema: { type: "object", properties: { status: { type: "string", example: "OK" } } } + }) + @ApiException(() => UnauthorizedException, { description: "Authorization failed" }) + @ApiException(() => BadRequestException, { description: "Query params were invalid" }) + async updateAll(@Query() { updatedSince }: UpdateAllQueryDto, @Request() { authenticatedUserId }) { + await this.authorize(authenticatedUserId); + await this.airtableService.updateAll(updatedSince); return { status: "OK" }; } diff --git a/apps/user-service/Dockerfile b/apps/user-service/Dockerfile index 3cf5522a..1afa33c7 100644 --- a/apps/user-service/Dockerfile +++ b/apps/user-service/Dockerfile @@ -8,8 +8,12 @@ RUN npx nx build user-service ${BUILD_FLAG} FROM terramatch-microservices-base:nx-base ARG NODE_ENV +ARG SENTRY_DSN +ARG DEPLOY_ENV WORKDIR /app COPY --from=builder /app/builder ./ ENV NODE_ENV=${NODE_ENV} +ENV SENTRY_DSN=${SENTRY_DSN} +ENV DEPLOY_ENV=${DEPLOY_ENV} CMD ["node", "./dist/apps/user-service/main.js"] diff --git a/cdk/service-stack/lib/service-stack.ts b/cdk/service-stack/lib/service-stack.ts index 773a8a7a..b22fffaa 100644 --- a/cdk/service-stack/lib/service-stack.ts +++ b/cdk/service-stack/lib/service-stack.ts @@ -57,9 +57,12 @@ export class ServiceStack extends Stack { vpc }); + // The staging redis security group has an inconsistent name + const redisSecurityGroup = env === "staging" ? "chache-stage" : `cache-${env}`; const securityGroups = [ SecurityGroup.fromLookupByName(this, "default", "default", vpc), - SecurityGroup.fromLookupByName(this, `db-${env}`, `db-${env}`, vpc) + SecurityGroup.fromLookupByName(this, `db-${env}`, `db-${env}`, vpc), + SecurityGroup.fromLookupByName(this, redisSecurityGroup, redisSecurityGroup, vpc) ]; const privateSubnets = [ PrivateSubnet.fromPrivateSubnetAttributes(this, "eu-west-1a", { diff --git a/instrument-sentry.ts b/instrument-sentry.ts index 8f849d82..bd6da5c8 100644 --- a/instrument-sentry.ts +++ b/instrument-sentry.ts @@ -4,6 +4,7 @@ import { nodeProfilingIntegration } from "@sentry/profiling-node"; if (process.env.SENTRY_DSN != null) { Sentry.init({ dsn: process.env.SENTRY_DSN, + environment: process.env.DEPLOY_ENV, integrations: [nodeProfilingIntegration()], tracesSampleRate: 1.0, profilesSampleRate: 1.0 diff --git a/libs/database/src/lib/constants/entity-selects.ts b/libs/database/src/lib/constants/entity-selects.ts new file mode 100644 index 00000000..fb379a2e --- /dev/null +++ b/libs/database/src/lib/constants/entity-selects.ts @@ -0,0 +1,2 @@ +export const SITING_STRATEGIES = ["concentred", "distributed", "hybrid", "not-applicable"] as const; +export type SitingStrategy = (typeof SITING_STRATEGIES)[number]; diff --git a/libs/database/src/lib/constants/framework.ts b/libs/database/src/lib/constants/framework.ts new file mode 100644 index 00000000..5b48037c --- /dev/null +++ b/libs/database/src/lib/constants/framework.ts @@ -0,0 +1,11 @@ +export const FRAMEWORK_NAMES = { + terrafund: "TerraFund Top 100", + "terrafund-landscapes": "TerraFund Landscapes", + ppc: "Priceless Planet Coalition (PPC)", + enterprises: "TerraFund Enterprises", + hbf: "Harit Bharat Fund", + "epa-ghana-pilot": "EPA-Ghana Pilot" +} as const; + +export const FRAMEWORK_KEYS = Object.keys(FRAMEWORK_NAMES); +export type FrameworkKey = keyof typeof FRAMEWORK_NAMES; diff --git a/libs/database/src/lib/constants/status.ts b/libs/database/src/lib/constants/status.ts new file mode 100644 index 00000000..efaf4a51 --- /dev/null +++ b/libs/database/src/lib/constants/status.ts @@ -0,0 +1,29 @@ +export const STARTED = "started"; +export const AWAITING_APPROVAL = "awaiting-approval"; +export const APPROVED = "approved"; +export const NEEDS_MORE_INFORMATION = "needs-more-information"; +export const ENTITY_STATUSES = [STARTED, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; +export type EntityStatus = (typeof ENTITY_STATUSES)[number]; + +export const DUE = "due"; +export const REPORT_STATUSES = [DUE, ...ENTITY_STATUSES] as const; +export type ReportStatus = (typeof REPORT_STATUSES)[number]; + +export const DRAFT = "draft"; +export const UPDATE_REQUEST_STATUSES = [DRAFT, AWAITING_APPROVAL, APPROVED, NEEDS_MORE_INFORMATION] as const; +export type UpdateRequestStatus = (typeof UPDATE_REQUEST_STATUSES)[number]; + +export const REJECTED = "rejected"; +export const REQUIRES_MORE_INFORMATION = "requires-more-information"; +export const FORM_SUBMISSION_STATUSES = [ + APPROVED, + AWAITING_APPROVAL, + REJECTED, + REQUIRES_MORE_INFORMATION, + STARTED +] as const; +export type FormSubmissionStatus = (typeof FORM_SUBMISSION_STATUSES)[number]; + +export const PENDING = "pending"; +export const ORGANISATION_STATUSES = [APPROVED, PENDING, REJECTED, DRAFT]; +export type OrganisationStatus = (typeof ORGANISATION_STATUSES)[number]; diff --git a/libs/database/src/lib/decorators/json-column.decorator.ts b/libs/database/src/lib/decorators/json-column.decorator.ts new file mode 100644 index 00000000..2728e376 --- /dev/null +++ b/libs/database/src/lib/decorators/json-column.decorator.ts @@ -0,0 +1,20 @@ +import { Column, Model } from "sequelize-typescript"; +import { Attributes, JSON as JSON_TYPE, ModelAttributeColumnOptions } from "sequelize"; + +/** + * Sequelize has a bug where when the data for this model is fetched as part of an include on + * findAll, the JSON value isn't getting deserialized. This decorator should be used for all + * columns that should deserialize text to JSON instead of @Column(JSON) + */ +export const JsonColumn = + >(options: Partial = {}) => + (target: unknown, propertyName: string, propertyDescriptor?: PropertyDescriptor) => + Column({ + ...options, + + type: JSON_TYPE, + get(this: T): object { + const value = this.getDataValue(propertyName as keyof Attributes); + return typeof value === "string" ? JSON.parse(value) : value; + } + })(target, propertyName, propertyDescriptor); diff --git a/libs/database/src/lib/entities/application.entity.ts b/libs/database/src/lib/entities/application.entity.ts index d76d77f5..75b43cd7 100644 --- a/libs/database/src/lib/entities/application.entity.ts +++ b/libs/database/src/lib/entities/application.entity.ts @@ -2,6 +2,7 @@ import { AllowNull, AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, import { BIGINT, UUID } from "sequelize"; import { User } from "./user.entity"; +// Incomplete stub: not all indexes are specified @Table({ tableName: "applications", underscored: true, @@ -23,6 +24,10 @@ export class Application extends Model { @Column(UUID) fundingProgrammeUuid: string | null; + @AllowNull + @Column(UUID) + organisationUuid: string | null; + @ForeignKey(() => User) @Column(BIGINT.UNSIGNED) updatedBy: number | null; diff --git a/libs/database/src/lib/entities/delayed-job.entity.ts b/libs/database/src/lib/entities/delayed-job.entity.ts index bdfad61b..e58e717d 100644 --- a/libs/database/src/lib/entities/delayed-job.entity.ts +++ b/libs/database/src/lib/entities/delayed-job.entity.ts @@ -9,8 +9,9 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, BOOLEAN, INTEGER, JSON, STRING, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, INTEGER, STRING, UUID } from "sequelize"; import { User } from "./user.entity"; +import { JsonColumn } from "../decorators/json-column.decorator"; // holds the definition for members that may exist in a job metadata that this codebase explicitly // references. @@ -37,7 +38,7 @@ export class DelayedJob extends Model { statusCode: number | null; @AllowNull - @Column(JSON) + @JsonColumn() payload: object | null; @AllowNull @@ -65,6 +66,6 @@ export class DelayedJob extends Model { name: string | null; @AllowNull - @Column(JSON) + @JsonColumn() metadata: Metadata | null; } diff --git a/libs/database/src/lib/entities/form-submission.entity.ts b/libs/database/src/lib/entities/form-submission.entity.ts new file mode 100644 index 00000000..f6888d71 --- /dev/null +++ b/libs/database/src/lib/entities/form-submission.entity.ts @@ -0,0 +1,38 @@ +import { + AllowNull, + AutoIncrement, + BelongsTo, + Column, + ForeignKey, + Model, + PrimaryKey, + Table, + Unique +} from "sequelize-typescript"; +import { BIGINT, STRING, UUID } from "sequelize"; +import { Application } from "./application.entity"; +import { FormSubmissionStatus } from "../constants/status"; + +// Incomplete stub +@Table({ tableName: "form_submissions", underscored: true, paranoid: true }) +export class FormSubmission extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique + @Column(UUID) + uuid: string; + + @Column(STRING) + status: FormSubmissionStatus; + + @AllowNull + @ForeignKey(() => Application) + @Column(BIGINT.UNSIGNED) + applicationId: number | null; + + @BelongsTo(() => Application) + application: Application | null; +} diff --git a/libs/database/src/lib/entities/funding-programme.entity.ts b/libs/database/src/lib/entities/funding-programme.entity.ts new file mode 100644 index 00000000..38ed498e --- /dev/null +++ b/libs/database/src/lib/entities/funding-programme.entity.ts @@ -0,0 +1,18 @@ +import { AutoIncrement, Column, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { BIGINT, STRING, UUID } from "sequelize"; + +// Incomplete stub +@Table({ tableName: "funding_programmes", underscored: true, paranoid: true }) +export class FundingProgramme extends Model { + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique + @Column(UUID) + uuid: string; + + @Column(STRING) + name: string; +} diff --git a/libs/database/src/lib/entities/index.ts b/libs/database/src/lib/entities/index.ts index aedbe1dc..a46b745d 100644 --- a/libs/database/src/lib/entities/index.ts +++ b/libs/database/src/lib/entities/index.ts @@ -1,8 +1,10 @@ export * from "./application.entity"; export * from "./delayed-job.entity"; export * from "./demographic.entity"; +export * from "./form-submission.entity"; export * from "./framework.entity"; export * from "./framework-user.entity"; +export * from "./funding-programme.entity"; export * from "./indicator-output-field-monitoring.entity"; export * from "./indicator-output-hectares.entity"; export * from "./indicator-output-msu-carbon.entity"; @@ -18,8 +20,10 @@ export * from "./permission.entity"; export * from "./point-geometry.entity"; export * from "./polygon-geometry.entity"; export * from "./project.entity"; +export * from "./project-pitch.entity"; export * from "./project-report.entity"; export * from "./project-user.entity"; +export * from "./restoration-partner.entity"; export * from "./role.entity"; export * from "./seeding.entity"; export * from "./site.entity"; diff --git a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts index c463fe51..47aa594a 100644 --- a/libs/database/src/lib/entities/indicator-output-hectares.entity.ts +++ b/libs/database/src/lib/entities/indicator-output-hectares.entity.ts @@ -1,7 +1,8 @@ import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, INTEGER, JSON as JSON_TYPE, STRING } from "sequelize"; +import { BIGINT, INTEGER, STRING } from "sequelize"; import { SitePolygon } from "./site-polygon.entity"; import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; +import { JsonColumn } from "../decorators/json-column.decorator"; @Table({ tableName: "indicator_output_hectares", underscored: true, paranoid: true }) export class IndicatorOutputHectares extends Model { @@ -23,14 +24,6 @@ export class IndicatorOutputHectares extends Model { @Column(INTEGER) yearOfAnalysis: number; - @Column({ - type: JSON_TYPE, - // Sequelize has a bug where when the data for this model is fetched as part of an include on - // findAll, the JSON value isn't getting deserialized. - get(this: IndicatorOutputHectares): object { - const value = this.getDataValue("value"); - return typeof value === "string" ? JSON.parse(value) : value; - } - }) + @JsonColumn() value: object; } diff --git a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts index 24d5012a..462a7907 100644 --- a/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts +++ b/libs/database/src/lib/entities/indicator-output-tree-cover-loss.entity.ts @@ -1,7 +1,8 @@ import { AutoIncrement, Column, ForeignKey, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; -import { BIGINT, INTEGER, JSON as JSON_TYPE, STRING } from "sequelize"; +import { BIGINT, INTEGER, STRING } from "sequelize"; import { SitePolygon } from "./site-polygon.entity"; import { INDICATOR_SLUGS, IndicatorSlug } from "../constants"; +import { JsonColumn } from "../decorators/json-column.decorator"; @Table({ tableName: "indicator_output_tree_cover_loss", underscored: true, paranoid: true }) export class IndicatorOutputTreeCoverLoss extends Model { @@ -23,14 +24,6 @@ export class IndicatorOutputTreeCoverLoss extends Model { static readonly TREE_ASSOCIATIONS = ["seedlings"]; + static readonly APPROVED_STATUSES = ["approved"]; static readonly PARENT_ID = "nurseryId"; static readonly LARAVEL_TYPE = "App\\Models\\V2\\Nurseries\\NurseryReport"; @@ -30,6 +33,10 @@ export class NurseryReport extends Model { @Column(UUID) uuid: string; + @AllowNull + @Column(STRING) + frameworkKey: FrameworkKey | null; + @ForeignKey(() => Nursery) @Column(BIGINT.UNSIGNED) nurseryId: number; @@ -37,10 +44,26 @@ export class NurseryReport extends Model { @BelongsTo(() => Nursery) nursery: Nursery | null; + // TODO foreign key for task + @AllowNull + @Column(BIGINT.UNSIGNED) + taskId: number; + + @Column(STRING) + status: ReportStatus; + + @AllowNull + @Column(STRING) + updateRequestStatus: UpdateRequestStatus | null; + @AllowNull @Column(DATE) dueAt: Date | null; + @AllowNull + @Column(INTEGER.UNSIGNED) + seedlingsYoungTrees: number | null; + @HasMany(() => TreeSpecies, { foreignKey: "speciesableId", constraints: false, diff --git a/libs/database/src/lib/entities/nursery.entity.ts b/libs/database/src/lib/entities/nursery.entity.ts index 07b60c4b..6a893dbd 100644 --- a/libs/database/src/lib/entities/nursery.entity.ts +++ b/libs/database/src/lib/entities/nursery.entity.ts @@ -1,4 +1,5 @@ import { + AllowNull, AutoIncrement, BelongsTo, Column, @@ -9,14 +10,16 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, UUID } from "sequelize"; +import { BIGINT, STRING, UUID } from "sequelize"; import { Project } from "./project.entity"; import { TreeSpecies } from "./tree-species.entity"; import { NurseryReport } from "./nursery-report.entity"; +import { EntityStatus, UpdateRequestStatus } from "../constants/status"; -// A quick stub for the tree service endpoints. +// Incomplete stub @Table({ tableName: "v2_nurseries", underscored: true, paranoid: true }) export class Nursery extends Model { + static readonly APPROVED_STATUSES = ["approved"]; static readonly TREE_ASSOCIATIONS = ["seedlings"]; static readonly LARAVEL_TYPE = "App\\Models\\V2\\Nurseries\\Nursery"; @@ -29,6 +32,17 @@ export class Nursery extends Model { @Column(UUID) uuid: string; + @Column(STRING) + status: EntityStatus; + + @AllowNull + @Column(STRING) + updateRequestStatus: UpdateRequestStatus | null; + + @AllowNull + @Column(STRING) + name: string | null; + @ForeignKey(() => Project) @Column(BIGINT.UNSIGNED) projectId: number; diff --git a/libs/database/src/lib/entities/organisation.entity.ts b/libs/database/src/lib/entities/organisation.entity.ts index 051e7e6e..3ff173a6 100644 --- a/libs/database/src/lib/entities/organisation.entity.ts +++ b/libs/database/src/lib/entities/organisation.entity.ts @@ -1,8 +1,12 @@ import { AllowNull, AutoIncrement, Column, Default, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { JsonColumn } from "../decorators/json-column.decorator"; +import { OrganisationStatus } from "../constants/status"; @Table({ tableName: "organisations", underscored: true, paranoid: true }) export class Organisation extends Model { + static readonly LARAVEL_TYPE = "App\\Models\\V2\\Organisation"; + @PrimaryKey @AutoIncrement @Column(BIGINT.UNSIGNED) @@ -14,7 +18,7 @@ export class Organisation extends Model { @Default("draft") @Column(STRING) - status: string; + status: OrganisationStatus; @AllowNull @Column(STRING) @@ -73,12 +77,12 @@ export class Organisation extends Model { description: string | null; @AllowNull - @Column(TEXT) - countries: string | null; + @JsonColumn() + countries: string[] | null; @AllowNull - @Column(TEXT) - languages: string | null; + @JsonColumn() + languages: string[] | null; @AllowNull @Column(TEXT) @@ -221,8 +225,8 @@ export class Organisation extends Model { restoredAreasDescription: string | null; @AllowNull - @Column(TEXT) - restorationTypesImplemented: string | null; + @JsonColumn() + restorationTypesImplemented: string[] | null; @AllowNull @Column(TEXT) @@ -237,24 +241,24 @@ export class Organisation extends Model { fundingHistory: string | null; @AllowNull - @Column(TEXT) - engagementFarmers: string | null; + @JsonColumn() + engagementFarmers: string[] | null; @AllowNull - @Column(TEXT) - engagementWomen: string | null; + @JsonColumn() + engagementWomen: string[] | null; @AllowNull - @Column(TEXT) - engagementYouth: string | null; + @JsonColumn() + engagementYouth: string[] | null; @Default("usd") @Column(STRING) currency: string; @AllowNull - @Column(TEXT) - states: string | null; + @JsonColumn() + states: string[] | null; @AllowNull @Column(TEXT) @@ -273,8 +277,8 @@ export class Organisation extends Model { loanStatusAmount: string | null; @AllowNull - @Column(TEXT) - loanStatusTypes: string | null; + @JsonColumn() + loanStatusTypes: string[] | null; @AllowNull @Column(TEXT) @@ -285,16 +289,16 @@ export class Organisation extends Model { communityEngagementNumbersMarginalized: string | null; @AllowNull - @Column(TEXT) - landSystems: string | null; + @JsonColumn() + landSystems: string[] | null; @AllowNull - @Column(TEXT) - fundUtilisation: string | null; + @JsonColumn() + fundUtilisation: string[] | null; @AllowNull - @Column(TEXT) - detailedInterventionTypes: string | null; + @JsonColumn() + detailedInterventionTypes: string[] | null; @AllowNull @Column({ type: INTEGER({ length: 11 }), field: "community_members_engaged_3yr" }) @@ -349,8 +353,8 @@ export class Organisation extends Model { engagementNonYouth: string | null; @AllowNull - @Column(TEXT) - treeRestorationPractices: string | null; + @JsonColumn() + treeRestorationPractices: string[] | null; @AllowNull @Column(TEXT) @@ -393,12 +397,12 @@ export class Organisation extends Model { boardEngagementFpcCompany: string | null; @AllowNull - @Column(STRING) - biodiversityFocus: string | null; + @JsonColumn() + biodiversityFocus: string[] | null; @AllowNull - @Column(TEXT) - globalPlanningFrameworks: string | null; + @JsonColumn() + globalPlanningFrameworks: string[] | null; @AllowNull @Column(TEXT) diff --git a/libs/database/src/lib/entities/project-pitch.entity.ts b/libs/database/src/lib/entities/project-pitch.entity.ts new file mode 100644 index 00000000..5141ae20 --- /dev/null +++ b/libs/database/src/lib/entities/project-pitch.entity.ts @@ -0,0 +1,17 @@ +import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript"; +import { BIGINT, UUID } from "sequelize"; + +// Incomplete stub +@Table({ tableName: "project_pitches", underscored: true, paranoid: true }) +export class ProjectPitch extends Model { + static readonly LARAVEL_TYPE = "App\\Models\\V2\\ProjectPitch"; + + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Index + @Column(UUID) + uuid: string; +} diff --git a/libs/database/src/lib/entities/project-report.entity.ts b/libs/database/src/lib/entities/project-report.entity.ts index 7a26cbaa..6951d612 100644 --- a/libs/database/src/lib/entities/project-report.entity.ts +++ b/libs/database/src/lib/entities/project-report.entity.ts @@ -10,17 +10,50 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, DATE, INTEGER, STRING, UUID } from "sequelize"; +import { BIGINT, DATE, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Project } from "./project.entity"; +import { FrameworkKey } from "../constants/framework"; -// A quick stub for the tree endpoints +// Incomplete stub @Table({ tableName: "v2_project_reports", underscored: true, paranoid: true }) export class ProjectReport extends Model { static readonly TREE_ASSOCIATIONS = ["treesPlanted"]; static readonly PARENT_ID = "projectId"; static readonly APPROVED_STATUSES = ["approved"]; static readonly LARAVEL_TYPE = "App\\Models\\V2\\Projects\\ProjectReport"; + static readonly WORKDAY_COLLECTIONS = [ + "paid-nursery-operations", + "paid-project-management", + "paid-other-activities", + "volunteer-nursery-operations", + "volunteer-project-management", + "volunteer-other-activities", + "direct", + "convergence" + ]; + static readonly RESTORATION_PARTNER_COLLECTIONS = [ + "direct-income", + "indirect-income", + "direct-benefits", + "indirect-benefits", + "direct-conservation-payments", + "indirect-conservation-payments", + "direct-market-access", + "indirect-market-access", + "direct-capacity", + "indirect-capacity", + "direct-training", + "indirect-training", + "direct-land-title", + "indirect-land-title", + "direct-livelihoods", + "indirect-livelihoods", + "direct-productivity", + "indirect-productivity", + "direct-other", + "indirect-other" + ]; @PrimaryKey @AutoIncrement @@ -31,6 +64,10 @@ export class ProjectReport extends Model { @Column(UUID) uuid: string; + @AllowNull + @Column(STRING) + frameworkKey: FrameworkKey | null; + @ForeignKey(() => Project) @Column(BIGINT.UNSIGNED) projectId: number; @@ -38,13 +75,62 @@ export class ProjectReport extends Model { @BelongsTo(() => Project) project: Project | null; + // TODO foreign key for task + @AllowNull + @Column(BIGINT.UNSIGNED) + taskId: number; + @Column(STRING) status: string; + @AllowNull + @Column(STRING) + updateRequestStatus: string; + @AllowNull @Column(DATE) dueAt: Date | null; + @AllowNull + @Column(TEXT) + landscapeCommunityContribution: string | null; + + @AllowNull + @Column(TEXT) + topThreeSuccesses: string | null; + + @AllowNull + @Column(TEXT) + challengesFaced: string | null; + + @AllowNull + @Column(TEXT) + lessonsLearned: string | null; + + @AllowNull + @Column(TEXT) + maintenanceAndMonitoringActivities: string | null; + + @AllowNull + @Column(TEXT) + significantChange: string | null; + + @AllowNull + @Column(TINYINT.UNSIGNED) + pctSurvivalToDate: number | null; + + @AllowNull + @Column(TEXT) + survivalCalculation: string | null; + + @AllowNull + @Column(TEXT) + survivalComparison: string | null; + + @AllowNull + @Column(TEXT) + newJobsDescription: string | null; + @AllowNull @Column(INTEGER({ unsigned: true, length: 10 })) ftTotal: number | null; @@ -57,6 +143,10 @@ export class ProjectReport extends Model { @Column(INTEGER({ unsigned: true, length: 10 })) ftMen: number | null; + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + ftOther: number | null; + @AllowNull // There is also an `ft_jobs_youth` field, but it appears to be unused. @Column(INTEGER({ unsigned: true, length: 10 })) @@ -86,6 +176,10 @@ export class ProjectReport extends Model { @Column(INTEGER({ unsigned: true, length: 10 })) ptNonYouth: number | null; + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + ptOther: number | null; + @AllowNull @Column(INTEGER({ unsigned: true, length: 10 })) volunteerTotal: number | null; @@ -106,6 +200,166 @@ export class ProjectReport extends Model { @Column(INTEGER({ unsigned: true, length: 10 })) volunteerNonYouth: number | null; + @AllowNull + @Column(TEXT) + volunteersWorkDescription: string | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + volunteerOther: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiaries: number | null; + + @AllowNull + @Column(TEXT) + beneficiariesDescription: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesWomen: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesLargeScale: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesSmallholder: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesNonYouth: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesYouth: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesMen: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesOther: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesTrainingWomen: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesTrainingMen: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesTrainingOther: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesTrainingYouth: number | null; + + @AllowNull + @Column(INTEGER({ unsigned: true, length: 10 })) + beneficiariesTrainingNonYouth: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesIncomeIncrease: number | null; + + @AllowNull + @Column(TEXT) + beneficiariesIncomeIncreaseDescription: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesSkillsKnowledgeIncrease: number | null; + + @AllowNull + @Column(TEXT) + beneficiariesSkillsKnowledgeIncreaseDescription: string | null; + + @AllowNull + @Column(INTEGER) + indirectBeneficiaries: number | null; + + @AllowNull + @Column(TEXT) + indirectBeneficiariesDescription: string | null; + + @AllowNull + @Column(TEXT) + sharedDriveLink: string | null; + + @AllowNull + @Column(TEXT) + communityProgress: string | null; + + @AllowNull + @Column(TEXT) + localEngagementDescription: string | null; + + @AllowNull + @Column(TEXT) + equitableOpportunities: string | null; + + @AllowNull + @Column(TEXT) + resilienceProgress: string | null; + + @AllowNull + @Column(TEXT) + localGovernance: string | null; + + @AllowNull + @Column(TEXT) + adaptiveManagement: string | null; + + @AllowNull + @Column(TEXT) + scalabilityReplicability: string | null; + + @AllowNull + @Column(TEXT) + convergenceJobsDescription: string | null; + + @AllowNull + @Column(TEXT) + convergenceSchemes: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + volunteerScstobc: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesScstobc: number | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + beneficiariesScstobcFarmers: number | null; + + @AllowNull + @Column(TEXT) + communityPartnersAssetsDescription: string | null; + + @AllowNull + @Column(INTEGER) + peopleKnowledgeSkillsIncreased: number | null; + + @AllowNull + @Column(TEXT) + technicalNarrative: string | null; + + @AllowNull + @Column(TEXT) + publicNarrative: string | null; + + @AllowNull + @Column(INTEGER.UNSIGNED) + totalUniqueRestorationPartners: number | null; + @HasMany(() => TreeSpecies, { foreignKey: "speciesableId", constraints: false, diff --git a/libs/database/src/lib/entities/project.entity.ts b/libs/database/src/lib/entities/project.entity.ts index e064ab50..2c6b4abf 100644 --- a/libs/database/src/lib/entities/project.entity.ts +++ b/libs/database/src/lib/entities/project.entity.ts @@ -11,13 +11,15 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { JSON, BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; +import { BIGINT, BOOLEAN, DATE, DECIMAL, ENUM, INTEGER, STRING, TEXT, TINYINT, UUID } from "sequelize"; import { Organisation } from "./organisation.entity"; import { TreeSpecies } from "./tree-species.entity"; import { ProjectReport } from "./project-report.entity"; import { Application } from "./application.entity"; import { Site } from "./site.entity"; import { Nursery } from "./nursery.entity"; +import { JsonColumn } from "../decorators/json-column.decorator"; +import { FrameworkKey } from "../constants/framework"; @Table({ tableName: "v2_projects", underscored: true, paranoid: true }) export class Project extends Model { @@ -35,7 +37,7 @@ export class Project extends Model { @AllowNull @Column(STRING) - frameworkKey: string | null; + frameworkKey: FrameworkKey | null; @Default(false) @Column(BOOLEAN) @@ -81,11 +83,11 @@ export class Project extends Model { boundaryGeojson: string | null; @AllowNull - @Column(JSON) + @JsonColumn() landUseTypes: string[] | null; @AllowNull - @Column(JSON) + @JsonColumn() restorationStrategy: string[] | null; @AllowNull @@ -193,7 +195,7 @@ export class Project extends Model { sitingStrategy: string | null; @AllowNull - @Column(JSON) + @JsonColumn() landTenureProjectArea: string[] | null; @AllowNull @@ -253,8 +255,8 @@ export class Project extends Model { pctBeneficiariesYouth: number | null; @AllowNull - @Column(TEXT) - detailedInterventionTypes: string | null; + @JsonColumn() + detailedInterventionTypes: string[] | null; @AllowNull @Column(TEXT) @@ -285,8 +287,8 @@ export class Project extends Model { projBoundary: string | null; @AllowNull - @Column(TEXT) - states: string | null; + @JsonColumn() + states: string[] | null; @AllowNull @Column(TEXT) diff --git a/libs/database/src/lib/entities/restoration-partner.entity.ts b/libs/database/src/lib/entities/restoration-partner.entity.ts new file mode 100644 index 00000000..61710887 --- /dev/null +++ b/libs/database/src/lib/entities/restoration-partner.entity.ts @@ -0,0 +1,49 @@ +import { AllowNull, AutoIncrement, Column, HasMany, Model, PrimaryKey, Table, Unique } from "sequelize-typescript"; +import { Demographic } from "./demographic.entity"; +import { BIGINT, BOOLEAN, STRING, TEXT, UUID } from "sequelize"; + +@Table({ + tableName: "restoration_partners", + underscored: true, + paranoid: true, + indexes: [ + // @Index doesn't work with underscored column names + { name: "partner_morph_index", fields: ["partnerable_id", "partnerable_type"] } + ] +}) +export class RestorationPartner extends Model { + static readonly LARAVEL_TYPE = "App\\Models\\V2\\RestorationPartners\\RestorationPartner"; + + @PrimaryKey + @AutoIncrement + @Column(BIGINT.UNSIGNED) + override id: number; + + @Unique + @Column(UUID) + uuid: string; + + @AllowNull + @Column(STRING) + collection: string | null; + + @Column(STRING) + partnerableType: string; + + @Column(BIGINT.UNSIGNED) + partnerableId: number; + + @AllowNull + @Column(TEXT) + description: string; + + @Column({ type: BOOLEAN, defaultValue: false }) + hidden: boolean; + + @HasMany(() => Demographic, { + foreignKey: "demographicalId", + constraints: false, + scope: { demographicalType: RestorationPartner.LARAVEL_TYPE } + }) + demographics: Demographic[] | null; +} diff --git a/libs/database/src/lib/entities/site-report.entity.ts b/libs/database/src/lib/entities/site-report.entity.ts index 3b266795..bd1e9faa 100644 --- a/libs/database/src/lib/entities/site-report.entity.ts +++ b/libs/database/src/lib/entities/site-report.entity.ts @@ -10,10 +10,11 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, DATE, STRING, UUID } from "sequelize"; +import { BIGINT, DATE, INTEGER, STRING, TEXT, UUID } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { Site } from "./site.entity"; import { Seeding } from "./seeding.entity"; +import { FrameworkKey } from "../constants/framework"; // A quick stub for the research endpoints @Table({ tableName: "v2_site_reports", underscored: true, paranoid: true }) @@ -22,6 +23,18 @@ export class SiteReport extends Model { static readonly PARENT_ID = "siteId"; static readonly APPROVED_STATUSES = ["approved"]; static readonly LARAVEL_TYPE = "App\\Models\\V2\\Sites\\SiteReport"; + static readonly WORKDAY_COLLECTIONS = [ + "paid-site-establishment", + "paid-planting", + "paid-site-maintenance", + "paid-site-monitoring", + "paid-other-activities", + "volunteer-site-establishment", + "volunteer-planting", + "volunteer-site-maintenance", + "volunteer-site-monitoring", + "volunteer-other-activities" + ]; @PrimaryKey @AutoIncrement @@ -32,6 +45,10 @@ export class SiteReport extends Model { @Column(UUID) uuid: string; + @AllowNull + @Column(STRING) + frameworkKey: FrameworkKey | null; + @ForeignKey(() => Site) @Column(BIGINT.UNSIGNED) siteId: number; @@ -39,9 +56,18 @@ export class SiteReport extends Model { @BelongsTo(() => Site) site: Site | null; + // TODO foreign key for task + @AllowNull + @Column(BIGINT.UNSIGNED) + taskId: number; + @Column(STRING) status: string; + @AllowNull + @Column(STRING) + updateRequestStatus: string; + @AllowNull @Column(DATE) dueAt: Date | null; @@ -50,6 +76,46 @@ export class SiteReport extends Model { @Column(DATE) submittedAt: Date | null; + @AllowNull + @Column(INTEGER.UNSIGNED) + pctSurvivalToDate: number | null; + + @AllowNull + @Column(TEXT) + survivalCalculation: string | null; + + @AllowNull + @Column(TEXT) + survivalDescription: string | null; + + @AllowNull + @Column(TEXT) + maintenanceActivities: string | null; + + @AllowNull + @Column(TEXT) + regenerationDescription: string | null; + + @AllowNull + @Column(TEXT) + technicalNarrative: string | null; + + @AllowNull + @Column(TEXT) + publicNarrative: string | null; + + @AllowNull + @Column(INTEGER) + numTreesRegenerating: number | null; + + @AllowNull + @Column(TEXT) + soilWaterRestorationDescription: string | null; + + @AllowNull + @Column(TEXT) + waterStructures: string | null; + @HasMany(() => TreeSpecies, { foreignKey: "speciesableId", constraints: false, diff --git a/libs/database/src/lib/entities/site.entity.ts b/libs/database/src/lib/entities/site.entity.ts index ec399b9a..d643a061 100644 --- a/libs/database/src/lib/entities/site.entity.ts +++ b/libs/database/src/lib/entities/site.entity.ts @@ -1,4 +1,5 @@ import { + AllowNull, AutoIncrement, BelongsTo, Column, @@ -9,13 +10,16 @@ import { PrimaryKey, Table } from "sequelize-typescript"; -import { BIGINT, STRING, UUID } from "sequelize"; +import { BIGINT, STRING, TEXT, UUID } from "sequelize"; import { TreeSpecies } from "./tree-species.entity"; import { SiteReport } from "./site-report.entity"; import { Project } from "./project.entity"; import { SitePolygon } from "./site-polygon.entity"; +import { EntityStatus, UpdateRequestStatus } from "../constants/status"; +import { SitingStrategy } from "../constants/entity-selects"; +import { Seeding } from "./seeding.entity"; -// A quick stub for the research endpoints +// Incomplete stub @Table({ tableName: "v2_sites", underscored: true, paranoid: true }) export class Site extends Model { static readonly TREE_ASSOCIATIONS = ["treesPlanted", "nonTrees"]; @@ -31,7 +35,11 @@ export class Site extends Model { name: string; @Column(STRING) - status: string; + status: EntityStatus; + + @AllowNull + @Column(STRING) + updateRequestStatus: UpdateRequestStatus | null; @Index @Column(UUID) @@ -44,6 +52,14 @@ export class Site extends Model { @BelongsTo(() => Project) project: Project | null; + @AllowNull + @Column(STRING) + sitingStrategy: SitingStrategy | null; + + @AllowNull + @Column(TEXT) + descriptionSitingStrategy: string | null; + @HasMany(() => TreeSpecies, { foreignKey: "speciesableId", constraints: false, @@ -68,6 +84,18 @@ export class Site extends Model { return this.nonTrees; } + @HasMany(() => Seeding, { + foreignKey: "seedableId", + constraints: false, + scope: { seedableType: Site.LARAVEL_TYPE } + }) + seedsPlanted: Seeding[] | null; + + async loadSeedsPlanted() { + this.seedsPlanted ??= await this.$get("seedsPlanted"); + return this.seedsPlanted; + } + @HasMany(() => SiteReport) reports: SiteReport[] | null; diff --git a/libs/database/src/lib/entities/workday.entity.ts b/libs/database/src/lib/entities/workday.entity.ts index e4ed0892..9562abb5 100644 --- a/libs/database/src/lib/entities/workday.entity.ts +++ b/libs/database/src/lib/entities/workday.entity.ts @@ -23,6 +23,10 @@ export class Workday extends Model { @Column(UUID) uuid: string; + @AllowNull + @Column(STRING) + collection: string | null; + @Column(STRING) workdayableType: string; diff --git a/libs/database/src/lib/factories/application.factory.ts b/libs/database/src/lib/factories/application.factory.ts new file mode 100644 index 00000000..a39ec929 --- /dev/null +++ b/libs/database/src/lib/factories/application.factory.ts @@ -0,0 +1,10 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { Application } from "../entities"; +import { OrganisationFactory } from "./organisation.factory"; +import { FundingProgrammeFactory } from "./funding-programme.factory"; + +export const ApplicationFactory = FactoryGirl.define(Application, async () => ({ + uuid: crypto.randomUUID(), + organisationUuid: OrganisationFactory.associate("uuid"), + fundingProgrammeUuid: FundingProgrammeFactory.associate("uuid") +})); diff --git a/libs/database/src/lib/factories/demographic.factory.ts b/libs/database/src/lib/factories/demographic.factory.ts new file mode 100644 index 00000000..c60150d2 --- /dev/null +++ b/libs/database/src/lib/factories/demographic.factory.ts @@ -0,0 +1,44 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { Demographic, RestorationPartner, Workday } from "../entities"; +import { WorkdayFactory } from "./workday.factory"; +import { faker } from "@faker-js/faker"; +import { RestorationPartnerFactory } from "./restoration-partner.factory"; + +const TYPES = ["gender", "age", "ethnicity", "caste"]; +const NAMES: Record = { + gender: ["male", "female", "non-binary"], + age: ["youth", "adult", "elder"], + ethnicity: ["Cuyono", "Tupiniquim", "Visaya"], + caste: ["marginalized"] +}; +const SUBTYPES: Record = { + gender: [null], + age: [null], + ethnicity: ["indigenous", "unknown", "other"], + caste: [null] +}; + +const defaultAttributesFactory = async () => { + const type = faker.helpers.arrayElement(TYPES); + return { + uuid: crypto.randomUUID(), + amount: faker.number.int({ min: 10, max: 100 }), + type, + subtype: faker.helpers.arrayElement(SUBTYPES[type] ?? [null]), + name: faker.helpers.arrayElement(NAMES[type] ?? [null]) + }; +}; + +export const DemographicFactory = { + forWorkday: FactoryGirl.define(Demographic, async () => ({ + ...(await defaultAttributesFactory()), + demographicalType: Workday.LARAVEL_TYPE, + demographicalId: WorkdayFactory.forProjectReport.associate("id") + })), + + forRestorationPartner: FactoryGirl.define(Demographic, async () => ({ + ...(await defaultAttributesFactory()), + demographicalType: RestorationPartner.LARAVEL_TYPE, + demographicalId: RestorationPartnerFactory.forProjectReport.associate("id") + })) +}; diff --git a/libs/database/src/lib/factories/form-submission.factory.ts b/libs/database/src/lib/factories/form-submission.factory.ts new file mode 100644 index 00000000..dfb3b2b4 --- /dev/null +++ b/libs/database/src/lib/factories/form-submission.factory.ts @@ -0,0 +1,12 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { FormSubmission } from "../entities"; +import { ApplicationFactory } from "./application.factory"; +import { FORM_SUBMISSION_STATUSES } from "../constants/status"; + +export const FormSubmissionFactory = FactoryGirl.define(FormSubmission, async () => ({ + uuid: crypto.randomUUID(), + applicationId: ApplicationFactory.associate("id"), + name: faker.animal.petName(), + status: faker.helpers.arrayElement(FORM_SUBMISSION_STATUSES) +})); diff --git a/libs/database/src/lib/factories/funding-programme.factory.ts b/libs/database/src/lib/factories/funding-programme.factory.ts new file mode 100644 index 00000000..e2243b5a --- /dev/null +++ b/libs/database/src/lib/factories/funding-programme.factory.ts @@ -0,0 +1,8 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { FundingProgramme } from "../entities"; +import { faker } from "@faker-js/faker"; + +export const FundingProgrammeFactory = FactoryGirl.define(FundingProgramme, async () => ({ + uuid: crypto.randomUUID(), + name: faker.animal.petName() +})); diff --git a/libs/database/src/lib/factories/index.ts b/libs/database/src/lib/factories/index.ts index 56931f85..3b9252b2 100644 --- a/libs/database/src/lib/factories/index.ts +++ b/libs/database/src/lib/factories/index.ts @@ -1,4 +1,8 @@ +export * from "./application.factory"; export * from "./delayed-job.factory"; +export * from "./demographic.factory"; +export * from "./form-submission.factory"; +export * from "./funding-programme.factory"; export * from "./indicator-output-field-monitoring.factory"; export * from "./indicator-output-hectares.factory"; export * from "./indicator-output-msu-carbon.factory"; @@ -11,9 +15,12 @@ export * from "./organisation.factory"; export * from "./polygon-geometry.factory"; export * from "./project.factory"; export * from "./project-report.factory"; +export * from "./restoration-partner.factory"; +export * from "./seeding.factory"; export * from "./site.factory"; export * from "./site-polygon.factory"; export * from "./site-report.factory"; export * from "./tree-species.factory"; export * from "./tree-species-research.factory"; export * from "./user.factory"; +export * from "./workday.factory"; diff --git a/libs/database/src/lib/factories/nursery-report.factory.ts b/libs/database/src/lib/factories/nursery-report.factory.ts index 458354ec..bd40ced9 100644 --- a/libs/database/src/lib/factories/nursery-report.factory.ts +++ b/libs/database/src/lib/factories/nursery-report.factory.ts @@ -3,13 +3,17 @@ import { NurseryReport } from "../entities"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; import { NurseryFactory } from "./nursery.factory"; +import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; export const NurseryReportFactory = FactoryGirl.define(NurseryReport, async () => { const dueAt = faker.date.past({ years: 2 }); + dueAt.setMilliseconds(0); return { uuid: crypto.randomUUID(), nurseryId: NurseryFactory.associate("id"), dueAt, - submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }) + submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), + status: faker.helpers.arrayElement(REPORT_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) }; }); diff --git a/libs/database/src/lib/factories/nursery.factory.ts b/libs/database/src/lib/factories/nursery.factory.ts index d8c5d1fc..58eaeb89 100644 --- a/libs/database/src/lib/factories/nursery.factory.ts +++ b/libs/database/src/lib/factories/nursery.factory.ts @@ -1,8 +1,13 @@ import { FactoryGirl } from "factory-girl-ts"; import { Nursery } from "../entities"; import { ProjectFactory } from "./project.factory"; +import { faker } from "@faker-js/faker"; +import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; export const NurseryFactory = FactoryGirl.define(Nursery, async () => ({ uuid: crypto.randomUUID(), - projectId: ProjectFactory.associate("id") + projectId: ProjectFactory.associate("id"), + name: faker.animal.petName(), + status: faker.helpers.arrayElement(ENTITY_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) })); diff --git a/libs/database/src/lib/factories/organisation.factory.ts b/libs/database/src/lib/factories/organisation.factory.ts index ab846ac8..adf0beeb 100644 --- a/libs/database/src/lib/factories/organisation.factory.ts +++ b/libs/database/src/lib/factories/organisation.factory.ts @@ -1,10 +1,11 @@ -import { Organisation } from '../entities'; -import { FactoryGirl } from 'factory-girl-ts'; -import { faker } from '@faker-js/faker'; +import { Organisation } from "../entities"; +import { FactoryGirl } from "factory-girl-ts"; +import { faker } from "@faker-js/faker"; +import { ORGANISATION_STATUSES } from "../constants/status"; export const OrganisationFactory = FactoryGirl.define(Organisation, async () => ({ uuid: crypto.randomUUID(), - status: 'approved', - type: 'non-profit-organisation', + status: faker.helpers.arrayElement(ORGANISATION_STATUSES), + type: "non-profit-organisation", name: faker.company.name() -})) +})); diff --git a/libs/database/src/lib/factories/project-report.factory.ts b/libs/database/src/lib/factories/project-report.factory.ts index 05b2e519..72c80156 100644 --- a/libs/database/src/lib/factories/project-report.factory.ts +++ b/libs/database/src/lib/factories/project-report.factory.ts @@ -3,13 +3,17 @@ import { ProjectReport } from "../entities"; import { faker } from "@faker-js/faker"; import { ProjectFactory } from "./project.factory"; import { DateTime } from "luxon"; +import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; export const ProjectReportFactory = FactoryGirl.define(ProjectReport, async () => { const dueAt = faker.date.past({ years: 2 }); + dueAt.setMilliseconds(0); return { uuid: crypto.randomUUID(), projectId: ProjectFactory.associate("id"), dueAt, - submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }) + submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), + status: faker.helpers.arrayElement(REPORT_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) }; }); diff --git a/libs/database/src/lib/factories/project.factory.ts b/libs/database/src/lib/factories/project.factory.ts index a77443b0..8ff371ab 100644 --- a/libs/database/src/lib/factories/project.factory.ts +++ b/libs/database/src/lib/factories/project.factory.ts @@ -1,6 +1,20 @@ import { FactoryGirl } from "factory-girl-ts"; import { Project } from "../entities"; +import { faker } from "@faker-js/faker"; +import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { ApplicationFactory } from "./application.factory"; +import { OrganisationFactory } from "./organisation.factory"; +import { FRAMEWORK_KEYS } from "../constants/framework"; + +const CONTINENTS = ["africa", "australia", "south-america", "asia", "north-america"]; export const ProjectFactory = FactoryGirl.define(Project, async () => ({ - uuid: crypto.randomUUID() + uuid: crypto.randomUUID(), + name: faker.animal.petName(), + frameworkKey: faker.helpers.arrayElement(FRAMEWORK_KEYS), + status: faker.helpers.arrayElement(ENTITY_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), + applicationId: ApplicationFactory.associate("id"), + organisationId: OrganisationFactory.associate("id"), + continent: faker.helpers.arrayElement(CONTINENTS) })); diff --git a/libs/database/src/lib/factories/restoration-partner.factory.ts b/libs/database/src/lib/factories/restoration-partner.factory.ts new file mode 100644 index 00000000..44b23cda --- /dev/null +++ b/libs/database/src/lib/factories/restoration-partner.factory.ts @@ -0,0 +1,19 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { ProjectReport, RestorationPartner } from "../entities"; +import { ProjectReportFactory } from "./project-report.factory"; +import { faker } from "@faker-js/faker"; + +const defaultAttributesFactory = async () => ({ + uuid: crypto.randomUUID(), + description: null, + hidden: false +}); + +export const RestorationPartnerFactory = { + forProjectReport: FactoryGirl.define(RestorationPartner, async () => ({ + ...(await defaultAttributesFactory()), + partnerableType: ProjectReport.LARAVEL_TYPE, + partnerableId: ProjectReportFactory.associate("id"), + collection: faker.helpers.arrayElement(ProjectReport.RESTORATION_PARTNER_COLLECTIONS) + })) +}; diff --git a/libs/database/src/lib/factories/seeding.factory.ts b/libs/database/src/lib/factories/seeding.factory.ts new file mode 100644 index 00000000..a82a2f8b --- /dev/null +++ b/libs/database/src/lib/factories/seeding.factory.ts @@ -0,0 +1,26 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { Seeding, Site, SiteReport } from "../entities"; +import { faker } from "@faker-js/faker"; +import { SiteReportFactory } from "./site-report.factory"; +import { SiteFactory } from "./site.factory"; + +const defaultAttributesFactory = async () => ({ + uuid: crypto.randomUUID(), + name: faker.lorem.words(2), + amount: faker.number.int({ min: 10, max: 1000 }), + hidden: false +}); + +export const SeedingFactory = { + forSite: FactoryGirl.define(Seeding, async () => ({ + ...(await defaultAttributesFactory()), + seedableType: Site.LARAVEL_TYPE, + seedableId: SiteFactory.associate("id") + })), + + forSiteReport: FactoryGirl.define(Seeding, async () => ({ + ...(await defaultAttributesFactory()), + seedableType: SiteReport.LARAVEL_TYPE, + seedableId: SiteReportFactory.associate("id") + })) +}; diff --git a/libs/database/src/lib/factories/site-polygon.factory.ts b/libs/database/src/lib/factories/site-polygon.factory.ts index 56dce319..7161aa66 100644 --- a/libs/database/src/lib/factories/site-polygon.factory.ts +++ b/libs/database/src/lib/factories/site-polygon.factory.ts @@ -36,6 +36,7 @@ export const SitePolygonFactory = FactoryGirl.define(SitePolygon, async () => { source: "terramatch", createdBy: createdBy.get("id"), isActive: true, - versionName: name + versionName: name, + calcArea: faker.number.float({ min: 0.5, max: 1000 }) }; }); diff --git a/libs/database/src/lib/factories/site-report.factory.ts b/libs/database/src/lib/factories/site-report.factory.ts index 4ba73f1d..2787bf89 100644 --- a/libs/database/src/lib/factories/site-report.factory.ts +++ b/libs/database/src/lib/factories/site-report.factory.ts @@ -3,13 +3,17 @@ import { FactoryGirl } from "factory-girl-ts"; import { SiteFactory } from "./site.factory"; import { faker } from "@faker-js/faker"; import { DateTime } from "luxon"; +import { REPORT_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; export const SiteReportFactory = FactoryGirl.define(SiteReport, async () => { const dueAt = faker.date.past({ years: 2 }); + dueAt.setMilliseconds(0); return { uuid: crypto.randomUUID(), siteId: SiteFactory.associate("id"), dueAt, - submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }) + submittedAt: faker.date.between({ from: dueAt, to: DateTime.fromJSDate(dueAt).plus({ days: 14 }).toJSDate() }), + status: faker.helpers.arrayElement(REPORT_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES) }; }); diff --git a/libs/database/src/lib/factories/site.factory.ts b/libs/database/src/lib/factories/site.factory.ts index 82967897..5a436840 100644 --- a/libs/database/src/lib/factories/site.factory.ts +++ b/libs/database/src/lib/factories/site.factory.ts @@ -1,8 +1,16 @@ import { Site } from "../entities"; import { FactoryGirl } from "factory-girl-ts"; import { ProjectFactory } from "./project.factory"; +import { faker } from "@faker-js/faker"; +import { ENTITY_STATUSES, UPDATE_REQUEST_STATUSES } from "../constants/status"; +import { SITING_STRATEGIES } from "../constants/entity-selects"; export const SiteFactory = FactoryGirl.define(Site, async () => ({ uuid: crypto.randomUUID(), - projectId: ProjectFactory.associate("id") + projectId: ProjectFactory.associate("id"), + name: faker.animal.petName(), + status: faker.helpers.arrayElement(ENTITY_STATUSES), + updateRequestStatus: faker.helpers.arrayElement(UPDATE_REQUEST_STATUSES), + sitingStrategy: faker.helpers.arrayElement(SITING_STRATEGIES), + descriptionSitingStrategy: faker.lorem.paragraph() })); diff --git a/libs/database/src/lib/factories/workday.factory.ts b/libs/database/src/lib/factories/workday.factory.ts new file mode 100644 index 00000000..52017086 --- /dev/null +++ b/libs/database/src/lib/factories/workday.factory.ts @@ -0,0 +1,27 @@ +import { FactoryGirl } from "factory-girl-ts"; +import { ProjectReport, SiteReport, Workday } from "../entities"; +import { ProjectReportFactory } from "./project-report.factory"; +import { SiteReportFactory } from "./site-report.factory"; +import { faker } from "@faker-js/faker"; + +const defaultAttributesFactory = async () => ({ + uuid: crypto.randomUUID(), + description: null, + hidden: false +}); + +export const WorkdayFactory = { + forProjectReport: FactoryGirl.define(Workday, async () => ({ + ...(await defaultAttributesFactory()), + workdayableType: ProjectReport.LARAVEL_TYPE, + workdayableId: ProjectReportFactory.associate("id"), + collection: faker.helpers.arrayElement(ProjectReport.WORKDAY_COLLECTIONS) + })), + + forSiteReport: FactoryGirl.define(Workday, async () => ({ + ...(await defaultAttributesFactory()), + workdayableType: SiteReport.LARAVEL_TYPE, + workdayableId: SiteReportFactory.associate("id"), + collection: faker.helpers.arrayElement(SiteReport.WORKDAY_COLLECTIONS) + })) +}; diff --git a/package-lock.json b/package-lock.json index ca6aedce..f8a29782 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@nestjs/core": "^10.0.2", "@nestjs/jwt": "^10.2.0", "@nestjs/platform-express": "^10.0.2", + "@nestjs/schedule": "^4.1.2", "@nestjs/sequelize": "^10.0.1", "@nestjs/swagger": "^7.4.0", "@nestjs/terminus": "^10.2.3", @@ -34,6 +35,7 @@ "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", + "nestjs-slack": "^2.0.0", "reflect-metadata": "^0.1.14", "rxjs": "^7.8.0", "sequelize": "^6.37.3", @@ -461,7 +463,6 @@ "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", - "dev": true, "engines": { "node": ">=6.9.0" } @@ -470,7 +471,6 @@ "version": "7.25.9", "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", - "dev": true, "engines": { "node": ">=6.9.0" } @@ -515,7 +515,6 @@ "version": "7.26.3", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", - "dev": true, "dependencies": { "@babel/types": "^7.26.3" }, @@ -1910,7 +1909,6 @@ "version": "7.26.3", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", - "dev": true, "dependencies": { "@babel/helper-string-parser": "^7.25.9", "@babel/helper-validator-identifier": "^7.25.9" @@ -2135,6 +2133,130 @@ "integrity": "sha512-QnxP42Qu9M2UogdrF2kxpZcgWeW9R3WoUr+LdpcsbkvX3LjEoDYgrJ/PnV/QUCbxB1gaKbhR0ZxlDxE1cPkpDg==", "dev": true }, + "node_modules/@google-cloud/common": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz", + "integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==", + "dependencies": { + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^8.0.2", + "retry-request": "^5.0.0", + "teeny-request": "^8.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/logging": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/@google-cloud/logging/-/logging-10.5.0.tgz", + "integrity": "sha512-XmlNs6B8lDZvFwFB5M55g9ch873AA2rXSuFOczQ3FOAzuyd/qksf18suFJfcrLMu8lYSr3SQhTE45FlXz4p9pg==", + "dependencies": { + "@google-cloud/common": "^4.0.0", + "@google-cloud/paginator": "^4.0.0", + "@google-cloud/projectify": "^3.0.0", + "@google-cloud/promisify": "^3.0.0", + "arrify": "^2.0.1", + "dot-prop": "^6.0.0", + "eventid": "^2.0.0", + "extend": "^3.0.2", + "gcp-metadata": "^4.0.0", + "google-auth-library": "^8.0.2", + "google-gax": "^3.5.8", + "on-finished": "^2.3.0", + "pumpify": "^2.0.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz", + "integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz", + "integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz", + "integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.8.22", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.22.tgz", + "integrity": "sha512-oAjDdN7fzbUi+4hZjKG96MR6KTEubAeMpQEb+77qy+3r0Ua5xTFuie6JOLr4ZZgl5g+W5/uRTS2M1V8mVAFPuA==", + "dependencies": { + "@grpc/proto-loader": "^0.7.0", + "@types/node": ">=12.12.47" + }, + "engines": { + "node": "^8.13.0 || >=10.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.13", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.13.tgz", + "integrity": "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw==", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader/node_modules/protobufjs": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", + "integrity": "sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -2589,6 +2711,17 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@jsdoc/salty": { + "version": "0.2.9", + "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.9.tgz", + "integrity": "sha512-yYxMVH7Dqw6nO0d5NIV8OQWnitU8k6vXH8NtgqAfIa/IUqRMxRv/NUJJ08VEKbAakwxlgBl5PJdrU0dMPStsnw==", + "dependencies": { + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=v12.0.0" + } + }, "node_modules/@jsonjoy.com/base64": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-1.1.2.tgz", @@ -3195,6 +3328,31 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.1.2.tgz", + "integrity": "sha512-hCTQ1lNjIA5EHxeu8VvQu2Ed2DBLS1GSC6uKPYlBiQe6LL9a7zfE9iVSK+zuK8E2odsApteEBmfAQchc8Hx0Gg==", + "dependencies": { + "cron": "3.2.1", + "uuid": "11.0.3" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, + "node_modules/@nestjs/schedule/node_modules/uuid": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.0.3.tgz", + "integrity": "sha512-d0z310fCWv5dJwnX1Y/MncBAqGMKEzlBb1AOf7z9K8ALnd0utBX/msg/fA0+sbyN1ihbMsLhrBlnl1ak7Wa0rg==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, "node_modules/@nestjs/schematics": { "version": "10.2.3", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.2.3.tgz", @@ -4915,6 +5073,60 @@ "@opentelemetry/api": "^1.3.0" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" + }, "node_modules/@sentry/core": { "version": "8.45.0", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.45.0.tgz", @@ -5044,6 +5256,93 @@ "@sinonjs/commons": "^3.0.1" } }, + "node_modules/@slack/logger": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@slack/logger/-/logger-3.0.0.tgz", + "integrity": "sha512-DTuBFbqu4gGfajREEMrkq5jBhcnskinhr4+AnfJEk48zhVeEv3XnUKGIX98B74kxhYsIMfApGGySTn7V3b5yBA==", + "dependencies": { + "@types/node": ">=12.0.0" + }, + "engines": { + "node": ">= 12.13.0", + "npm": ">= 6.12.0" + } + }, + "node_modules/@slack/types": { + "version": "2.14.0", + "resolved": "https://registry.npmjs.org/@slack/types/-/types-2.14.0.tgz", + "integrity": "sha512-n0EGm7ENQRxlXbgKSrQZL69grzg1gHLAVd+GlRVQJ1NSORo0FrApR7wql/gaKdu2n4TO83Sq/AmeUOqD60aXUA==", + "engines": { + "node": ">= 12.13.0", + "npm": ">= 6.12.0" + } + }, + "node_modules/@slack/web-api": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/@slack/web-api/-/web-api-6.13.0.tgz", + "integrity": "sha512-dv65crIgdh9ZYHrevLU6XFHTQwTyDmNqEqzuIrV+Vqe/vgiG6w37oex5ePDU1RGm2IJ90H8iOvHFvzdEO/vB+g==", + "dependencies": { + "@slack/logger": "^3.0.0", + "@slack/types": "^2.11.0", + "@types/is-stream": "^1.1.0", + "@types/node": ">=12.0.0", + "axios": "^1.7.4", + "eventemitter3": "^3.1.0", + "form-data": "^2.5.0", + "is-electron": "2.2.2", + "is-stream": "^1.1.0", + "p-queue": "^6.6.1", + "p-retry": "^4.0.0" + }, + "engines": { + "node": ">= 12.13.0", + "npm": ">= 6.12.0" + } + }, + "node_modules/@slack/web-api/node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" + }, + "node_modules/@slack/web-api/node_modules/eventemitter3": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", + "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==" + }, + "node_modules/@slack/web-api/node_modules/form-data": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.2.tgz", + "integrity": "sha512-GgwY0PS7DbXqajuGf4OYlsrIu3zgxD6Vvql43IBhm6MahqA5SK/7mwhtNj2AdH2z35YR34ujJ7BN+3fFC3jP5Q==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12", + "safe-buffer": "^5.2.1" + }, + "engines": { + "node": ">= 0.12" + } + }, + "node_modules/@slack/web-api/node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@slack/web-api/node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/@swc-node/core": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/@swc-node/core/-/core-1.13.3.tgz", @@ -5334,6 +5633,14 @@ "@swc/counter": "^0.1.3" } }, + "node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, "node_modules/@trysound/sax": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/@trysound/sax/-/sax-0.2.0.tgz", @@ -5542,6 +5849,15 @@ "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.14.tgz", "integrity": "sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==" }, + "node_modules/@types/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==", + "dependencies": { + "@types/minimatch": "^5.1.2", + "@types/node": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", @@ -5566,6 +5882,14 @@ "@types/node": "*" } }, + "node_modules/@types/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", @@ -5614,17 +5938,40 @@ "@types/node": "*" } }, + "node_modules/@types/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-sVDA58zAw4eWAffKOaQH5/5j3XeayukzDk+ewSsnv3p4yJEZHCCzMDiZM8e0OUrRvmpGZ85jf4yDHkHsgBNr9Q==" + }, "node_modules/@types/lodash": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.13.tgz", "integrity": "sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==", "dev": true }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==" + }, "node_modules/@types/luxon": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", - "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==", - "dev": true + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==" + }, + "node_modules/@types/markdown-it": { + "version": "14.1.2", + "resolved": "https://registry.npmjs.org/@types/markdown-it/-/markdown-it-14.1.2.tgz", + "integrity": "sha512-promo4eFwuiW+TfGxhi+0x3czqTYJkG8qB17ZUJiVF10Xm7NLVRSLUsfRTU/6h1e24VvRnXCx+hG7li58lkzog==", + "dependencies": { + "@types/linkify-it": "^5", + "@types/mdurl": "^2" + } + }, + "node_modules/@types/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-RGdgjQUZba5p6QEFAVx2OGb8rQDL/cPRG7GiedRzMcJ1tYnUANBncjbSB1NRGwbvjcPeikRABz2nshyPk1bhWg==" }, "node_modules/@types/methods": { "version": "1.1.4", @@ -5638,6 +5985,11 @@ "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", "dev": true }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==" + }, "node_modules/@types/ms": { "version": "0.7.34", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", @@ -5707,6 +6059,15 @@ "integrity": "sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==", "dev": true }, + "node_modules/@types/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==", + "dependencies": { + "@types/glob": "*", + "@types/node": "*" + } + }, "node_modules/@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", @@ -6371,7 +6732,6 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } @@ -6406,6 +6766,17 @@ "node": ">=12.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/airtable": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/airtable/-/airtable-0.12.2.tgz", @@ -6610,6 +6981,14 @@ "node": ">=8" } }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -6918,7 +7297,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -6972,6 +7350,14 @@ "node": "*" } }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -7009,6 +7395,11 @@ "node": ">= 6" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -7097,7 +7488,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, "dependencies": { "balanced-match": "^1.0.0" } @@ -7292,6 +7682,33 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz", + "integrity": "sha512-BhYE+WDaywFg2TBWYNXAE+8B1ATnThNBqXHP5nQu0jWJdVvY2hvkpyB3qOmtmDePiS5/BDQ8wASEWGMWRG148g==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz", + "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "get-intrinsic": "^1.2.6" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -7342,6 +7759,17 @@ } ] }, + "node_modules/catharsis": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.9.0.tgz", + "integrity": "sha512-prMTQVpcns/tzFgFVkVp6ak6RykZyWb3gu8ckUpd6YkTlacOd3DXGJjIpD4Q6zJirizvaiAjSSHlOsA+6sNh2A==", + "dependencies": { + "lodash": "^4.17.15" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -7493,7 +7921,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -7966,6 +8393,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.2.1.tgz", + "integrity": "sha512-w2n5l49GMmmkBFEsH9FIDhjZ1n1QgTMOCMGuQtOXs5veNiosZmso6bQGuqOJSYAXXrG84WQFVneNk+Yt0Ua9iw==", + "dependencies": { + "@types/luxon": "~3.4.0", + "luxon": "~3.5.0" + } + }, "node_modules/cron-parser": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", @@ -8278,8 +8714,7 @@ "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" }, "node_modules/deepmerge": { "version": "4.3.1", @@ -8553,6 +8988,20 @@ "url": "https://github.com/fb55/domutils?sponsor=1" } }, + "node_modules/dot-prop": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-6.0.1.tgz", + "integrity": "sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==", + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dotenv": { "version": "16.4.5", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", @@ -8577,27 +9026,64 @@ "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", "integrity": "sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==" }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "dev": true }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", "dependencies": { - "safe-buffer": "^5.0.1" + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" } }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" - }, - "node_modules/ejs": { - "version": "3.1.10", + "node_modules/duplexify/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "node_modules/ejs": { + "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "dev": true, @@ -8655,7 +9141,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "dependencies": { "once": "^1.4.0" } @@ -8685,11 +9170,29 @@ "node": ">=8.6" } }, + "node_modules/ent": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.2.tgz", + "integrity": "sha512-kKvD1tO6BM+oK9HzCPpUdRb4vKFQY/FPTFmurMvh6LlN68VMrdj77w8yp51/kDbpkFOS9J8w5W6zIzgM2H8/hw==", + "dependencies": { + "call-bound": "^1.0.3", + "es-errors": "^1.3.0", + "punycode": "^1.4.1", + "safe-regex-test": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ent/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "dev": true, "engines": { "node": ">=0.12" }, @@ -8732,12 +9235,9 @@ } }, "node_modules/es-define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", - "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", - "dependencies": { - "get-intrinsic": "^1.2.4" - }, + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", "engines": { "node": ">= 0.4" } @@ -8756,11 +9256,21 @@ "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==", "dev": true }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, "engines": { "node": ">=6" } @@ -8782,6 +9292,91 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-1.14.3.tgz", + "integrity": "sha512-qFcX0XJkdg+PB3xjZZG/wKSuT1PnQWx57+TVSjIMmILd2yC/6ByYElPwJnslDsuWuSAp4AwJGumarAAmJch5Kw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^4.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=4.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/escodegen/node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/escodegen/node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/eslint": { "version": "8.57.1", "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", @@ -8870,7 +9465,6 @@ "version": "3.4.3", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, @@ -8987,7 +9581,6 @@ "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, "dependencies": { "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", @@ -9004,7 +9597,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -9041,7 +9633,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "engines": { "node": ">=4.0" } @@ -9050,7 +9641,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -9074,8 +9664,26 @@ "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "node_modules/eventid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/eventid/-/eventid-2.0.1.tgz", + "integrity": "sha512-sPNTqiMokAvV048P2c9+foqVJzk49o6d4e0D/sq5jog3pw+4kBgyR0gaM1FM7Mx6Kzd9dztesh9oYz1LWWOpzw==", + "dependencies": { + "uuid": "^8.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/eventid/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } }, "node_modules/events": { "version": "3.3.0", @@ -9205,6 +9813,11 @@ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/factory-girl-ts": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/factory-girl-ts/-/factory-girl-ts-2.3.1.tgz", @@ -9262,14 +9875,18 @@ "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, "node_modules/fast-safe-stringify": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, + "node_modules/fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -9789,6 +10406,33 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-4.3.3.tgz", + "integrity": "sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA==", + "dependencies": { + "abort-controller": "^3.0.0", + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.7" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/gcp-metadata": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.3.1.tgz", + "integrity": "sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==", + "dependencies": { + "gaxios": "^4.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/generate-function": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", @@ -9818,21 +10462,25 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } }, "node_modules/get-intrinsic": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", - "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.7.tgz", + "integrity": "sha512-VW6Pxhsrk0KAOqs3WEd0klDiF/+V7gQOpAvY1jVU/LHmaD/kQO4523aiJuikX/QAKYiW6x8Jh+RJej1almdtCA==", "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-define-property": "^1.0.1", "es-errors": "^1.3.0", + "es-object-atoms": "^1.0.0", "function-bind": "^1.1.2", - "has-proto": "^1.0.1", - "has-symbols": "^1.0.3", - "hasown": "^2.0.0" + "get-proto": "^1.0.0", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" }, "engines": { "node": ">= 0.4" @@ -9850,6 +10498,18 @@ "node": ">=8.0.0" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -10022,6 +10682,130 @@ "node": ">= 6" } }, + "node_modules/google-auth-library": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.9.0.tgz", + "integrity": "sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==", + "dependencies": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.3.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-auth-library/node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-auth-library/node_modules/gcp-metadata": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.3.0.tgz", + "integrity": "sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==", + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-auth-library/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/google-auth-library/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/google-gax": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/google-gax/-/google-gax-3.6.1.tgz", + "integrity": "sha512-g/lcUjGcB6DSw2HxgEmCDOrI/CByOwqRvsuUvNalHUK2iPPPlmAIpbMbl62u0YufGMr8zgE3JL7th6dCb1Ry+w==", + "dependencies": { + "@grpc/grpc-js": "~1.8.0", + "@grpc/proto-loader": "^0.7.0", + "@types/long": "^4.0.0", + "@types/rimraf": "^3.0.2", + "abort-controller": "^3.0.0", + "duplexify": "^4.0.0", + "fast-text-encoding": "^1.0.3", + "google-auth-library": "^8.0.2", + "is-stream-ended": "^0.1.4", + "node-fetch": "^2.6.1", + "object-hash": "^3.0.0", + "proto3-json-serializer": "^1.0.0", + "protobufjs": "7.2.4", + "protobufjs-cli": "1.1.1", + "retry-request": "^5.0.0" + }, + "bin": { + "compileProtos": "build/tools/compileProtos.js", + "minifyProtoJson": "build/tools/minify.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "deprecated": "Package is no longer maintained", + "dependencies": { + "node-forge": "^1.3.1" + }, + "bin": { + "gp12-pem": "build/src/bin/gp12-pem.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -10036,8 +10820,7 @@ "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/graphemer": { "version": "1.4.0", @@ -10045,6 +10828,52 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "dependencies": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/gtoken/node_modules/gaxios": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.1.3.tgz", + "integrity": "sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gtoken/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gtoken/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/handle-thing": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.1.tgz", @@ -10085,20 +10914,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-proto": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.1.0.tgz", - "integrity": "sha512-QLdzI9IIO1Jg7f9GT1gXpPpXArAn6cS31R1eEZqz08Gc+uQ8/XiqHWt17Fiw+2p6oTTIq5GXEpQkAlA88YRl/Q==", - "dependencies": { - "call-bind": "^1.0.7" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -10114,7 +10929,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "dependencies": { "has-symbols": "^1.0.3" }, @@ -10318,6 +11132,19 @@ "node": ">=8.0.0" } }, + "node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/http-proxy-middleware": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-3.0.3.tgz", @@ -10359,7 +11186,19 @@ "http-server": "bin/http-server" }, "engines": { - "node": ">=12" + "node": ">=12" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" } }, "node_modules/human-signals": { @@ -10655,6 +11494,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-electron": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-electron/-/is-electron-2.2.2.tgz", + "integrity": "sha512-FO/Rhvz5tuw4MCWkpMzHFKWD2LsfHzIb7i6MdPYZ/KW7AlxawyLkqdy+jPZP1WubqEADE3O4FUENlJHDfQASRg==" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -10771,6 +11615,14 @@ "node": ">=0.12.0" } }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "engines": { + "node": ">=8" + } + }, "node_modules/is-path-inside": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", @@ -10806,11 +11658,27 @@ "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==" }, + "node_modules/is-regex": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz", + "integrity": "sha512-MjYsKHO5O7mCsmRGxWcLWheFqN9DJ/2TmngvjKXihe6efViPqc274+Fx/4fYj/r03+ESvBdTXK0V6tA3rgez1g==", + "dependencies": { + "call-bound": "^1.0.2", + "gopd": "^1.2.0", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "engines": { "node": ">=8" }, @@ -10818,6 +11686,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-stream-ended": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-stream-ended/-/is-stream-ended-0.1.4.tgz", + "integrity": "sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==" + }, "node_modules/is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", @@ -11679,6 +12552,61 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/js2xmlparser": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/js2xmlparser/-/js2xmlparser-4.0.2.tgz", + "integrity": "sha512-6n4D8gLlLf1n5mNLQPRfViYzu9RATblzPEtm1SthMX1Pjao0r9YI9nw7ZIfRxQMERS87mcswrg+r/OYrPRX6jA==", + "dependencies": { + "xmlcreate": "^2.0.4" + } + }, + "node_modules/jsdoc": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/jsdoc/-/jsdoc-4.0.4.tgz", + "integrity": "sha512-zeFezwyXeG4syyYHbvh1A967IAqq/67yXtXvuL5wnqCkFZe8I0vKfm+EO+YEvLguo6w9CDUbrAXVtJSHh2E8rw==", + "dependencies": { + "@babel/parser": "^7.20.15", + "@jsdoc/salty": "^0.2.1", + "@types/markdown-it": "^14.1.1", + "bluebird": "^3.7.2", + "catharsis": "^0.9.0", + "escape-string-regexp": "^2.0.0", + "js2xmlparser": "^4.0.2", + "klaw": "^3.0.0", + "markdown-it": "^14.1.0", + "markdown-it-anchor": "^8.6.7", + "marked": "^4.0.10", + "mkdirp": "^1.0.4", + "requizzle": "^0.2.3", + "strip-json-comments": "^3.1.0", + "underscore": "~1.13.2" + }, + "bin": { + "jsdoc": "jsdoc.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/jsdoc/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/jsdoc/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/jsesc": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", @@ -11691,6 +12619,14 @@ "node": ">=6" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -11833,6 +12769,14 @@ "node": ">=0.10.0" } }, + "node_modules/klaw": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz", + "integrity": "sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g==", + "dependencies": { + "graceful-fs": "^4.1.9" + } + }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -12112,6 +13056,14 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/linkify-it": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/loader-runner": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz", @@ -12152,6 +13104,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" + }, "node_modules/lodash.clonedeepwith": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeepwith/-/lodash.clonedeepwith-4.5.0.tgz", @@ -12395,12 +13352,61 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" }, + "node_modules/markdown-it": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", + "dependencies": { + "argparse": "^2.0.1", + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" + }, + "bin": { + "markdown-it": "bin/markdown-it.mjs" + } + }, + "node_modules/markdown-it-anchor": { + "version": "8.6.7", + "resolved": "https://registry.npmjs.org/markdown-it-anchor/-/markdown-it-anchor-8.6.7.tgz", + "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", + "peerDependencies": { + "@types/markdown-it": "*", + "markdown-it": "*" + } + }, + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mdn-data": { "version": "2.0.30", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.30.tgz", "integrity": "sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==", "dev": true }, + "node_modules/mdurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -12788,6 +13794,32 @@ "@nestjs/common": "^10.0.0" } }, + "node_modules/nestjs-slack": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/nestjs-slack/-/nestjs-slack-2.0.0.tgz", + "integrity": "sha512-05ATly30VJ+MjBnBfg9BORMW4Cs315kSrgbdOUn6hNcKY1i9Ri3baP8tamT4PFwbPZyW6uCCoLHFExMBdAuGRg==", + "dependencies": { + "@google-cloud/logging": "^10.0.1", + "@slack/web-api": "^6", + "axios": "^0.27.2", + "node-fetch": "^2.6.7", + "slack-block-builder": "^2", + "ts-invariant": "^0.10.3" + }, + "peerDependencies": { + "@nestjs/common": ">=9 <=10", + "@nestjs/core": ">=9 <=10" + } + }, + "node_modules/nestjs-slack/node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, "node_modules/node-abi": { "version": "3.71.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", @@ -12834,7 +13866,6 @@ "version": "1.3.1", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", - "dev": true, "engines": { "node": ">= 6.13.0" } @@ -13065,6 +14096,14 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", @@ -13197,6 +14236,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -13239,6 +14286,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-6.2.1.tgz", @@ -13256,6 +14318,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -14291,6 +15364,97 @@ "node": ">= 6" } }, + "node_modules/proto3-json-serializer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/proto3-json-serializer/-/proto3-json-serializer-1.1.1.tgz", + "integrity": "sha512-AwAuY4g9nxx0u52DnSMkqqgyLHaW/XaPLtaAo3y/ZCfeaQB/g4YDH4kb8Wc/mWzWvu0YjOznVnfn373MVZZrgw==", + "dependencies": { + "protobufjs": "^7.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "hasInstallScript": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/protobufjs-cli": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/protobufjs-cli/-/protobufjs-cli-1.1.1.tgz", + "integrity": "sha512-VPWMgIcRNyQwWUv8OLPyGQ/0lQY/QTQAVN5fh+XzfDwsVw1FZ2L3DM/bcBf8WPiRz2tNpaov9lPZfNcmNo6LXA==", + "dependencies": { + "chalk": "^4.0.0", + "escodegen": "^1.13.0", + "espree": "^9.0.0", + "estraverse": "^5.1.0", + "glob": "^8.0.0", + "jsdoc": "^4.0.0", + "minimist": "^1.2.0", + "semver": "^7.1.2", + "tmp": "^0.2.1", + "uglify-js": "^3.7.7" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "protobufjs": "^7.0.0" + } + }, + "node_modules/protobufjs-cli/node_modules/glob": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", + "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^5.0.1", + "once": "^1.3.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/protobufjs-cli/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -14315,6 +15479,25 @@ "dev": true, "optional": true }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pumpify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz", + "integrity": "sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw==", + "dependencies": { + "duplexify": "^4.1.1", + "inherits": "^2.0.3", + "pump": "^3.0.0" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -14324,6 +15507,14 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "engines": { + "node": ">=6" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -14613,7 +15804,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -14646,6 +15836,14 @@ "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", "dev": true }, + "node_modules/requizzle": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/requizzle/-/requizzle-0.2.4.tgz", + "integrity": "sha512-JRrFk1D4OQ4SqovXOgdav+K8EAhSB/LJZqCz8tbX0KObcdeM15Ss59ozWMBWmmINMagCwmqn4ZNryUGpBsl6Jw==", + "dependencies": { + "lodash": "^4.17.21" + } + }, "node_modules/resolve": { "version": "1.22.8", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", @@ -14722,7 +15920,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "dev": true, "engines": { "node": ">= 4" } @@ -14732,6 +15929,18 @@ "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" }, + "node_modules/retry-request": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz", + "integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==", + "dependencies": { + "debug": "^4.1.1", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -14826,6 +16035,22 @@ } ] }, + "node_modules/safe-regex-test": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.1.0.tgz", + "integrity": "sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "is-regex": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -15373,6 +16598,11 @@ "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", "dev": true }, + "node_modules/slack-block-builder": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/slack-block-builder/-/slack-block-builder-2.8.0.tgz", + "integrity": "sha512-iisM+j99iKRuQFVfdWo0FiszDAl3r8Snq704oZH6C0RbDqvoVQStiptt6Y7kc6RX/5hSAqTqjhgvZ/di8cvaIA==" + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -15569,6 +16799,19 @@ "node": ">= 0.8" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, "node_modules/streamroller": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.1.5.tgz", @@ -15695,7 +16938,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, "engines": { "node": ">=8" }, @@ -15720,6 +16962,11 @@ "node": ">=4" } }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, "node_modules/style-loader": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.4.tgz", @@ -15965,6 +17212,21 @@ "node": ">= 6" } }, + "node_modules/teeny-request": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.3.tgz", + "integrity": "sha512-jJZpA5He2y52yUhA7pyAGZlgQpcB+xLjcN0eUFxr9c8hP/H7uOXbBNVo/O0C/xVfJLJs680jvkFgVJEEvk9+ww==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.1", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/terser": { "version": "5.36.0", "resolved": "https://registry.npmjs.org/terser/-/terser-5.36.0.tgz", @@ -16190,7 +17452,6 @@ "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", - "dev": true, "engines": { "node": ">=14.14" } @@ -16259,6 +17520,17 @@ "typescript": ">=4.2.0" } }, + "node_modules/ts-invariant": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/ts-invariant/-/ts-invariant-0.10.3.tgz", + "integrity": "sha512-uivwYcQaxAucv1CzRp2n/QdYPo4ILf9VXgH19zEIjFx2EJufV16P0JtJVpYHy89DItG6Kwj2oIUjrcK5au+4tQ==", + "dependencies": { + "tslib": "^2.1.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/ts-jest": { "version": "29.2.5", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz", @@ -16489,6 +17761,22 @@ "node": ">=14.17" } }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -16500,6 +17788,11 @@ "node": ">=8" } }, + "node_modules/underscore": { + "version": "1.13.7", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", + "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==" + }, "node_modules/undici-types": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", @@ -17251,7 +18544,6 @@ "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -17311,6 +18603,11 @@ } } }, + "node_modules/xmlcreate": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/xmlcreate/-/xmlcreate-2.0.4.tgz", + "integrity": "sha512-nquOebG4sngPmGPICTS5EnxqhKbCmz5Ox5hsszI2T6U5qdrJizBc+0ilYSEjTSzU0yZcmvppztXe/5Al5fUwdg==" + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", @@ -17323,7 +18620,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -17347,7 +18643,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -17365,7 +18660,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } diff --git a/package.json b/package.json index a3dd0964..926cac23 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "@nestjs/core": "^10.0.2", "@nestjs/jwt": "^10.2.0", "@nestjs/platform-express": "^10.0.2", + "@nestjs/schedule": "^4.1.2", "@nestjs/sequelize": "^10.0.1", "@nestjs/swagger": "^7.4.0", "@nestjs/terminus": "^10.2.3", @@ -35,6 +36,7 @@ "mariadb": "^3.3.2", "mysql2": "^3.11.2", "nestjs-request-context": "^3.0.0", + "nestjs-slack": "^2.0.0", "reflect-metadata": "^0.1.14", "rxjs": "^7.8.0", "sequelize": "^6.37.3",