Merge pull request #45 from wri/release/willful-willow

[RELEASE] Willful Willow

.env.local.sample

@@ -4,6 +4,8 @@ RESEARCH_SERVICE_PORT=4030
 UNIFIED_DATABASE_SERVICE_PORT=4040
 ENTITY_SERVICE_PORT=4050
 
+DEPLOY_ENV=local
+
 DB_HOST=localhost
 DB_PORT=3360
 DB_DATABASE=wri_restoration_marketplace_api
@@ -16,4 +18,9 @@ REDIS_PORT=6379
 JWT_SECRET=qu3sep4GKdbg6PiVPCKLKljHukXALorq6nLHDBOCSwvs6BrgE6zb8gPmZfrNspKt
 
 # Only needed for the unified database service. Most developers should not have this defined.
-AIRTABLE_API_KEY
+AIRTABLE_API_KEY=
+AIRTABLE_BASE_ID=
+
+# Should only be defined locally for testing
+SLACK_API_KEY=
+UDB_SLACK_CHANNEL=

.github/workflows/deploy-service.yml

@@ -71,7 +71,7 @@ jobs:
           : # Don't build the base image with NODE_ENV because it'll limit the packages that are installed
           docker build -t terramatch-microservices-base:nx-base .
           SERVICE_IMAGE=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
-          docker build --build-arg NODE_ENV=production --build-arg BUILD_FLAG='--prod --verbose' -f apps/${{ inputs.service }}/Dockerfile -t $SERVICE_IMAGE .
+          docker build --build-arg NODE_ENV=production --build-arg DEPLOY_ENV=${{ inputs.env }} --build-arg SENTRY_DSN="${{ vars.SENTRY_DSN }}" --build-arg BUILD_FLAG='--prod --verbose' -f apps/${{ inputs.service }}/Dockerfile -t $SERVICE_IMAGE .
           docker push $SERVICE_IMAGE
           echo "image=$SERVICE_IMAGE"
 

.github/workflows/pull-request.yml

@@ -46,7 +46,13 @@ jobs:
       # in a clean way. For some reason, the `run-many` is necessary here. If this line simply uses
       # nx test database, the connection to the DB gets cut off before the sync is complete.
       - name: Sync DB Schema
-        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database --no-cache
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test database --skip-nx-cache
+
+      # Run the UDB service tests in isolation because they require clearing out DB tables and ensuring
+      # they know exactly what's in them, which is not conducive to parallel runs with other test
+      # suites.
+      - name: Unified DB Test
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx test unified-database-service --coverage
 
       - name: Test all
-        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests
+        run: NX_CLOUD_DISTRIBUTED_EXECUTION=false npx nx run-many -t test --coverage --passWithNoTests --exclude unified-database-service database

README.md

@@ -7,7 +7,6 @@ Repository for the Microservices API backend of the TerraMatch service
 - Node v20.11.1. Using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file) is recommended.
 - [Docker](https://www.docker.com/)
 - [CDK CLI](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html) (install globally)
-- [AWS SAM CLI](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/install-sam-cli.html)
 - [NX](https://nx.dev/getting-started/installation#installing-nx-globally) (install globally)
 - [NestJS](https://docs.nestjs.com/) (install globally, useful for development)
 

apps/entity-service/Dockerfile

@@ -8,8 +8,12 @@ RUN npx nx build entity-service ${BUILD_FLAG}
 FROM terramatch-microservices-base:nx-base
 
 ARG NODE_ENV
+ARG SENTRY_DSN
+ARG DEPLOY_ENV
 WORKDIR /app
 COPY --from=builder /app/builder ./
 ENV NODE_ENV=${NODE_ENV}
+ENV SENTRY_DSN=${SENTRY_DSN}
+ENV DEPLOY_ENV=${DEPLOY_ENV}
 
 CMD ["node", "./dist/apps/entity-service/main.js"]

apps/entity-service/src/trees/dto/establishment-trees.dto.ts

@@ -20,7 +20,9 @@ export class EstablishmentsTreesDto extends JsonApiAttributes<EstablishmentsTree
   @ApiProperty({
     type: "object",
     additionalProperties: { type: "array", items: { type: "string" } },
-    description: "The species that were specified at the establishment of the parent entity keyed by collection",
+    description:
+      "The species that were specified at the establishment of the parent entity keyed by collection. " +
+      'Note that for site reports, the seeds on the site establishment are included under the collection name "seeds"',
     example: { "tree-planted": ["Aster Peraliens", "Circium carniolicum"], "non-tree": ["Coffee"] }
   })
   establishmentTrees: Dictionary<string[]>;
@@ -33,7 +35,9 @@ export class EstablishmentsTreesDto extends JsonApiAttributes<EstablishmentsTree
     },
     nullable: true,
     description:
-      "If the entity in this request is a report, the sum totals of previous planting by species by collection.",
+      "If the entity in this request is a report, the sum totals of previous planting by species by collection. " +
+      "Note that for site reports, the seeds planted under previous site reports are included under the collection " +
+      'name "seeds"',
     example: {
       "tree-planted": {
         "Aster persaliens": { amount: 256 },

apps/entity-service/src/trees/tree.service.spec.ts

@@ -1,13 +1,14 @@
 import { TreeService } from "./tree.service";
 import { Test } from "@nestjs/testing";
-import { TreeSpecies, TreeSpeciesResearch } from "@terramatch-microservices/database/entities";
+import { Seeding, TreeSpecies, TreeSpeciesResearch } from "@terramatch-microservices/database/entities";
 import { pick, uniq } from "lodash";
 import { faker } from "@faker-js/faker";
 import {
   NurseryFactory,
   NurseryReportFactory,
   ProjectFactory,
   ProjectReportFactory,
+  SeedingFactory,
   SiteFactory,
   SiteReportFactory,
   TreeSpeciesFactory,
@@ -79,20 +80,33 @@ describe("TreeService", () => {
       )
         .map(({ name }) => name)
         .sort();
+      // hidden trees are ignored
+      await TreeSpeciesFactory.forProjectTreePlanted.create({
+        speciesableId: project.id,
+        hidden: true
+      });
       const siteTreesPlanted = (await TreeSpeciesFactory.forSiteTreePlanted.createMany(2, { speciesableId: site.id }))
         .map(({ name }) => name)
         .sort();
+      await TreeSpeciesFactory.forSiteTreePlanted.create({ speciesableId: site.id, hidden: true });
       const siteNonTrees = (await TreeSpeciesFactory.forSiteNonTree.createMany(3, { speciesableId: site.id }))
         .map(({ name }) => name)
         .sort();
+      const siteSeedings = (await SeedingFactory.forSite.createMany(3, { seedableId: site.id }))
+        .map(({ name }) => name)
+        .sort();
+      await SeedingFactory.forSite.create({ seedableId: site.id, hidden: true });
       const nurserySeedlings = (
         await TreeSpeciesFactory.forNurserySeedling.createMany(4, { speciesableId: nursery.id })
       )
         .map(({ name }) => name)
         .sort();
+      await TreeSpeciesFactory.forNurserySeedling.create({
+        speciesableId: nursery.id,
+        hidden: true
+      });
 
       let result = await service.getEstablishmentTrees("project-reports", projectReport.uuid);
-      console.log("result", result);
       expect(Object.keys(result).length).toBe(1);
       expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted);
       result = await service.getEstablishmentTrees("sites", site.uuid);
@@ -103,9 +117,11 @@ describe("TreeService", () => {
       expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted);
 
       result = await service.getEstablishmentTrees("site-reports", siteReport.uuid);
-      expect(Object.keys(result).length).toBe(2);
+      expect(Object.keys(result).length).toBe(3);
       expect(result["tree-planted"].sort()).toEqual(uniq([...siteTreesPlanted, ...projectTreesPlanted]).sort());
       expect(result["non-tree"].sort()).toEqual(siteNonTrees);
+      expect(result["seeds"].sort()).toEqual(siteSeedings);
+
       result = await service.getEstablishmentTrees("nursery-reports", nurseryReport.uuid);
       expect(Object.keys(result).length).toBe(2);
       expect(result["tree-planted"].sort()).toEqual(projectTreesPlanted);
@@ -145,7 +161,7 @@ describe("TreeService", () => {
         dueAt: DateTime.fromJSDate(nurseryReport1.dueAt).plus({ months: 3 }).toJSDate()
       });
 
-      const reduceTreeCounts = (counts: Record<string, PreviousPlantingCountDto>, tree: TreeSpecies) => ({
+      const reduceTreeCounts = (counts: Record<string, PreviousPlantingCountDto>, tree: TreeSpecies | Seeding) => ({
         ...counts,
         [tree.name]: {
           taxonId: counts[tree.name]?.taxonId ?? tree.taxonId,
@@ -161,9 +177,16 @@ describe("TreeService", () => {
           taxonId: "wfo-projectreporttree"
         })
       );
+      // hidden trees should be ignored
+      let hidden = await TreeSpeciesFactory.forProjectReportTreePlanted.create({
+        speciesableId: projectReport1.id,
+        hidden: true
+      });
+
       let result = await service.getPreviousPlanting("project-reports", projectReport2.uuid);
       expect(Object.keys(result)).toMatchObject(["tree-planted"]);
       expect(result).toMatchObject({ "tree-planted": projectReportTreesPlanted.reduce(reduceTreeCounts, {}) });
+      expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name);
 
       const siteReport1TreesPlanted = await TreeSpeciesFactory.forSiteReportTreePlanted.createMany(3, {
         speciesableId: siteReport1.id
@@ -174,33 +197,51 @@ describe("TreeService", () => {
           taxonId: "wfo-sitereporttree"
         })
       );
+      // hidden trees should be ignored
+      hidden = await TreeSpeciesFactory.forSiteReportTreePlanted.create({
+        speciesableId: siteReport1.id,
+        hidden: true
+      });
       const siteReport2TreesPlanted = await TreeSpeciesFactory.forSiteReportTreePlanted.createMany(3, {
         speciesableId: siteReport2.id
       });
       const siteReport2NonTrees = await TreeSpeciesFactory.forSiteReportNonTree.createMany(2, {
         speciesableId: siteReport2.id
       });
+      const siteReport2Seedings = await SeedingFactory.forSiteReport.createMany(2, { seedableId: siteReport2.id });
+      await SeedingFactory.forSiteReport.create({ seedableId: siteReport2.id, hidden: true });
+
       result = await service.getPreviousPlanting("site-reports", siteReport1.uuid);
       expect(result).toMatchObject({});
       result = await service.getPreviousPlanting("site-reports", siteReport2.uuid);
       const siteReport1TreesPlantedReduced = siteReport1TreesPlanted.reduce(reduceTreeCounts, {});
-      expect(Object.keys(result)).toMatchObject(["tree-planted"]);
-      expect(result).toMatchObject({ "tree-planted": siteReport1TreesPlantedReduced });
+      expect(Object.keys(result).sort()).toMatchObject(["seeds", "tree-planted"]);
+      expect(result).toMatchObject({ "tree-planted": siteReport1TreesPlantedReduced, seeds: {} });
+      expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name);
       result = await service.getPreviousPlanting("site-reports", siteReport3.uuid);
-      expect(Object.keys(result).sort()).toMatchObject(["non-tree", "tree-planted"]);
+      expect(Object.keys(result).sort()).toMatchObject(["non-tree", "seeds", "tree-planted"]);
       expect(result).toMatchObject({
         "tree-planted": siteReport2TreesPlanted.reduce(reduceTreeCounts, siteReport1TreesPlantedReduced),
-        "non-tree": siteReport2NonTrees.reduce(reduceTreeCounts, {})
+        "non-tree": siteReport2NonTrees.reduce(reduceTreeCounts, {}),
+        seeds: siteReport2Seedings.reduce(reduceTreeCounts, {})
       });
+      expect(Object.keys(result["tree-planted"])).not.toContain(hidden.name);
 
       result = await service.getPreviousPlanting("nursery-reports", nurseryReport2.uuid);
       expect(result).toMatchObject({});
       const nurseryReportSeedlings = await TreeSpeciesFactory.forNurseryReportSeedling.createMany(5, {
         speciesableId: nurseryReport1.id
       });
+      // hidden trees should be ignored
+      hidden = await TreeSpeciesFactory.forNurseryReportSeedling.create({
+        speciesableId: nurseryReport1.id,
+        hidden: true
+      });
+
       result = await service.getPreviousPlanting("nursery-reports", nurseryReport2.uuid);
       expect(Object.keys(result)).toMatchObject(["nursery-seedling"]);
       expect(result).toMatchObject({ "nursery-seedling": nurseryReportSeedlings.reduce(reduceTreeCounts, {}) });
+      expect(Object.keys(result["nursery-seedling"])).not.toContain(hidden.name);
     });
 
     it("handles bad input to get previous planting with undefined or an exception", async () => {

apps/entity-service/src/trees/tree.service.ts

@@ -8,7 +8,7 @@ import {
   TreeSpecies,
   TreeSpeciesResearch
 } from "@terramatch-microservices/database/entities";
-import { Op, WhereOptions } from "sequelize";
+import { Includeable, Op, WhereOptions } from "sequelize";
 import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common";
 import { Dictionary, filter, flatten, flattenDeep, groupBy, uniq } from "lodash";
 import { PreviousPlantingCountDto } from "./dto/establishment-trees.dto";
@@ -25,7 +25,12 @@ type TreeModelType = TreeReportModelType | typeof Project | typeof Site | typeof
 const isReport = (type: EstablishmentEntity): type is EstablishmentReport => type.endsWith("-reports");
 
 const treeAssociations = (model: TreeModelType, attributes: string[], where?: WhereOptions) =>
-  model.TREE_ASSOCIATIONS.map(association => ({ required: false, association, attributes, where }));
+  model.TREE_ASSOCIATIONS.map(association => ({
+    required: false,
+    association,
+    attributes,
+    where: { ...where, hidden: false }
+  }));
 
 const uniqueTreeNames = (trees: Dictionary<TreeSpecies[]>) =>
   Object.keys(trees).reduce(
@@ -60,29 +65,38 @@ export class TreeService {
       // For site and nursery reports, we fetch both the establishment species on the parent entity
       // and on the Project
       const parentModel = entity === "site-reports" ? Site : Nursery;
-      const whereOptions = {
-        where: { uuid },
-        attributes: [],
+      const include = {
+        model: parentModel,
+        // This id isn't necessary for the data we want to fetch, but sequelize requires it for
+        // the nested includes
+        attributes: ["id"],
         include: [
+          ...treeAssociations(parentModel, ["name", "collection"]),
           {
-            model: parentModel,
+            model: Project,
             // This id isn't necessary for the data we want to fetch, but sequelize requires it for
             // the nested includes
             attributes: ["id"],
-            include: [
-              ...treeAssociations(parentModel, ["name", "collection"]),
-              {
-                model: Project,
-                // This id isn't necessary for the data we want to fetch, but sequelize requires it for
-                // the nested includes
-                attributes: ["id"],
-                include: treeAssociations(Project, ["name", "collection"])
-              }
-            ]
+            include: treeAssociations(Project, ["name", "collection"])
           }
         ]
       };
 
+      if (entity === "site-reports") {
+        include.include.push({
+          required: false,
+          association: "seedsPlanted",
+          attributes: ["name"],
+          where: { hidden: false }
+        });
+      }
+
+      const whereOptions = {
+        where: { uuid },
+        attributes: [],
+        include: [include]
+      };
+
       const report = await (entity === "site-reports"
         ? SiteReport.findOne(whereOptions)
         : NurseryReport.findOne(whereOptions));
@@ -97,7 +111,11 @@ export class TreeService {
         "collection"
       );
 
-      return uniqueTreeNames(trees);
+      const treeNames = uniqueTreeNames(trees);
+      if (entity === "site-reports") {
+        treeNames["seeds"] = uniq(((parent as Site).seedsPlanted ?? []).map(({ name }) => name));
+      }
+      return treeNames;
     } else if (["sites", "nurseries", "project-reports"].includes(entity)) {
       // for these we simply pull the project's trees
       const whereOptions = {
@@ -160,14 +178,26 @@ export class TreeService {
     });
     if (report == null) throw new NotFoundException();
 
+    const modelIncludes: Includeable[] = treeAssociations(model, ["taxonId", "name", "collection", "amount"], {
+      amount: { [Op.gt]: 0 }
+    });
+    if (entity === "site-reports") {
+      modelIncludes.push({
+        required: false,
+        association: "seedsPlanted",
+        attributes: ["name", "taxonId", "amount"],
+        where: { hidden: false, amount: { [Op.gt]: 0 } }
+      });
+    }
+
     // @ts-expect-error Can't narrow the union TreeReportModelType automatically
     const records: InstanceType<TreeReportModelType>[] = await model.findAll({
       attributes: [],
       where: {
         [model.PARENT_ID]: report[model.PARENT_ID],
         dueAt: { [Op.lt]: report.dueAt }
       },
-      include: treeAssociations(model, ["taxonId", "name", "collection", "amount"], { amount: { [Op.gt]: 0 } })
+      include: modelIncludes
     });
 
     const trees = groupBy(
@@ -177,7 +207,7 @@ export class TreeService {
       "collection"
     );
 
-    return Object.keys(trees).reduce(
+    const planting = Object.keys(trees).reduce(
       (dict, collection) => ({
         ...dict,
         [collection]: trees[collection].reduce(
@@ -193,5 +223,20 @@ export class TreeService {
       }),
       {} as Dictionary<Dictionary<PreviousPlantingCountDto>>
     );
+
+    if (entity === "site-reports") {
+      planting["seeds"] = flatten((records as SiteReport[]).map(({ seedsPlanted }) => seedsPlanted)).reduce(
+        (counts, seeding) => ({
+          ...counts,
+          [seeding.name]: {
+            taxonId: undefined,
+            amount: (counts[seeding.name]?.amount ?? 0) + (seeding.amount ?? 0)
+          }
+        }),
+        {} as Dictionary<PreviousPlantingCountDto>
+      );
+    }
+
+    return planting;
   }
 }

apps/job-service/Dockerfile

@@ -8,8 +8,12 @@ RUN npx nx build job-service ${BUILD_FLAG}
 FROM terramatch-microservices-base:nx-base
 
 ARG NODE_ENV
+ARG SENTRY_DSN
+ARG DEPLOY_ENV
 WORKDIR /app
 COPY --from=builder /app/builder ./
 ENV NODE_ENV=${NODE_ENV}
+ENV SENTRY_DSN=${SENTRY_DSN}
+ENV DEPLOY_ENV=${DEPLOY_ENV}
 
 CMD ["node", "./dist/apps/job-service/main.js"]