Merge pull request #49 from wri/feat/TM-1675-projects-reports-airtabl…

…e-update

[TM-1675] Update some columns on project and project-report.

apps/unified-database-service/src/airtable/entities/airtable-entity.spec.ts

@@ -3,6 +3,7 @@ import { faker } from "@faker-js/faker";
 import {
   Application,
   Demographic,
+  Framework,
   Nursery,
   NurseryReport,
   Organisation,
@@ -49,7 +50,7 @@ import {
 } from "./";
 import { orderBy, sortBy } from "lodash";
 import { Model } from "sequelize-typescript";
-import { FRAMEWORK_NAMES, FrameworkKey } from "@terramatch-microservices/database/constants/framework";
+import { FrameworkKey } from "@terramatch-microservices/database/constants/framework";
 import { FindOptions, Op } from "sequelize";
 import { DateTime } from "luxon";
 
@@ -417,9 +418,21 @@ describe("AirtableEntity", () => {
     let projects: Project[];
     let calculatedValues: Record<string, Record<string, string | number>>;
 
+    const FRAMEWORK_NAMES = {
+      ppc: "PPC",
+      terrafund: "TerraFund Top 100"
+    };
+
     beforeAll(async () => {
       await Project.truncate();
 
+      for (const framework of await Framework.findAll()) {
+        await framework.destroy();
+      }
+      for (const [slug, name] of Object.entries(FRAMEWORK_NAMES)) {
+        await Framework.create({ uuid: faker.string.uuid(), slug, name });
+      }
+
       const orgs = await OrganisationFactory.createMany(3);
       organisationUuids = orgs.reduce((uuids, { id, uuid }) => ({ ...uuids, [id]: uuid }), {});
       const orgIds = orgs.reduce((ids, { id }) => [...ids, id], [] as number[]);
@@ -444,6 +457,9 @@ describe("AirtableEntity", () => {
         })
       );
 
+      // make sure test projects are not included
+      await ProjectFactory.create({ organisationId: orgId(), isTest: true });
+
       projects = allProjects.filter(project => !project.isSoftDeleted());
       applicationUuids = (
         await Application.findAll({
@@ -487,7 +503,7 @@ describe("AirtableEntity", () => {
           fields: {
             uuid,
             name,
-            cohort: FRAMEWORK_NAMES[frameworkKey] ?? frameworkKey,
+            framework: FRAMEWORK_NAMES[frameworkKey] ?? frameworkKey,
             organisationUuid: organisationUuids[organisationId],
             applicationUuid: applicationUuids[applicationId],
             hectaresRestoredToDate: calculatedValues[uuid]?.hectaresRestoredToDate ?? 0

apps/unified-database-service/src/airtable/entities/airtable-entity.ts

@@ -1,5 +1,5 @@
 import { Model, ModelCtor, ModelType } from "sequelize-typescript";
-import { cloneDeep, flatten, groupBy, isObject, uniq } from "lodash";
+import { cloneDeep, flatten, groupBy, isEmpty, isObject, uniq } from "lodash";
 import { Attributes, FindOptions, Op, WhereOptions } from "sequelize";
 import { TMLogService } from "@terramatch-microservices/common/util/tm-log.service";
 import { LoggerService } from "@nestjs/common";
@@ -16,7 +16,7 @@ export abstract class AirtableEntity<ModelType extends Model<ModelType>, Associa
   abstract readonly MODEL: ModelCtor<ModelType>;
   readonly IDENTITY_COLUMN: string = "uuid";
   readonly SUPPORTS_UPDATED_SINCE: boolean = true;
-  readonly HAS_HIDDEN_FLAG: boolean = false;
+  readonly FILTER_FLAGS: string[] = [];
 
   protected readonly logger: LoggerService = new TMLogService(AirtableEntity.name);
 
@@ -69,13 +69,16 @@ export abstract class AirtableEntity<ModelType extends Model<ModelType>, Associa
     if (this.SUPPORTS_UPDATED_SINCE && updatedSince != null) {
       where["updatedAt"] = { [Op.gte]: updatedSince };
     }
-    if (this.HAS_HIDDEN_FLAG) {
-      where["hidden"] = false;
+    if (!isEmpty(this.FILTER_FLAGS)) {
+      for (const flag of this.FILTER_FLAGS) {
+        where[flag] = false;
+      }
     }
 
     return {
       attributes: selectAttributes(this.COLUMNS),
       include: selectIncludes(this.COLUMNS),
+      order: ["id"],
       limit: AIRTABLE_PAGE_SIZE,
       offset: page * AIRTABLE_PAGE_SIZE,
       where
@@ -86,25 +89,26 @@ export abstract class AirtableEntity<ModelType extends Model<ModelType>, Associa
     const where = {} as WhereOptions<ModelType>;
 
     const deletedAtCondition = { [Op.gte]: deletedSince };
-    if (this.HAS_HIDDEN_FLAG) {
+    if (isEmpty(this.FILTER_FLAGS)) {
+      where["deletedAt"] = deletedAtCondition;
+    } else {
       where[Op.or] = {
         deletedAt: deletedAtCondition,
         // include records that have been hidden since the timestamp as well
         [Op.and]: {
           updatedAt: { ...deletedAtCondition },
-          hidden: true
+          [Op.or]: this.FILTER_FLAGS.reduce((flags, flag) => ({ ...flags, [flag]: true }), {})
         }
       };
-    } else {
-      where["deletedAt"] = deletedAtCondition;
     }
 
     return {
       attributes: [this.IDENTITY_COLUMN],
       paranoid: false,
-      where,
+      order: ["id"],
       limit: AIRTABLE_PAGE_SIZE,
-      offset: page * AIRTABLE_PAGE_SIZE
+      offset: page * AIRTABLE_PAGE_SIZE,
+      where
     } as FindOptions<ModelType>;
   }
 

apps/unified-database-service/src/airtable/entities/project-report.airtable-entity.ts

@@ -90,7 +90,8 @@ const COLUMNS: ColumnMapping<ProjectReport, ProjectReportAssociations>[] = [
   },
   "technicalNarrative",
   "publicNarrative",
-  "totalUniqueRestorationPartners"
+  "totalUniqueRestorationPartners",
+  "businessMilestones"
 ];
 
 export class ProjectReportEntity extends AirtableEntity<ProjectReport, ProjectReportAssociations> {

apps/unified-database-service/src/airtable/entities/project.airtable-entity.ts

@@ -1,7 +1,13 @@
-import { Application, Organisation, Project, Site, SitePolygon } from "@terramatch-microservices/database/entities";
+import {
+  Application,
+  Framework,
+  Organisation,
+  Project,
+  Site,
+  SitePolygon
+} from "@terramatch-microservices/database/entities";
 import { AirtableEntity, ColumnMapping, commonEntityColumns } from "./airtable-entity";
 import { flatten, groupBy } from "lodash";
-import { FRAMEWORK_NAMES } from "@terramatch-microservices/database/constants/framework";
 
 const loadApprovedSites = async (projectIds: number[]) =>
   groupBy(
@@ -29,9 +35,10 @@ const COLUMNS: ColumnMapping<Project, ProjectAssociations>[] = [
   ...commonEntityColumns<Project, ProjectAssociations>("project"),
   "name",
   {
+    airtableColumn: "framework",
     dbColumn: "frameworkKey",
-    airtableColumn: "cohort",
-    valueMap: async ({ frameworkKey }) => FRAMEWORK_NAMES[frameworkKey] ?? frameworkKey
+    include: [{ model: Framework, attributes: ["name"] }],
+    valueMap: async ({ framework, frameworkKey }) => framework?.name ?? frameworkKey
   },
   {
     airtableColumn: "applicationUuid",
@@ -101,6 +108,7 @@ export class ProjectEntity extends AirtableEntity<Project, ProjectAssociations>
   readonly COLUMNS = COLUMNS;
   readonly MODEL = Project;
   readonly SUPPORTS_UPDATED_SINCE = false;
+  readonly FILTER_FLAGS = ["isTest"];
 
   async loadAssociations(projects: Project[]) {
     const projectIds = projects.map(({ id }) => id);

apps/unified-database-service/src/airtable/entities/restoration-partner.airtable-entity.ts

@@ -23,7 +23,7 @@ export class RestorationPartnerEntity extends AirtableEntity<RestorationPartner,
   readonly TABLE_NAME = "Restoration Partners";
   readonly COLUMNS = COLUMNS;
   readonly MODEL = RestorationPartner;
-  readonly HAS_HIDDEN_FLAG = true;
+  readonly FILTER_FLAGS = ["hidden"];
 
   protected async loadAssociations(partners: RestorationPartner[]) {
     return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPINGS, "partnerableType", "partnerableId", partners);

apps/unified-database-service/src/airtable/entities/tree-species.airtable-entity.ts

@@ -77,7 +77,7 @@ export class TreeSpeciesEntity extends AirtableEntity<TreeSpecies, TreeSpeciesAs
   readonly TABLE_NAME = "Tree Species";
   readonly COLUMNS = COLUMNS;
   readonly MODEL = TreeSpecies;
-  readonly HAS_HIDDEN_FLAG = true;
+  readonly FILTER_FLAGS = ["hidden"];
 
   protected async loadAssociations(treeSpecies: TreeSpecies[]) {
     return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPING, "speciesableType", "speciesableId", treeSpecies);

apps/unified-database-service/src/airtable/entities/workday.airtable-entity.ts

@@ -29,7 +29,7 @@ export class WorkdayEntity extends AirtableEntity<Workday, WorkdayAssociations>
   readonly TABLE_NAME = "Workdays";
   readonly COLUMNS = COLUMNS;
   readonly MODEL = Workday;
-  readonly HAS_HIDDEN_FLAG = true;
+  readonly FILTER_FLAGS = ["hidden"];
 
   protected async loadAssociations(workdays: Workday[]) {
     return this.loadPolymorphicUuidAssociations(LARAVEL_TYPE_MAPPINGS, "workdayableType", "workdayableId", workdays);

libs/database/src/lib/constants/framework.ts

@@ -1,11 +1,9 @@
-export const FRAMEWORK_NAMES = {
-  terrafund: "TerraFund Top 100",
-  "terrafund-landscapes": "TerraFund Landscapes",
-  ppc: "Priceless Planet Coalition (PPC)",
-  enterprises: "TerraFund Enterprises",
-  hbf: "Harit Bharat Fund",
-  "epa-ghana-pilot": "EPA-Ghana Pilot"
-} as const;
-
-export const FRAMEWORK_KEYS = Object.keys(FRAMEWORK_NAMES);
-export type FrameworkKey = keyof typeof FRAMEWORK_NAMES;
+export const FRAMEWORK_KEYS = [
+  "terrafund",
+  "terrafund-landscapes",
+  "ppc",
+  "enterprises",
+  "hbf",
+  "epa-ghana-pilot"
+] as const;
+export type FrameworkKey = (typeof FRAMEWORK_KEYS)[number];

libs/database/src/lib/entities/framework.entity.ts

@@ -1,14 +1,19 @@
-import { AutoIncrement, Column, Model, PrimaryKey, Table } from "sequelize-typescript";
-import { BIGINT, STRING } from "sequelize";
+import { AutoIncrement, Column, Index, Model, PrimaryKey, Table } from "sequelize-typescript";
+import { BIGINT, STRING, UUID } from "sequelize";
 
-// A quick stub to get the information needed for users/me
+// Incomplete stub
 @Table({ tableName: "frameworks", underscored: true })
 export class Framework extends Model<Framework> {
   @PrimaryKey
   @AutoIncrement
   @Column(BIGINT.UNSIGNED)
   override id: number;
 
+  @Index
+  @Column(UUID)
+  uuid: string;
+
+  @Index
   @Column(STRING(20))
   slug: string;
 

libs/database/src/lib/entities/project-report.entity.ts

@@ -360,6 +360,10 @@ export class ProjectReport extends Model<ProjectReport> {
   @Column(INTEGER.UNSIGNED)
   totalUniqueRestorationPartners: number | null;
 
+  @AllowNull
+  @Column(TEXT)
+  businessMilestones: string | null;
+
   @HasMany(() => TreeSpecies, {
     foreignKey: "speciesableId",
     constraints: false,

libs/database/src/lib/entities/project.entity.ts

@@ -20,6 +20,7 @@ import { Site } from "./site.entity";
 import { Nursery } from "./nursery.entity";
 import { JsonColumn } from "../decorators/json-column.decorator";
 import { FrameworkKey } from "../constants/framework";
+import { Framework } from "./framework.entity";
 
 @Table({ tableName: "v2_projects", underscored: true, paranoid: true })
 export class Project extends Model<Project> {
@@ -39,6 +40,9 @@ export class Project extends Model<Project> {
   @Column(STRING)
   frameworkKey: FrameworkKey | null;
 
+  @BelongsTo(() => Framework, { foreignKey: "frameworkKey", targetKey: "slug", constraints: false })
+  framework: Framework | null;
+
   @Default(false)
   @Column(BOOLEAN)
   isTest: boolean;