[ASAP-516] - collaboration filters (#4324)

* update queries * update data provider * update export scripts * update frontend * update test description * added some tests
yldio · Jul 10, 2024 · e25fe8b · e25fe8b
1 parent 1db30ba
commit e25fe8b
Show file tree

Hide file tree

Showing 20 changed files with 980 additions and 214 deletions.
diff --git a/apps/asap-cli/src/scripts/algolia/performance/collaboration.ts b/apps/asap-cli/src/scripts/algolia/performance/collaboration.ts
@@ -3,6 +3,8 @@ import {
   timeRanges,
   TeamOutputDocumentType,
   PerformanceMetrics,
+  outputTypes,
+  documentCategories,
 } from '@asap-hub/model';
 
 import {
@@ -43,35 +45,38 @@ export const processUserCollaborationPerformance = async (
   await deletePreviousObjects(index, type);
 
   timeRanges.forEach(async (range) => {
-    const getPaginatedHits = (page: number) =>
-      index.search<UserCollaborationHit>('', {
-        filters: `__meta.range:"${range}" AND (__meta.type:"${type}")`,
-        attributesToRetrieve: ['teams'],
-        page,
-        hitsPerPage: 50,
-      });
-
-    const hits = await getAllHits<UserCollaborationHit>(getPaginatedHits);
-    const flatHits = hits.reduce(
-      (items: UserTeam[], item) => [...items, ...item.teams],
-      [],
-    );
-
-    await index.saveObject(
-      {
-        withinTeam: getBellCurveMetrics(
-          flatHits.map((hit) => hit.outputsCoAuthoredWithinTeam),
-        ),
-        acrossTeam: getBellCurveMetrics(
-          flatHits.map((hit) => hit.outputsCoAuthoredAcrossTeams),
-        ),
-        __meta: {
-          range,
-          type: `${type}-performance`,
+    documentCategories.forEach(async (documentCategory) => {
+      const getPaginatedHits = (page: number) =>
+        index.search<UserCollaborationHit>('', {
+          filters: `__meta.range:"${range}" AND __meta.documentCategory:"${documentCategory}" AND __meta.type:"${type}"`,
+          attributesToRetrieve: ['teams'],
+          page,
+          hitsPerPage: 50,
+        });
+
+      const hits = await getAllHits<UserCollaborationHit>(getPaginatedHits);
+      const flatHits = hits.reduce(
+        (items: UserTeam[], item) => [...items, ...item.teams],
+        [],
+      );
+
+      await index.saveObject(
+        {
+          withinTeam: getBellCurveMetrics(
+            flatHits.map((hit) => hit.outputsCoAuthoredWithinTeam),
+          ),
+          acrossTeam: getBellCurveMetrics(
+            flatHits.map((hit) => hit.outputsCoAuthoredAcrossTeams),
+          ),
+          __meta: {
+            range,
+            type: `${type}-performance`,
+            documentCategory,
+          },
         },
-      },
-      { autoGenerateObjectIDIfNotExist: true },
-    );
+        { autoGenerateObjectIDIfNotExist: true },
+      );
+    });
   });
 };
 
@@ -83,52 +88,55 @@ export const processTeamCollaborationPerformance = async (
   await deletePreviousObjects(index, type);
 
   timeRanges.forEach(async (range) => {
-    const getPaginatedHits = (page: number) =>
-      index.search<TeamCollaborationHit>('', {
-        filters: `__meta.range:"${range}" AND (__meta.type:"${type}")`,
-        attributesToRetrieve: [
-          'outputsCoProducedAcross',
-          'outputsCoProducedWithin',
-        ],
-        page,
-        hitsPerPage: 50,
-      });
-
-    const hits = await getAllHits<TeamCollaborationHit>(getPaginatedHits);
-
-    const fields = [
-      { name: 'Article', documentType: 'article' },
-      { name: 'Bioinformatics', documentType: 'bioinformatics' },
-      { name: 'Dataset', documentType: 'dataset' },
-      { name: 'Lab Resource', documentType: 'labResource' },
-      { name: 'Protocol', documentType: 'protocol' },
-    ];
-
-    const teamPerformanceByDocumentType = (rawMetrics: DocumentMetric[]) =>
-      fields.reduce(
-        (metrics, { name, documentType }) => ({
-          ...metrics,
-          [documentType]: getBellCurveMetrics(
-            rawMetrics.map((item) => item[name as TeamOutputDocumentType]),
+    outputTypes.forEach(async (outputType) => {
+      const getPaginatedHits = (page: number) =>
+        index.search<TeamCollaborationHit>('', {
+          filters: `__meta.range:"${range}" AND __meta.outputType:"${outputType}" AND __meta.type:"${type}"`,
+          attributesToRetrieve: [
+            'outputsCoProducedAcross',
+            'outputsCoProducedWithin',
+          ],
+          page,
+          hitsPerPage: 50,
+        });
+
+      const hits = await getAllHits<TeamCollaborationHit>(getPaginatedHits);
+
+      const fields = [
+        { name: 'Article', documentType: 'article' },
+        { name: 'Bioinformatics', documentType: 'bioinformatics' },
+        { name: 'Dataset', documentType: 'dataset' },
+        { name: 'Lab Resource', documentType: 'labResource' },
+        { name: 'Protocol', documentType: 'protocol' },
+      ];
+
+      const teamPerformanceByDocumentType = (rawMetrics: DocumentMetric[]) =>
+        fields.reduce(
+          (metrics, { name, documentType }) => ({
+            ...metrics,
+            [documentType]: getBellCurveMetrics(
+              rawMetrics.map((item) => item[name as TeamOutputDocumentType]),
+            ),
+          }),
+          {} as Record<string, PerformanceMetrics>,
+        );
+
+      await index.saveObject(
+        {
+          withinTeam: teamPerformanceByDocumentType(
+            hits.map((hit) => hit.outputsCoProducedWithin),
           ),
-        }),
-        {} as Record<string, PerformanceMetrics>,
-      );
-
-    await index.saveObject(
-      {
-        withinTeam: teamPerformanceByDocumentType(
-          hits.map((hit) => hit.outputsCoProducedWithin),
-        ),
-        acrossTeam: teamPerformanceByDocumentType(
-          hits.map((hit) => hit.outputsCoProducedAcross.byDocumentType),
-        ),
-        __meta: {
-          range,
-          type: `${type}-performance`,
+          acrossTeam: teamPerformanceByDocumentType(
+            hits.map((hit) => hit.outputsCoProducedAcross.byDocumentType),
+          ),
+          __meta: {
+            range,
+            type: `${type}-performance`,
+            outputType,
+          },
         },
-      },
-      { autoGenerateObjectIDIfNotExist: true },
-    );
+        { autoGenerateObjectIDIfNotExist: true },
+      );
+    });
   });
 };
diff --git a/apps/asap-cli/src/scripts/algolia/performance/tests/collaboration.test.ts b/apps/asap-cli/src/scripts/algolia/performance/tests/collaboration.test.ts
@@ -1,4 +1,4 @@
-import { timeRanges } from '@asap-hub/model';
+import { documentCategories, outputTypes, timeRanges } from '@asap-hub/model';
 import { SearchIndex } from 'algoliasearch';
 import {
   processTeamCollaborationPerformance,
@@ -62,19 +62,25 @@ describe('processUserCollaborationPerformance', () => {
     ]);
 
     timeRanges.forEach((range) => {
-      expect(mockIndex.search).toHaveBeenCalledWith('', {
-        filters: `__meta.range:"${range}" AND (__meta.type:"user-collaboration")`,
-        attributesToRetrieve: ['teams'],
-        page: expect.any(Number),
-        hitsPerPage: 50,
+      documentCategories.forEach((documentCategory) => {
+        expect(mockIndex.search).toHaveBeenCalledWith('', {
+          filters: `__meta.range:"${range}" AND __meta.documentCategory:"${documentCategory}" AND __meta.type:"user-collaboration"`,
+          attributesToRetrieve: ['teams'],
+          page: expect.any(Number),
+          hitsPerPage: 50,
+        });
       });
     });
 
-    // one for each time range
-    expect(await mockIndex.saveObject).toHaveBeenCalledTimes(5);
+    // one for each time range and document category combination
+    expect(await mockIndex.saveObject).toHaveBeenCalledTimes(30);
     expect(await mockIndex.saveObject).toHaveBeenLastCalledWith(
       {
-        __meta: { range: 'all', type: 'user-collaboration-performance' },
+        __meta: {
+          range: 'all',
+          type: 'user-collaboration-performance',
+          documentCategory: 'protocol',
+        },
         acrossTeam: {
           aboveAverageMax: 4,
           aboveAverageMin: 4,
@@ -199,22 +205,122 @@ describe('processTeamCollaborationPerformance', () => {
     ]);
 
     timeRanges.forEach((range) => {
-      expect(mockIndex.search).toHaveBeenCalledWith('', {
-        filters: `__meta.range:"${range}" AND (__meta.type:"team-collaboration")`,
-        attributesToRetrieve: [
-          'outputsCoProducedAcross',
-          'outputsCoProducedWithin',
-        ],
-        page: expect.any(Number),
-        hitsPerPage: 50,
+      outputTypes.forEach((outputType) => {
+        expect(mockIndex.search).toHaveBeenCalledWith('', {
+          filters: `__meta.range:"${range}" AND __meta.outputType:"${outputType}" AND __meta.type:"team-collaboration"`,
+          attributesToRetrieve: [
+            'outputsCoProducedAcross',
+            'outputsCoProducedWithin',
+          ],
+          page: expect.any(Number),
+          hitsPerPage: 50,
+        });
       });
     });
 
-    // one for each time range
-    expect(await mockIndex.saveObject).toHaveBeenCalledTimes(5);
-    expect(await mockIndex.saveObject).toHaveBeenLastCalledWith(
+    // one for each time range and output type combination
+    expect(await mockIndex.saveObject).toHaveBeenCalledTimes(10);
+    expect(await mockIndex.saveObject).toHaveBeenCalledWith(
       {
-        __meta: { range: 'all', type: 'team-collaboration-performance' },
+        __meta: {
+          range: 'all',
+          type: 'team-collaboration-performance',
+          outputType: 'public',
+        },
+        acrossTeam: {
+          article: {
+            aboveAverageMax: 50,
+            aboveAverageMin: 43,
+            averageMax: 42,
+            averageMin: 7,
+            belowAverageMax: 6,
+            belowAverageMin: 0,
+          },
+          bioinformatics: {
+            aboveAverageMax: 10,
+            aboveAverageMin: 9,
+            averageMax: 8,
+            averageMin: 3,
+            belowAverageMax: 2,
+            belowAverageMin: 1,
+          },
+          dataset: {
+            aboveAverageMax: 30,
+            aboveAverageMin: 23,
+            averageMax: 22,
+            averageMin: 1,
+            belowAverageMax: 0,
+            belowAverageMin: 3,
+          },
+          labResource: {
+            aboveAverageMax: 20,
+            aboveAverageMin: 18,
+            averageMax: 17,
+            averageMin: 5,
+            belowAverageMax: 4,
+            belowAverageMin: 4,
+          },
+          protocol: {
+            aboveAverageMax: 19,
+            aboveAverageMin: 18,
+            averageMax: 17,
+            averageMin: 4,
+            belowAverageMax: 3,
+            belowAverageMin: 0,
+          },
+        },
+        withinTeam: {
+          article: {
+            aboveAverageMax: 20,
+            aboveAverageMin: 20,
+            averageMax: 19,
+            averageMin: 3,
+            belowAverageMax: 2,
+            belowAverageMin: 0,
+          },
+          bioinformatics: {
+            aboveAverageMax: 10,
+            aboveAverageMin: 9,
+            averageMax: 8,
+            averageMin: 3,
+            belowAverageMax: 2,
+            belowAverageMin: 1,
+          },
+          dataset: {
+            aboveAverageMax: 30,
+            aboveAverageMin: 24,
+            averageMax: 23,
+            averageMin: 3,
+            belowAverageMax: 2,
+            belowAverageMin: 4,
+          },
+          labResource: {
+            aboveAverageMax: 20,
+            aboveAverageMin: 18,
+            averageMax: 17,
+            averageMin: 5,
+            belowAverageMax: 4,
+            belowAverageMin: 4,
+          },
+          protocol: {
+            aboveAverageMax: 14,
+            aboveAverageMin: 14,
+            averageMax: 13,
+            averageMin: 4,
+            belowAverageMax: 3,
+            belowAverageMin: 0,
+          },
+        },
+      },
+      { autoGenerateObjectIDIfNotExist: true },
+    );
+    expect(await mockIndex.saveObject).toHaveBeenCalledWith(
+      {
+        __meta: {
+          range: 'all',
+          type: 'team-collaboration-performance',
+          outputType: 'all',
+        },
         acrossTeam: {
           article: {
             aboveAverageMax: 50,

diff --git a/apps/crn-frontend/src/analytics/collaboration/Collaboration.tsx b/apps/crn-frontend/src/analytics/collaboration/Collaboration.tsx
@@ -12,7 +12,7 @@ const Collaboration = () => {
     metric: 'user' | 'team';
     type: 'within-team' | 'across-teams';
   }>();
-  const { timeRange } = useAnalytics();
+  const { timeRange, documentCategory, outputType } = useAnalytics();
   const { currentPage } = usePaginationParams();
 
   const setMetric = (newMetric: 'user' | 'team') =>
@@ -35,6 +35,8 @@ const Collaboration = () => {
       setMetric={setMetric}
       setType={setType}
       timeRange={timeRange}
+      outputType={metric === 'team' ? outputType : undefined}
+      documentCategory={metric === 'user' ? documentCategory : undefined}
       currentPage={currentPage}
     >
       {metric === 'user' ? (

diff --git a/apps/crn-frontend/src/analytics/collaboration/TeamCollaboration.tsx b/apps/crn-frontend/src/analytics/collaboration/TeamCollaboration.tsx
@@ -36,17 +36,21 @@ const getDataForType = (
 
 const TeamCollaboration: React.FC<CollaborationProps> = ({ type }) => {
   const { currentPage, pageSize } = usePaginationParams();
-  const { timeRange } = useAnalytics();
+  const { timeRange, outputType } = useAnalytics();
 
   const { items: data, total } = useAnalyticsTeamCollaboration({
     currentPage,
     pageSize,
     timeRange,
+    outputType,
     tags: [],
     sort: '',
   });
 
-  const performance = useTeamCollaborationPerformance({ timeRange });
+  const performance = useTeamCollaborationPerformance({
+    timeRange,
+    outputType,
+  });
 
   const { numberOfPages, renderPageHref } = usePagination(total, pageSize);