Speed up resources loading by using _changes after initial load (#2130)

src/app/resources/resources.component.ts

@@ -4,17 +4,15 @@ import { DialogsPromptComponent } from '../shared/dialogs/dialogs-prompt.compone
 import { MatTableDataSource, MatPaginator, MatSort, MatDialog, PageEvent } from '@angular/material';
 import { SelectionModel } from '@angular/cdk/collections';
 import { Router, ActivatedRoute } from '@angular/router';
-import { HttpClient } from '@angular/common/http';
 import { takeUntil, map, switchMap } from 'rxjs/operators';
 import { Subject, of } from 'rxjs';
 import { PlanetMessageService } from '../shared/planet-message.service';
 import { UserService } from '../shared/user.service';
-import { filterSpecificFields, composeFilterFunctions, filterArrayField, filterTags } from '../shared/table-helpers';
+import { filterSpecificFields, composeFilterFunctions, filterTags } from '../shared/table-helpers';
 import { ResourcesService } from './resources.service';
 import { environment } from '../../environments/environment';
 import { debug } from '../debug-operator';
 import { SyncService } from '../shared/sync.service';
-import { dedupeShelfReduce } from '../shared/utils';
 import { FormControl } from '../../../node_modules/@angular/forms';
 import { PlanetTagInputComponent } from '../shared/forms/planet-tag-input.component';
 
@@ -72,15 +70,15 @@ export class ResourcesComponent implements OnInit, AfterViewInit, OnDestroy {
     private dialog: MatDialog,
     private router: Router,
     private route: ActivatedRoute,
-    private httpclient: HttpClient,
     private planetMessageService: PlanetMessageService,
     private userService: UserService,
     private resourcesService: ResourcesService,
     private syncService: SyncService
   ) {}
 
   ngOnInit() {
-    this.resourcesService.resourcesUpdated$.pipe(takeUntil(this.onDestroy$)).pipe(
+    this.resourcesService.resourcesListener(this.parent).pipe(
+      takeUntil(this.onDestroy$),
       map((resources) => {
         // Sort in descending createdDate order, so the new resource can be shown on the top
         resources.sort((a, b) => b.createdDate - a.createdDate);
@@ -91,7 +89,7 @@ export class ResourcesComponent implements OnInit, AfterViewInit, OnDestroy {
       this.resources.data = resources;
       this.emptyData = !this.resources.data.length;
     });
-    this.resourcesService.updateResources({ opts: this.getOpts });
+    this.resourcesService.requestResourcesUpdate(this.parent);
     this.resources.filterPredicate = composeFilterFunctions(
       [ filterTags('tags', this.tagFilter), filterSpecificFields([ 'title' ]) ]
     );
@@ -156,27 +154,6 @@ export class ResourcesComponent implements OnInit, AfterViewInit, OnDestroy {
     this.resources.data.slice(start, end).forEach((row: any) => this.selection.select(row._id));
   }
 
-  // Keeping for reference.  Need to refactor for service.
-  /*
-  getExternalResources() {
-    return this.couchService.post('nations/_find',
-    { 'selector': { 'name': this.nationName },
-    'fields': [ 'name', 'nationurl' ] })
-      .pipe(switchMap(data => {
-        this.nationName = data.docs[0].name;
-        const nationUrl = data.docs[0].nationurl;
-        if (nationUrl) {
-          return this.httpclient.jsonp('http://' + nationUrl +
-            '/resources/_all_docs?include_docs=true&callback=JSONP_CALLBACK',
-            'callback'
-          );
-        }
-        // If there is no url, return an observable of an empty array
-        return of([]);
-    }));
-  }
-  */
-
   updateResource(resource) {
     const { _id: resourceId } = resource;
     this.router.navigate([ '/resources/update/' + resource._id ]);
@@ -236,7 +213,7 @@ export class ResourcesComponent implements OnInit, AfterViewInit, OnDestroy {
       });
       this.couchService.post(this.dbName + '/_bulk_docs', { docs: deleteArray })
         .subscribe((data) => {
-          this.resourcesService.updateResources({ opts: this.getOpts });
+          this.resourcesService.requestResourcesUpdate(this.parent);
           this.selection.clear();
           this.deleteDialog.close();
           this.planetMessageService.showMessage('You have deleted ' + deleteArray.length + ' resources');

src/app/resources/resources.service.ts

@@ -1,72 +1,80 @@
 import { Injectable } from '@angular/core';
 import { CouchService } from '../shared/couchdb.service';
 import { findDocuments } from '../shared/mangoQueries';
-import { Subject, forkJoin } from 'rxjs';
-import { switchMap, map } from 'rxjs/operators';
+import { Subject, forkJoin, of } from 'rxjs';
+import { map } from 'rxjs/operators';
 import { RatingService } from '../shared/forms/rating.service';
 import { UserService } from '../shared/user.service';
-import { dedupeShelfReduce } from '../shared/utils';
 import { PlanetMessageService } from '../shared/planet-message.service';
+import { ConfigurationService } from '../configuration/configuration.service';
 
 @Injectable()
 export class ResourcesService {
   private dbName = 'resources';
-  private resourcesUpdated = new Subject<any[]>();
-  resourcesUpdated$ = this.resourcesUpdated.asObservable();
-  private currentResources = [];
-  currentParams: any;
+  private resourcesUpdated = new Subject<any>();
+  resources = { local: [], parent: [] };
+  ratings = { local: [], parent: [] };
+  lastSeq = '';
+  isActiveResourceFetch = false;
 
   constructor(
     private couchService: CouchService,
     private ratingService: RatingService,
     private userService: UserService,
-    private planetMessageService: PlanetMessageService
+    private planetMessageService: PlanetMessageService,
+    private configurationService: ConfigurationService
   ) {
-    this.ratingService.ratingsUpdated$.pipe(switchMap(() => {
-      const { resourceIds, opts } = this.currentParams;
-      return this.getRatings(resourceIds, opts);
-    })).subscribe((ratings) => {
-      this.setResources(this.currentResources, ratings.docs, this.currentParams.updateCurrentResources);
+    this.ratingService.ratingsUpdated$.subscribe((res: any) => {
+      const planetField = res.parent ? 'parent' : 'local';
+      this.ratings[planetField] = res.ratings.filter((rating: any) => rating.type === 'resource');
+      if (!this.isActiveResourceFetch) {
+        this.setResources(this.resources[planetField], [], res.ratings, planetField);
+      }
     });
   }
 
-  updateResources({ resourceIds = [], opts = {}, updateCurrentResources = false }:
-    { resourceIds?: string[], opts?: any, updateCurrentResources?: boolean} = {}) {
-    this.currentParams = { resourceIds, opts, updateCurrentResources };
-    const resourceQuery = resourceIds.length > 0 ?
-      this.getResources(resourceIds, opts) : this.getAllResources(opts);
-    forkJoin(resourceQuery, this.getRatings(resourceIds, opts)).subscribe((results: any) => {
-      this.setResources(results[0].docs || results[0], results[1].docs, updateCurrentResources);
-    }, (err) => console.log(err));
+  resourcesListener(parent: boolean) {
+    return this.resourcesUpdated.pipe(
+      map((resources: any) => parent ? resources.parent : resources.local)
+    );
   }
 
-  setResources(resourcesRes, ratings, updateCurrentResources) {
-    const resources = this.createResourceList(resourcesRes, ratings);
-    if (updateCurrentResources && this.currentResources.length) {
-      this.currentResources.map((currentResource, cIndex) => {
-        resources.map(newResource => {
-          if (currentResource._id === newResource._id) {
-            this.currentResources[cIndex] = newResource;
-          }
-        });
-      });
-      this.resourcesUpdated.next(this.currentResources);
-      return;
-    }
-    this.currentResources = resources;
-    this.resourcesUpdated.next(resources);
+  requestResourcesUpdate(parent: boolean) {
+    const opts = parent ? { domain: this.configurationService.configuration.parentDomain } : {};
+    const currentResources = parent ?
+      this.resources.parent : this.resources.local;
+    const planetField = parent ? 'parent' : 'local';
+    const getCurrentResources = currentResources.length === 0 ?
+      this.getAllResources(opts) : of(currentResources);
+    this.isActiveResourceFetch = true;
+    forkJoin([ getCurrentResources, this.updateResourcesChanges(opts, parent) ])
+    .subscribe(([ resources, newResources ]) => {
+      this.isActiveResourceFetch = false;
+      this.setResources(resources, newResources, this.ratings[planetField], planetField);
+    });
+    this.ratingService.newRatings(parent);
+  }
+
+  setResources(currentResources, newResources, ratings, planetField) {
+    const resources = newResources.length > 0 ?
+      this.couchService.combineChanges(currentResources, newResources) : currentResources;
+    this.resources[planetField] = this.createResourceList(resources, ratings);
+    this.resourcesUpdated.next(this.resources);
   }
 
   getAllResources(opts: any) {
     return this.couchService.findAll(this.dbName, findDocuments({
       '_id': { '$gt': null }
-    }, [ '_id', '_rev', 'title', 'description', 'createdDate', 'tags', 'isDownloadable', 'filename' ], [], 1000), opts);
+    }, [], [], 1000), opts);
   }
 
-  getResources(resourceIds: string[], opts: any) {
-    return this.couchService.post(this.dbName + '/_find', findDocuments({
-      '_id': { '$in': resourceIds }
-    }, 0, [], 1000), opts);
+  updateResourcesChanges(opts: any, parent: boolean) {
+    return this.couchService
+    .get(this.dbName + '/_changes?include_docs=true&since=' + (this.lastSeq || 'now'), opts)
+    .pipe(map((res: any) => {
+      this.lastSeq = res.last_seq;
+      return res.results.map((r: any) => r.doc);
+    }));
   }
 
   getRatings(resourceIds: string[], opts: any) {

src/app/resources/view-resources/resources-view.component.ts

@@ -2,8 +2,6 @@ import { Component, OnInit, OnDestroy } from '@angular/core';
 import { CouchService } from '../../shared/couchdb.service';
 
 import { ActivatedRoute, ParamMap } from '@angular/router';
-import { DomSanitizer } from '@angular/platform-browser';
-import { Router } from '@angular/router';
 import { environment } from '../../../environments/environment';
 import { takeUntil } from 'rxjs/operators';
 import { Subject } from 'rxjs';
@@ -21,8 +19,6 @@ export class ResourcesViewComponent implements OnInit, OnDestroy {
   constructor(
     private couchService: CouchService,
     private route: ActivatedRoute,
-    private sanitizer: DomSanitizer,
-    private router: Router,
     private userService: UserService,
     private resourcesService: ResourcesService
   ) { }
@@ -53,16 +49,12 @@ export class ResourcesViewComponent implements OnInit, OnDestroy {
       .pipe(debug('Getting resource id from parameters'), takeUntil(this.onDestroy$))
       .subscribe((params: ParamMap) => {
         this.resourceId = params.get('id');
-        const getOpts: any = { resourceIds: [ this.resourceId ] };
-        if (this.parent) {
-          getOpts.opts = { domain: this.userService.getConfig().parentDomain };
-        }
         this.resourceActivity(this.resourceId, 'visit');
-        this.resourcesService.updateResources(getOpts);
+        this.resourcesService.requestResourcesUpdate(this.parent);
       }, error => console.log(error), () => console.log('complete getting resource id'));
-    this.resourcesService.resourcesUpdated$.pipe(takeUntil(this.onDestroy$))
-      .subscribe((resourceArr) => {
-        this.resource = resourceArr[0];
+    this.resourcesService.resourcesListener(this.parent).pipe(takeUntil(this.onDestroy$))
+      .subscribe((resources) => {
+        this.resource = resources.find((r: any) => r._id === this.resourceId);
         this.isUserEnrolled = this.userService.shelf.resourceIds.includes(this.resource._id);
       });
   }
@@ -93,8 +85,8 @@ export class ResourcesViewComponent implements OnInit, OnDestroy {
     this.fullView = this.fullView === 'on' ? 'off' : 'on';
   }
 
-  updateRating(itemId) {
-    this.resourcesService.updateResources({ resourceIds: [ itemId ], updateCurrentResources: true });
+  updateRating() {
+    this.resourcesService.requestResourcesUpdate(this.parent);
   }
 
   libraryToggle(resourceId, type) {

src/app/resources/view-resources/resources-viewer.component.ts

@@ -42,11 +42,11 @@ export class ResourcesViewerComponent implements OnChanges, OnDestroy {
 
   ngOnChanges() {
     if (this.resource === undefined || this.resource._id !== this.resourceId) {
-      this.resourcesService.resourcesUpdated$.pipe(takeUntil(this.onDestroy$))
-        .subscribe((resourceArr) => {
-          this.setResource(resourceArr[0]);
+      this.resourcesService.resourcesListener(this.parent).pipe(takeUntil(this.onDestroy$))
+        .subscribe((resources) => {
+          this.setResource(resources.find((r: any) => r._id === this.resourceId));
         });
-      this.resourcesService.updateResources({ resourceIds: [ this.resourceId ] });
+      this.resourcesService.requestResourcesUpdate(this.parent);
     } else {
       this.setResource(this.resource);
     }

src/app/shared/couchdb.service.ts

@@ -138,4 +138,18 @@ export class CouchService {
     return (local < parent) ? 'newerAvailable' : (local > parent) ? 'parentOlder' : 'mismatch';
   }
 
+  combineChanges(docs: any[], changesDocs: any[]) {
+    return docs.reduce((newDocs: any[], doc: any) => {
+      const changesDoc = changesDocs.find((cDoc: any) => doc._id === cDoc._id);
+      if (changesDoc && changesDoc._deleted === true) {
+        return newDocs;
+      }
+      newDocs.push(changesDoc !== undefined ? changesDoc : doc);
+      return newDocs;
+    }, []).concat(
+      changesDocs.filter((cDoc: any) =>
+        cDoc._deleted !== true && docs.findIndex((doc: any) => doc._id === cDoc._id) === -1)
+    );
+  }
+
 }

src/app/shared/forms/planet-rating.component.ts

@@ -115,7 +115,7 @@ export class PlanetRatingComponent implements OnChanges {
       newRating._rev = res.rev;
       newRating._id = res.id;
       this.rating.userRating = newRating;
-      this.ratingService.updateRatings();
+      this.ratingService.updateRatings(false);
       return res;
     }));
   }

src/app/shared/forms/rating.service.ts

@@ -4,22 +4,37 @@ import { findDocuments } from '../mangoQueries';
 import { UserService } from '../user.service';
 import { of, Subject } from 'rxjs';
 import { catchError } from 'rxjs/operators';
+import { ConfigurationService } from '../../configuration/configuration.service';
 
 const startingRating = { rateSum: 0, totalRating: 0, maleRating: 0, femaleRating: 0, userRating: {} };
 
 @Injectable()
 export class RatingService {
   private dbName = 'ratings';
-  private ratingsUpdated = new Subject<void>();
+  private ratingsUpdated = new Subject<any>();
   ratingsUpdated$ = this.ratingsUpdated.asObservable();
+  ratings: any[];
 
   constructor(
     private couchService: CouchService,
-    private userService: UserService
+    private userService: UserService,
+    private configurationService: ConfigurationService
   ) {}
 
-  updateRatings() {
-    this.ratingsUpdated.next();
+  updateRatings(parent: boolean) {
+    this.ratingsUpdated.next({ ratings: this.ratings, parent });
+  }
+
+  newRatings(parent: boolean) {
+    const opts = parent ? { domain: this.configurationService.configuration.parentDomain } : {};
+    this.couchService.findAll(this.dbName, undefined, opts).pipe(catchError(err => {
+      // If there's an error, return a fake couchDB empty response
+      // so resources can be displayed.
+      return of([]);
+    })).subscribe((res: any) => {
+      this.ratings = res;
+      this.ratingsUpdated.next({ ratings: res, parent });
+    });
   }
 
   getRatings({ itemIds, type }: {itemIds: string[], type: string}, opts: any) {