Merge pull request bitpay#555 from bitpay/revert-553-revert-548-opt/t…

…xlist-cache

add txIdList cache

lib/services/address/index.js

@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
+var LRU = require('lru-cache');
+var XXHash = require('xxhash');
+
+
+
+// See rationale about this cache at function getTxList(next)
+const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
+                                                // clients downloading their tx history)
+const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
+const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
+const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -24,6 +35,11 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
+  this._txIdListCache = new LRU({
+    max: TXID_LIST_CACHE_ITEMS,        
+    maxAge: TXID_LIST_CACHE_EXPIRATION 
+  });
+
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
+//
+// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
+//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
+  var cacheUsed = false;
 
   options = options || {};
   options.from = options.from || 0;
@@ -65,21 +85,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
-  async.eachLimit(addresses, 4, function(address, next) {
-    self._getAddressTxidHistory(address, options, next);
 
-  }, function(err) {
+  function getTxList(next) {
 
-    if(err) {
-      return callback(err);
+
+    function hashAddresses(addresses) {
+
+      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
+      // a 32 bits hash is secure enough
+
+      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
+    };
+
+    var calculatedCacheKey;
+
+    // We use the cache ONLY on from > 0 queries.
+    //
+    // Rationale: The a full history is downloaded, the client do
+    // from =0, to=x
+    // then from =x+1 to=y
+    // then [...]
+    // The objective of this cache is to speed up the from>0 queries, and also
+    // "freeze" the txid list during download.
+    //
+    if (options.from >0 ) {
+
+      let cacheKey  = options.cacheKey;
+      if (!cacheKey) {
+         calculatedCacheKey = hashAddresses(addresses);
+         cacheKey =  calculatedCacheKey;
+      }
+
+      var txIdList  = self._txIdListCache.get(cacheKey);
+      if (txIdList) {
+        options.txIdList = txIdList;
+        cacheUsed = true;
+        return next();
+      }
     }
 
-    var list = lodash.uniqBy(options.txIdList, function(x) { 
-      return x.txid + x.height;
+    // Get the list from the db
+    async.eachLimit(addresses, 4, function(address, next) {
+      self._getAddressTxidHistory(address, options, next);
+    }, function(err) {
+      if (err) return next(err);
+
+      var list = lodash.uniqBy(options.txIdList, function(x) { 
+        return x.txid + x.height;
+      });
+
+
+      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
+
+      if (list.length > TXID_LIST_CACHE_MIN) {
+        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
+
+        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
+      }
+
+      return next();
     });
 
+  };
+
+
+  getTxList(function(err) {
+    if(err) {
+      return callback(err);
+    }
 
-    options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -88,10 +162,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList
+        items: txList,
       };
 
-      callback(null, results);
+      // cacheUsed is returned for testing
+      callback(null, results, cacheUsed);
 
     });
   });

package.json

@@ -43,12 +43,13 @@
     "levelup": "^2.0.0",
     "liftoff": "^2.2.0",
     "lodash": "^4.17.4",
-    "lru-cache": "^4.0.2",
+    "lru-cache": "^4.1.1",
     "memwatch-next": "^0.3.0",
     "mkdirp": "0.5.0",
     "path-is-absolute": "^1.0.0",
     "socket.io": "^1.4.5",
-    "socket.io-client": "^1.4.5"
+    "socket.io-client": "^1.4.5",
+    "xxhash": "^0.2.4"
   },
   "devDependencies": {
     "chai": "^3.5.0",