-
Notifications
You must be signed in to change notification settings - Fork 1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Basic TTL engine #4
base: master
Are you sure you want to change the base?
Changes from 5 commits
df39131
f21230c
ee304d9
f406159
8bfd8c3
6309ba9
31a5af2
06f18b8
baef528
470d477
70d13ea
961de50
56bd47d
a081f42
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,6 @@ | |
report | ||
tmp/ | ||
.env | ||
dist | ||
.cache | ||
.github | ||
.github | ||
docs |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# TTL cache engine Architecture | ||
|
||
![](./images/ttl-arct.png) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,91 @@ | ||
import DefaultHashTable from '../dataStructure/HashTable'; | ||
import DoublyLinkedList from '../dataStructure/DoublyLinkedList'; | ||
import * as hashTableProp from '../hashTableSymbol'; | ||
|
||
function TimeToLive({ HashTable = DefaultHashTable, defaultTTL } = {}) { | ||
const store = new HashTable(); | ||
const timeSeriesIndex = new HashTable(); | ||
const timeIndexInterval = 5 * 60 * 1000; // milliseconds. | ||
|
||
let lastRunGC = Date.now(); | ||
|
||
this.add = (key, value, ttl = defaultTTL) => { | ||
if (!ttl) | ||
benhurdavies marked this conversation as resolved.
Show resolved
Hide resolved
|
||
throw Error( | ||
'Expected ttl value. you can have to mention it in add method or mention as defaultTTL at constructor', | ||
); | ||
|
||
const expireTTL = Date.now() + ttl; | ||
const bucket = getTimeBucket(expireTTL); | ||
bucket.addFirst(key); | ||
const tNode = bucket.getFirst(); | ||
benhurdavies marked this conversation as resolved.
Show resolved
Hide resolved
|
||
const payload = { value, ttl: expireTTL, tNode }; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ttl is misleading here. Per definition https://en.wikipedia.org/wiki/Time_to_live is a timespan. expireTTL is a timestamp. So |
||
store[hashTableProp.add](key, payload); | ||
Comment on lines
+23
to
+24
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why buffe |
||
}; | ||
|
||
this.get = key => { | ||
const payload = store[hashTableProp.get](key); | ||
if (payload) { | ||
const { ttl, value } = payload; | ||
if (checkIfElementExpire(payload)) return undefined; | ||
benhurdavies marked this conversation as resolved.
Show resolved
Hide resolved
|
||
else return value; | ||
} | ||
return undefined; | ||
}; | ||
|
||
this.has = key => { | ||
return store[hashTableProp.has](key); | ||
}; | ||
|
||
this.remove = (key) => { | ||
if (this.has(key)) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
const { ttl, tNode } = store[hashTableProp.get](key); | ||
const timeBucket = getTimeBucket(ttl); | ||
timeBucket.remove(tNode); | ||
store[hashTableProp.remove](key); | ||
} | ||
}; | ||
|
||
this.size = () => { | ||
return store[hashTableProp.size](); | ||
}; | ||
|
||
function getTimeBucket(expireTTL) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the context of partitions this function should be called getTimePartition right? |
||
const timeIndex = getTimeIndex({ time: expireTTL, interval:timeIndexInterval }); | ||
|
||
if (timeSeriesIndex[hashTableProp.has](timeIndex)) { | ||
return timeSeriesIndex[hashTableProp.get](timeIndex); | ||
} else { | ||
const list = new DoublyLinkedList(); | ||
timeSeriesIndex[hashTableProp.add](timeIndex, list); | ||
return list; | ||
} | ||
} | ||
|
||
function checkIfElementExpire({ ttl }) { | ||
if (ttl < Date.now()) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Your wish for indentations :-) Why not |
||
const timeIndex = getTimeIndex({ time: ttl, interval:timeIndexInterval }); | ||
cleanExpired(timeIndex); | ||
return true; | ||
} | ||
return false; | ||
} | ||
|
||
function cleanExpired(timeIndex) { | ||
const keys = timeSeriesIndex[hashTableProp.get][timeIndex]; | ||
for (key of keys) { | ||
store[hashTableProp.remove](key); | ||
} | ||
} | ||
} | ||
|
||
// time : unix timestamp milliseconds | ||
// interval : milliseconds (better to be factors of 60 (minutes)) | ||
function getTimeIndex({ time, interval }) { | ||
const timeParts = parseInt(time / interval, 10); | ||
const forwardIndexTime = timeParts * interval + interval; | ||
return forwardIndexTime; | ||
} | ||
|
||
export { getTimeIndex }; | ||
export default TimeToLive; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
import TimeToLive, { getTimeIndex } from './TimeToLive'; | ||
|
||
describe('TimeToLive (TTL) : getTimeIndex', () => { | ||
it('should index time to next upcoming interval', () => { | ||
const toMinute = val => val * 60 * 1000; | ||
|
||
const date1 = new Date('2020-06-14T03:23:34'); | ||
expect(getTimeIndex({ time: date1.getTime(), interval: toMinute(5) })).toBe( | ||
new Date('2020-06-14T03:25:00').getTime(), | ||
); | ||
|
||
const date2 = new Date('2020-06-14T03:43:36'); | ||
expect( | ||
getTimeIndex({ time: date2.getTime(), interval: toMinute(10) }), | ||
).toBe(new Date('2020-06-14T03:50:00').getTime()); | ||
|
||
const date3 = new Date('2020-06-14T03:43:36'); | ||
expect(getTimeIndex({ time: date3.getTime(), interval: toMinute(3) })).toBe( | ||
new Date('2020-06-14T03:45:00').getTime(), | ||
); | ||
}); | ||
}); | ||
|
||
describe('TimeToLive', () => { | ||
it('should have basic cache features', () => { | ||
const ttlCache = new TimeToLive(); | ||
ttlCache.add('apple',5,1000); | ||
ttlCache.add('orange',2,2000); | ||
expect(ttlCache.get('apple')).toBe(5); | ||
expect(ttlCache.has('apple')).toBe(true); | ||
expect(ttlCache.size()).toBe(2); | ||
|
||
ttlCache.remove('apple'); | ||
expect(ttlCache.size()).toBe(1); | ||
expect(ttlCache.get('apple')).toBe(undefined); | ||
expect(ttlCache.get('orange')).toBe(2); | ||
}); | ||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just something we might want to consider. When we remove this ttl and only have the default ttl we have more performance optimization potential.