From 444502bc1b1f5811c69e34687aad3cbeff85d946 Mon Sep 17 00:00:00 2001 From: chanchiem Date: Thu, 1 Aug 2019 18:26:18 -0700 Subject: [PATCH 1/4] Clean up on length call to give accurate dictionary length based on expiration --- expiringdict/__init__.py | 13 +++++++++++ tests/expiringdict_test.py | 45 +++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/expiringdict/__init__.py b/expiringdict/__init__.py index 56ba71d..ab12eaf 100755 --- a/expiringdict/__init__.py +++ b/expiringdict/__init__.py @@ -55,6 +55,19 @@ def __init__(self, max_len, max_age_seconds, items=None): else: raise ValueError('can not unpack items') + def __len__(self): + current_key = iter(self) + for k in current_key: + item = OrderedDict.__getitem__(self, k) + time_added = item[1] + item_age = time.time() - time_added + if item_age > self.max_age: + del self[k] + else: + break + + return super(ExpiringDict, self).__len__() + def __contains__(self, key): """ Return True if the dict has a key, else return False. """ try: diff --git a/tests/expiringdict_test.py b/tests/expiringdict_test.py index 78929f1..bf17a39 100644 --- a/tests/expiringdict_test.py +++ b/tests/expiringdict_test.py @@ -65,7 +65,7 @@ def test_repr(): d['a'] = 'x' eq_(str(d), "ExpiringDict([('a', 'x')])") sleep(0.01) - eq_(str(d), "ExpiringDict([])") + eq_(str(d), "ExpiringDict()") def test_iter(): @@ -122,3 +122,46 @@ def test_not_implemented(): assert_raises(NotImplementedError, d.viewitems) assert_raises(NotImplementedError, d.viewkeys) assert_raises(NotImplementedError, d.viewvalues) + + +def test_cleanup_on_length(): + d = ExpiringDict(max_len=10, max_age_seconds=0.01) + d['a'] = 'x' + eq_(1, len(d)) + sleep(0.01) + eq_(0, len(d)) + + d = ExpiringDict(max_len=10, max_age_seconds=0.02) + d['a'] = 1 + sleep(0.01) + eq_(1, len(d)) + d['b'] = 2 + sleep(0.01) + eq_(1, len(d)) + d['c'] = 3 + sleep(0.01) + eq_(1, len(d)) + d['d'] = 4 + sleep(0.01) + eq_(1, len(d)) + + d = ExpiringDict(max_len=1000, max_age_seconds=.5) + d[1] = 1 + d[2] = 2 + d[3] = 3 + d[4] = 4 + + ok_(1 in d) + ok_(2 in d) + ok_(3 in d) + ok_(4 in d) + eq_(4, len(d)) + + sleep(.25) + d[1] = 1 + d[2] = 2 + sleep(.40) + ok_(1 in d) + ok_(2 in d) + ok_(3 not in d) + ok_(4 not in d) From 4f4b2df8db604a1651d2f5ce0ff2c0898b91ff94 Mon Sep 17 00:00:00 2001 From: chanchiem Date: Thu, 1 Aug 2019 21:18:42 -0700 Subject: [PATCH 2/4] Added lock to the length method --- expiringdict/__init__.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/expiringdict/__init__.py b/expiringdict/__init__.py index ab12eaf..ad27546 100755 --- a/expiringdict/__init__.py +++ b/expiringdict/__init__.py @@ -56,15 +56,16 @@ def __init__(self, max_len, max_age_seconds, items=None): raise ValueError('can not unpack items') def __len__(self): - current_key = iter(self) - for k in current_key: - item = OrderedDict.__getitem__(self, k) - time_added = item[1] - item_age = time.time() - time_added - if item_age > self.max_age: - del self[k] - else: - break + with self.lock: + current_key = iter(self) + for k in current_key: + item = OrderedDict.__getitem__(self, k) + time_added = item[1] + item_age = time.time() - time_added + if item_age > self.max_age: + del self[k] + else: + break return super(ExpiringDict, self).__len__() From cea1c3a167eabaf412a16ad4eb0967780f72093b Mon Sep 17 00:00:00 2001 From: chanchiem Date: Thu, 1 Aug 2019 21:26:09 -0700 Subject: [PATCH 3/4] Delete keys after iterating to prevent mutation during iter --- expiringdict/__init__.py | 5 ++++- tests/expiringdict_test.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/expiringdict/__init__.py b/expiringdict/__init__.py index ad27546..b869190 100755 --- a/expiringdict/__init__.py +++ b/expiringdict/__init__.py @@ -58,14 +58,17 @@ def __init__(self, max_len, max_age_seconds, items=None): def __len__(self): with self.lock: current_key = iter(self) + keys_to_del = [] for k in current_key: item = OrderedDict.__getitem__(self, k) time_added = item[1] item_age = time.time() - time_added if item_age > self.max_age: - del self[k] + keys_to_del.append(k) else: break + for k in keys_to_del: + del self[k] return super(ExpiringDict, self).__len__() diff --git a/tests/expiringdict_test.py b/tests/expiringdict_test.py index bf17a39..6663981 100644 --- a/tests/expiringdict_test.py +++ b/tests/expiringdict_test.py @@ -65,7 +65,7 @@ def test_repr(): d['a'] = 'x' eq_(str(d), "ExpiringDict([('a', 'x')])") sleep(0.01) - eq_(str(d), "ExpiringDict()") + eq_(str(d), "ExpiringDict([])") def test_iter(): From cd318dd6781d771a9a526d4ebb25e63e6781d6ee Mon Sep 17 00:00:00 2001 From: chanchiem Date: Thu, 1 Aug 2019 21:33:33 -0700 Subject: [PATCH 4/4] Added weird edge cause with __repr__ method. Python 3.6 had diff represention for empty ordered dict --- expiringdict/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/expiringdict/__init__.py b/expiringdict/__init__.py index b869190..dc8ee00 100755 --- a/expiringdict/__init__.py +++ b/expiringdict/__init__.py @@ -72,6 +72,11 @@ def __len__(self): return super(ExpiringDict, self).__len__() + def __repr__(self): + if len(self) == 0: + return "ExpiringDict([])" + return super(ExpiringDict, self).__repr__() + def __contains__(self, key): """ Return True if the dict has a key, else return False. """ try: