diff --git a/CHANGES.rst b/CHANGES.rst index b0ed6f9..a19df23 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,9 @@ Changelog - Drop support for Python 3.7. +- Support lazy batching again, support general iterators + (`#75 `_) + 4.6 (2023-11-13) ---------------- diff --git a/src/DocumentTemplate/DT_In.py b/src/DocumentTemplate/DT_In.py index de81b21..820bd37 100644 --- a/src/DocumentTemplate/DT_In.py +++ b/src/DocumentTemplate/DT_In.py @@ -347,6 +347,7 @@ from .DT_Util import add_with_prefix from .DT_Util import name_param from .DT_Util import parse_params +from .DT_Util import sequence_ensure_subscription from .DT_Util import simple_name @@ -459,25 +460,25 @@ def renderwb(self, md): expr = self.expr name = self.__name__ if expr is None: - sequence = md[name] + sequence = sequence_ensure_subscription(md[name]) cache = {name: sequence} else: - sequence = expr(md) + sequence = sequence_ensure_subscription(expr(md)) cache = None - if not sequence: - if self.elses: - return render_blocks(self.elses, md, encoding=self.encoding) - return '' - if isinstance(sequence, str): raise ValueError( 'Strings are not allowed as input to the in tag.') - # Turn iterable like dict.keys() into a list. - sequence = list(sequence) - if cache is not None: - cache[name] = sequence + # below we do not use ``not sequence`` because the + # implied ``__len__`` is expensive for some (lazy) sequences + # if not sequence: + try: + sequence[0] + except IndexError: + if self.elses: + return render_blocks(self.elses, md, encoding=self.encoding) + return '' section = self.section params = self.args @@ -667,25 +668,25 @@ def renderwob(self, md): expr = self.expr name = self.__name__ if expr is None: - sequence = md[name] + sequence = sequence_ensure_subscription(md[name]) cache = {name: sequence} else: - sequence = expr(md) + sequence = sequence_ensure_subscription(expr(md)) cache = None - if not sequence: - if self.elses: - return render_blocks(self.elses, md, encoding=self.encoding) - return '' - if isinstance(sequence, str): raise ValueError( 'Strings are not allowed as input to the in tag.') - # Turn iterable like dict.keys() into a list. - sequence = list(sequence) - if cache is not None: - cache[name] = sequence + # below we do not use ``not sequence`` because the + # implied ``__len__`` is expensive for some (lazy) sequences + # if not sequence: + try: + sequence[0] + except IndexError: + if self.elses: + return render_blocks(self.elses, md, encoding=self.encoding) + return '' section = self.section mapping = self.mapping diff --git a/src/DocumentTemplate/DT_InSV.py b/src/DocumentTemplate/DT_InSV.py index 5957522..03760d5 100644 --- a/src/DocumentTemplate/DT_InSV.py +++ b/src/DocumentTemplate/DT_InSV.py @@ -17,6 +17,8 @@ import roman +from .DT_Util import sequence_ensure_subscription + try: import Missing @@ -37,8 +39,7 @@ def __init__(self, start_name_re=None, alt_prefix=''): if items is not None: - # Turn iterable into a list, to support key lookup - items = list(items) + items = sequence_ensure_subscription(items) self.items = items self.query_string = query_string self.start_name_re = start_name_re diff --git a/src/DocumentTemplate/DT_Util.py b/src/DocumentTemplate/DT_Util.py index 7bf3b41..ab7947f 100644 --- a/src/DocumentTemplate/DT_Util.py +++ b/src/DocumentTemplate/DT_Util.py @@ -1,6 +1,6 @@ ############################################################################## # -# Copyright (c) 2002 Zope Foundation and Contributors. +# Copyright (c) 2002-2024 Zope Foundation and Contributors. # # This software is subject to the provisions of the Zope Public License, # Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. @@ -450,3 +450,67 @@ def parse_params(text, return parse_params(text, result, **parms) else: return result + + +def sequence_supports_subscription(obj): + """Check whether *obj* supports sequence subscription. + + We are using a heuristics. + """ + # check wether *obj* might support sequence subscription + try: + obj[0] + except IndexError: + return True + except (AttributeError, TypeError, KeyError): + return False + # check that *obj* is not a mapping + try: + obj[None] + except (TypeError, ValueError, RuntimeError): + # we may need more exceptions above + # (to support sequence like objects using strange exceptions) + return True + except KeyError: + pass + return False + + +def sequence_ensure_subscription(obj): + """return an *obj* wrapper supporting sequence subscription. + + *obj* must either support sequence subscription itself + (and then is returned unwrapped) or be iterable. + """ + if sequence_supports_subscription(obj): + return obj + return SequenceFromIter(iter(obj)) + + +class SequenceFromIter: + """Iterator wrapper supporting lazy sequence subscription.""" + + finished = False + + def __init__(self, it): + self.it = it + self.data = [] + + def __getitem__(self, idx): + if idx < 0: + raise IndexError(f"negative indexes are not supported {idx}") + while not self.finished and idx >= len(self.data): + try: + self.data.append(next(self.it)) + except StopIteration: + self.finished = True + return self.data[idx] + + def __len__(self): + """the size -- ATT: expensive!""" + while not self.finished: + try: + self[len(self.data)] + except IndexError: + pass + return len(self.data) diff --git a/src/DocumentTemplate/tests/test_Util.py b/src/DocumentTemplate/tests/test_Util.py new file mode 100644 index 0000000..cc7434a --- /dev/null +++ b/src/DocumentTemplate/tests/test_Util.py @@ -0,0 +1,67 @@ +from unittest import TestCase + +from ..DT_Util import SequenceFromIter +from ..DT_Util import sequence_ensure_subscription +from ..DT_Util import sequence_supports_subscription + + +class SequenceTests(TestCase): + def test_supports_str(self): + self.assertTrue(sequence_supports_subscription("")) + + def test_supports_sequence(self): + self.assertTrue(sequence_supports_subscription([])) + self.assertTrue(sequence_supports_subscription([0])) + + def test_supports_mapping(self): + self.assertFalse(sequence_supports_subscription({})) + self.assertFalse(sequence_supports_subscription({0: 0})) + self.assertFalse(sequence_supports_subscription({0: 0, None: None})) + + def test_supports_iter(self): + self.assertFalse(sequence_supports_subscription((i for i in range(0)))) + self.assertFalse(sequence_supports_subscription((i for i in range(1)))) + + def test_supports_SequenceFromIter(self): + S = SequenceFromIter + self.assertTrue( + sequence_supports_subscription(S((i for i in range(0))))) + self.assertTrue( + sequence_supports_subscription(S((i for i in range(1))))) + + def test_supports_RuntimeError(self): + # check that ``ZTUtils.Lazy.Lazy`` is recognized + class RTSequence(list): + def __getitem__(self, idx): + if not isinstance(idx, int): + raise RuntimeError + + s = RTSequence(i for i in range(0)) + self.assertTrue(sequence_supports_subscription(s)) + s = RTSequence(i for i in range(2)) + self.assertTrue(sequence_supports_subscription(s)) + + def test_ensure_sequence(self): + s = [] + self.assertIs(s, sequence_ensure_subscription(s)) + + def test_ensure_iter(self): + self.assertIsInstance( + sequence_ensure_subscription(i for i in range(0)), + SequenceFromIter) + + def test_FromIter(self): + S = SequenceFromIter + with self.assertRaises(IndexError): + S(i for i in range(0))[0] + s = S(i for i in range(2)) + with self.assertRaises(IndexError): + s[-1] + self.assertEqual(s[0], 0) + self.assertEqual(s[0], 0) # ensure nothing bad happens + self.assertEqual(s[1], 1) + with self.assertRaises(IndexError): + s[2] + self.assertEqual(list(s), [0, 1]) + self.assertEqual(len(s), 2) + self.assertEqual(len(S(i for i in range(2))), 2)