some major updates

- rework logger implementation
- rework threadpool implementation
- migrate proxyobject into its own module

lazyops/libs/persistence/backends/base.py

@@ -10,7 +10,8 @@
 from typing import TypeVar, Generic, Any, Dict, Optional, Union, Iterable, List, Type, ItemsView, TYPE_CHECKING
 from lazyops.utils.helpers import  create_unique_id
 from lazyops.utils.logs import logger
-from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+# from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+from lazyops.utils.pooler import ThreadPooler
 from ..serializers import get_serializer, SerializerT
 
 if TYPE_CHECKING:

lazyops/libs/persistence/compression/base.py

@@ -7,7 +7,8 @@
 import abc
 
 from lazyops.utils.logs import logger
-from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+# from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+from lazyops.utils.pooler import ThreadPooler
 from typing import Any, Optional, Union, Dict, TypeVar
 
 

lazyops/libs/persistence/main.py

@@ -8,7 +8,8 @@
 import collections.abc
 from lazyops.utils.lazy import lazy_import, get_keydb_enabled
 from lazyops.utils.logs import logger, null_logger
-from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+# from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+from lazyops.utils.pooler import ThreadPooler
 
 from typing import Any, Dict, Optional, Union, Iterable, List, Type, TYPE_CHECKING
 from .backends import LocalStatefulBackend, RedisStatefulBackend, StatefulBackendT

lazyops/libs/persistence/serializers/base.py

@@ -9,7 +9,8 @@
 import hashlib
 from lazyops.types import BaseModel
 from lazyops.utils.logs import logger
-from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+# from lazyops.utils.pooler import ThreadPoolV2 as ThreadPooler
+from lazyops.utils.pooler import ThreadPooler
 from typing import Any, Optional, Union, Dict, TypeVar, TYPE_CHECKING
 
 try:

lazyops/libs/proxyobj/__init__.py

@@ -0,0 +1,5 @@
+"""
+Simple Proxied Objects
+"""
+
+from .main import ProxyObject, ProxyObjT

lazyops/libs/proxyobj/main.py

@@ -0,0 +1,90 @@
+
+import threading
+import contextlib
+from typing import Any, Type, Tuple, Dict, List, Union, Optional, Callable, TypeVar, Generic, TYPE_CHECKING
+
+
+ProxyObjT = TypeVar('ProxyObjT')
+
+
+class ProxyObject(Generic[ProxyObjT]):
+    def __init__(
+        self,
+        obj_cls: Optional[Type[ProxyObjT]] = None,
+        obj_getter: Optional[Union[Callable, str]] = None,
+        obj_args: Optional[List[Any]] = None,
+        obj_kwargs: Optional[Dict[str, Any]] = None,
+        threadsafe: Optional[bool] = False,
+        debug_enabled: Optional[bool] = False,
+    ):
+        """
+        Proxy Object
+
+        args:
+            obj_cls: the class of the object
+            obj_getter: the function to get the object
+            debug_enabled: if True, will raise an error if the object is not found
+        """
+        # Intentionally underscore on the end to avoid conflicts with the settings
+        assert obj_cls or obj_getter, "Either `obj_cls` or `obj_getter` must be provided"
+        self.__obj_cls_ = obj_cls
+        self.__obj_getter_ = obj_getter
+        if self.__obj_getter_ and isinstance(self.__obj_getter_, str):
+            from lazyops.utils.helpers import lazy_import
+            self.__obj_getter_ = lazy_import(self.__obj_getter_)
+        self.__threadlock_ = None if threadsafe else threading.Lock()
+        self.__obj_ = None
+        self.__obj_args_ = obj_args or []
+        self.__obj_kwargs_ = obj_kwargs or {}
+        self.__debug_enabled_ = debug_enabled
+        self.__last_attrs_: Dict[str, int] = {}
+
+    @contextlib.contextmanager
+    def _objlock_(self):
+        """
+        Returns the object lock
+        """
+        if self.__threadlock_ is not None:
+            try:
+                with self.__threadlock_:
+                    yield
+            except Exception as e:
+                raise e
+        else:
+            yield
+
+    @property
+    def _obj_(self) -> ProxyObjT:
+        """
+        Returns the object
+        """
+        if self.__obj_ is None:
+            with self._objlock_():
+                if self.__obj_getter_:
+                    self.__obj_ = self.__obj_getter_(*self.__obj_args_, **self.__obj_kwargs_)
+                elif self.__obj_cls_:
+                    self.__obj_ = self.__obj_cls_(*self.__obj_args_, **self.__obj_kwargs_)
+        return self.__obj_
+
+    def __getattr__(self, name):
+        """
+        Forward all unknown attributes to the proxy object
+        """
+        if not self.__debug_enabled_:
+            return getattr(self._obj_, name)
+
+        # Try to debug the attribute
+        if name not in self.__last_attrs_:
+            self.__last_attrs_[name] = 0
+        self.__last_attrs_[name] += 1
+        if self.__last_attrs_[name] > 5:
+            raise AttributeError(f"Proxy object has no attribute {name}")
+
+        if hasattr(self._obj_, name):
+            self.__last_attrs_[name] = 0
+            return getattr(self._obj_, name)
+        raise AttributeError(f"Settings object has no attribute {name}")
+
+    if TYPE_CHECKING:
+        def __new__(cls, *args, **kwargs) -> ProxyObjT:
+            ...

lazyops/types/models.py

@@ -513,80 +513,6 @@ def parse(
         return super().parse(url = url, scheme = scheme, adapter = adapter, **config)
 
 
-ProxyObjT = TypeVar('ProxyObjT')
-
-
-class ProxyObject(Generic[ProxyObjT]):
-    def __init__(
-        self,
-        obj_cls: Optional[Type[ProxyObjT]] = None,
-        obj_getter: Optional[Union[Callable, str]] = None,
-        obj_args: Optional[List[Any]] = None,
-        obj_kwargs: Optional[Dict[str, Any]] = None,
-        debug_enabled: Optional[bool] = False,
-    ):
-        """
-        Proxy Object
-
-        args:
-            obj_cls: the class of the object
-            obj_getter: the function to get the object
-            debug_enabled: if True, will raise an error if the object is not found
-        """
-        # Intentionally underscore on the end to avoid conflicts with the settings
-        assert obj_cls or obj_getter, "Either `obj_cls` or `obj_getter` must be provided"
-        self.__obj_cls_ = obj_cls
-        self.__obj_getter_ = obj_getter
-        if self.__obj_getter_ and isinstance(self.__obj_getter_, str):
-            from lazyops.utils.helpers import lazy_import
-            self.__obj_getter_ = lazy_import(self.__obj_getter_)
-        self.__obj_ = None
-        self.__obj_args_ = obj_args or []
-        self.__obj_kwargs_ = obj_kwargs or {}
-        self.__debug_enabled_ = debug_enabled
-        self.__last_attrs_: Dict[str, int] = {}
-
-    @property
-    def _obj_(self) -> ProxyObjT:
-        """
-        Returns the object
-        """
-        if self.__obj_ is None:
-            if self.__obj_getter_:
-                self.__obj_ = self.__obj_getter_(*self.__obj_args_, **self.__obj_kwargs_)
-            elif self.__obj_cls_:
-                self.__obj_ = self.__obj_cls_(*self.__obj_args_, **self.__obj_kwargs_)
-        return self.__obj_
-
-    def __getattr__(self, name):
-        """
-        Forward all unknown attributes to the proxy object
-        """
-        if not self.__debug_enabled_:
-            return getattr(self._obj_, name)
-
-        # Try to debug the attribute
-        if name not in self.__last_attrs_:
-            self.__last_attrs_[name] = 0
-        self.__last_attrs_[name] += 1
-        if self.__last_attrs_[name] > 5:
-            raise AttributeError(f"Proxy object has no attribute {name}")
-
-        if hasattr(self._obj_, name):
-            self.__last_attrs_[name] = 0
-            return getattr(self._obj_, name)
-        raise AttributeError(f"Settings object has no attribute {name}")
-
-    if TYPE_CHECKING:
-        def __new__(cls, *args, **kwargs) -> ProxyObjT:
-            ...
-
-
-    # def __setattr__(self, name: str, value: Any) -> None:
-    #     """
-    #     Forward all unknown attributes to the proxy object
-    #     """
-    #     return setattr(self._obj_, name, value)