From 3ee56a2e9b4ccda1d1a549a15d2a777ae61c7dd5 Mon Sep 17 00:00:00 2001 From: Mikael Koli Date: Tue, 12 Jul 2022 12:02:37 +0300 Subject: [PATCH] upd: Changed Pandas references to native Made alternatives to pd.Interval, pd.Timestamp and pd.Timedelta --- rocketry/core/task.py | 10 +-- rocketry/core/time/anchor.py | 30 ++++----- rocketry/core/time/base.py | 65 +++++++++---------- rocketry/pybox/query/base.py | 7 +- rocketry/pybox/time/__init__.py | 3 +- rocketry/pybox/time/convert.py | 34 +++++++++- rocketry/pybox/time/interval.py | 50 ++++++++++++++ rocketry/session.py | 4 +- rocketry/test/condition/task/test_time.py | 13 +--- .../condition/task/test_time_executable.py | 12 ++-- .../condition/task/test_time_optimized.py | 14 ++-- rocketry/test/helpers/log_helpers.py | 15 ++--- rocketry/test/pybox/test_convert.py | 27 ++++++++ rocketry/test/session/test_logs.py | 14 ++-- rocketry/test/time/delta/test_construct.py | 8 +-- rocketry/test/time/delta/test_roll.py | 18 ++--- .../time/interval/timeofweek/test_core.py | 6 +- rocketry/test/time/test_contains.py | 6 +- rocketry/time/interval.py | 10 +-- 19 files changed, 214 insertions(+), 132 deletions(-) create mode 100644 rocketry/pybox/time/interval.py create mode 100644 rocketry/test/pybox/test_convert.py diff --git a/rocketry/core/task.py b/rocketry/core/task.py index eb02c466..2edb94ee 100644 --- a/rocketry/core/task.py +++ b/rocketry/core/task.py @@ -14,7 +14,6 @@ import threading from queue import Empty -import pandas as pd from pydantic import BaseModel, Field, PrivateAttr, validator from rocketry._base import RedBase @@ -22,6 +21,7 @@ from rocketry.core.time import TimePeriod from rocketry.core.parameters import Parameters from rocketry.core.log import TaskAdapter +from rocketry.core.time.utils import to_timedelta from rocketry.core.utils import is_pickleable, filter_keyword_args, is_main_subprocess from rocketry.exc import SchedulerRestart, SchedulerExit, TaskInactionException, TaskTerminationException from rocketry.core.meta import _register @@ -91,7 +91,7 @@ class Task(RedBase, BaseModel): >= 40 if they require loaded tasks, >= 50 if they require loaded extensions. By default 0 - timeout : str, int, pd.Timedelta, optional + timeout : str, int, timedelta, optional If the task has not run in given timeout the task will be terminated. Only applicable for tasks with execution='process' or @@ -165,7 +165,7 @@ class Config: force_run: bool = False force_termination: bool = False status: Optional[Literal['run', 'fail', 'success', 'terminate', 'inaction']] = Field(description="Latest status of the task") - timeout: Optional[pd.Timedelta] + timeout: Optional[datetime.timedelta] parameters: Parameters = Parameters() @@ -222,9 +222,9 @@ def parse_logger_name(cls, value, values): @validator('timeout', pre=True, always=True) def parse_timeout(cls, value, values): if value == "never": - return pd.Timedelta.max.to_pytimedelta() + return datetime.timedelta.max elif value is not None: - return pd.Timedelta(value).to_pytimedelta() + return to_timedelta(value) else: return value diff --git a/rocketry/core/time/anchor.py b/rocketry/core/time/anchor.py index 2175f963..3ee4d724 100644 --- a/rocketry/core/time/anchor.py +++ b/rocketry/core/time/anchor.py @@ -4,9 +4,7 @@ from typing import Union from abc import abstractmethod -import pandas as pd - -from .utils import to_nanoseconds, timedelta_to_str, to_dict +from .utils import to_nanoseconds, timedelta_to_str, to_dict, to_timedelta from .base import TimeInterval @@ -76,7 +74,7 @@ def anchor_dict(self, d, **kwargs): kwargs = {key: val for key, val in d.items() if key in comps} return to_nanoseconds(**kwargs) - def anchor_dt(self, dt: Union[datetime, pd.Timestamp], **kwargs) -> int: + def anchor_dt(self, dt: datetime, **kwargs) -> int: "Turn datetime to nanoseconds according to the scope (by removing higher time elements)" components = self.components components = components[components.index(self._scope) + 1:] @@ -113,7 +111,7 @@ def set_end(self, val, time_point=False): @property def start(self): - delta = pd.Timedelta(self._start, unit="ns") + delta = to_timedelta(self._start, unit="ns") repr_scope = self.components[self.components.index(self._scope) + 1] return timedelta_to_str(delta, default_scope=repr_scope) @@ -123,7 +121,7 @@ def start(self, val): @property def end(self): - delta = pd.Timedelta(self._end, unit="ns") + delta = to_timedelta(self._end, unit="ns") repr_scope = self.components[self.components.index(self._scope) + 1] return timedelta_to_str(delta, default_scope=repr_scope) @@ -198,7 +196,7 @@ def next_start(self, dt): # dt # -->----------<----------->--------------<- # start | end start | end - offset = pd.Timedelta(int(ns_start) - int(ns), unit="ns") + offset = to_timedelta(int(ns_start) - int(ns), unit="ns") else: # not in period, later than start # dt @@ -215,7 +213,7 @@ def next_start(self, dt): # --<---------->-----------<-------------->-- # end | start end | start ns_scope = self.get_scope_forward(dt) - offset = pd.Timedelta(int(ns_start) - int(ns) + ns_scope, unit="ns") + offset = to_timedelta(int(ns_start) - int(ns) + ns_scope, unit="ns") return dt + offset def next_end(self, dt): @@ -239,7 +237,7 @@ def next_end(self, dt): # dt # --<---------->-----------<-------------->-- # end | start end | start - offset = pd.Timedelta(int(ns_end) - int(ns), unit="ns") + offset = to_timedelta(int(ns_end) - int(ns), unit="ns") else: # not in period, over night # dt @@ -256,7 +254,7 @@ def next_end(self, dt): # -->----------<----------->--------------<- # start | end start | end ns_scope = self.get_scope_forward(dt) - offset = pd.Timedelta(int(ns_end) - int(ns) + ns_scope, unit="ns") + offset = to_timedelta(int(ns_end) - int(ns) + ns_scope, unit="ns") return dt + offset def prev_start(self, dt): @@ -281,7 +279,7 @@ def prev_start(self, dt): # -->----------<----------->--------------<- # start | end start | end ns_scope = self.get_scope_back(dt) - offset = pd.Timedelta(int(ns_start) - int(ns) - ns_scope, unit="ns") + offset = to_timedelta(int(ns_start) - int(ns) - ns_scope, unit="ns") else: # not in period, later than start # dt @@ -297,7 +295,7 @@ def prev_start(self, dt): # dt # --<---------->-----------<-------------->-- # end | start end | start - offset = pd.Timedelta(int(ns_start) - int(ns), unit="ns") + offset = to_timedelta(int(ns_start) - int(ns), unit="ns") return dt + offset def prev_end(self, dt): @@ -322,7 +320,7 @@ def prev_end(self, dt): # --<---------->-----------<-------------->-- # end | start end | start ns_scope = self.get_scope_back(dt) - offset = pd.Timedelta(int(ns_end) - int(ns) - ns_scope, unit="ns") + offset = to_timedelta(int(ns_end) - int(ns) - ns_scope, unit="ns") else: # not in period, over night # dt @@ -338,7 +336,7 @@ def prev_end(self, dt): # dt # -->----------<----------->--------------<- # start | end start | end - offset = pd.Timedelta(int(ns_end) - int(ns), unit="ns") + offset = to_timedelta(int(ns_end) - int(ns), unit="ns") return dt + offset @@ -354,8 +352,8 @@ def __str__(self): scope = self._scope repr_scope = self.components[self.components.index(self._scope) + 1] - to_start = pd.Timedelta(start_ns, unit="ns") - to_end = pd.Timedelta(end_ns, unit="ns") + to_start = to_timedelta(start_ns, unit="ns") + to_end = to_timedelta(end_ns, unit="ns") start_str = timedelta_to_str(to_start, default_scope=repr_scope) end_str = timedelta_to_str(to_end, default_scope=repr_scope) diff --git a/rocketry/core/time/base.py b/rocketry/core/time/base.py index 9a6d11f2..cf12b596 100644 --- a/rocketry/core/time/base.py +++ b/rocketry/core/time/base.py @@ -4,10 +4,9 @@ from typing import Callable, Dict, List, Pattern, Union import itertools -import pandas as pd - from rocketry._base import RedBase from rocketry.core.meta import _add_parser +from rocketry.pybox.time import to_datetime, to_timedelta, Interval from rocketry.session import Session PARSERS: Dict[Union[str, Pattern], Union[Callable, 'TimePeriod']] = {} @@ -34,7 +33,7 @@ class TimePeriod(RedBase, metaclass=_TimeMeta): is in a given time span. """ - resolution = pd.Timestamp.resolution + resolution = datetime.timedelta.resolution min = datetime.datetime(1970, 1, 3, 2, 0) max = datetime.datetime(2260, 1, 1, 0, 0) @@ -133,7 +132,7 @@ def prev_end(self, dt): raise NotImplementedError("Contains not implemented.") @abstractmethod - def from_between(start, end) -> pd.Interval: + def from_between(start, end) -> Interval: raise NotImplementedError("__between__ not implemented.") def rollforward(self, dt): @@ -142,21 +141,21 @@ def rollforward(self, dt): start = self.rollstart(dt) end = self.next_end(dt) - start = pd.Timestamp(start) - end = pd.Timestamp(end) + start = to_datetime(start) + end = to_datetime(end) - return pd.Interval(start, end, closed="both") + return Interval(start, end, closed="both") - def rollback(self, dt) -> pd.Interval: + def rollback(self, dt) -> Interval: "Get previous time interval of the period" end = self.rollend(dt) start = self.prev_start(dt) - start = pd.Timestamp(start) - end = pd.Timestamp(end) + start = to_datetime(start) + end = to_datetime(end) - return pd.Interval(start, end, closed="both") + return Interval(start, end, closed="both") def __eq__(self, other): "Test whether self and other are essentially the same periods" @@ -188,8 +187,8 @@ def __init__(self, past=None, future=None, kws_past=None, kws_future=None): kws_past = {} if kws_past is None else kws_past kws_future = {} if kws_future is None else kws_future - self.past = abs(pd.Timedelta(past, **kws_past)) - self.future = abs(pd.Timedelta(future, **kws_future)) + self.past = abs(to_timedelta(past, **kws_past)) + self.future = abs(to_timedelta(future, **kws_future)) @abstractmethod def __contains__(self, dt): @@ -202,16 +201,16 @@ def __contains__(self, dt): def rollback(self, dt): "Get previous interval (including currently ongoing)" start = dt - abs(self.past) - start = pd.Timestamp(start) - end = pd.Timestamp(dt) - return pd.Interval(start, end) + start = to_datetime(start) + end = to_datetime(dt) + return Interval(start, end) def rollforward(self, dt): "Get next interval (including currently ongoing)" end = dt + abs(self.future) - start = pd.Timestamp(dt) - end = pd.Timestamp(end) - return pd.Interval(start, end) + start = to_datetime(dt) + end = to_datetime(end) + return Interval(start, end) def __eq__(self, other): "Test whether self and other are essentially the same periods" @@ -232,7 +231,7 @@ def __str__(self): def __repr__(self): return f"TimeDelta(past={repr(self.past)}, future={repr(self.future)})" -def all_overlap(times:List[pd.Interval]): +def all_overlap(times:List[Interval]): return all(a.overlaps(b) for a, b in itertools.combinations(times, 2)) def get_overlapping(times): @@ -246,7 +245,7 @@ def get_overlapping(times): start = max(starts) end = min(ends) - return pd.Interval(start, end) + return Interval(start, end) class All(TimePeriod): @@ -349,7 +348,7 @@ def rollback(self, dt): period.rollback(start - datetime.datetime.resolution) for period in self.periods ] - if any(pd.Interval(start, end).overlaps(interval) for interval in next_intervals): + if any(Interval(start, end).overlaps(interval) for interval in next_intervals): # Example: # A: <--> # B: <---> <---> @@ -358,7 +357,7 @@ def rollback(self, dt): extended = self.rollback(start - datetime.datetime.resolution) start = extended.left - return pd.Interval(start, end) + return Interval(start, end) def rollforward(self, dt): intervals = [ @@ -377,7 +376,7 @@ def rollforward(self, dt): for period in self.periods ] - if any(pd.Interval(start, end).overlaps(interval) for interval in next_intervals): + if any(Interval(start, end).overlaps(interval) for interval in next_intervals): # Example: # A: <--> # B: <---> <---> @@ -386,7 +385,7 @@ def rollforward(self, dt): extended = self.rollforward(end + datetime.datetime.resolution) end = extended.right - return pd.Interval(start, end) + return Interval(start, end) def __eq__(self, other): # self | other @@ -404,20 +403,20 @@ def __init__(self, start=None, end=None): self.end = end if end is not None else self.max def rollback(self, dt): - dt = pd.Timestamp(dt) - start = pd.Timestamp(self.start) + dt = to_datetime(dt) + start = to_datetime(self.start) if start > dt: # The actual interval is in the future - return pd.Interval(self.min, self.min) - return pd.Interval(start, dt) + return Interval(self.min, self.min) + return Interval(start, dt) def rollforward(self, dt): - dt = pd.Timestamp(dt) - end = pd.Timestamp(self.end) + dt = to_datetime(dt) + end = to_datetime(self.end) if end < dt: # The actual interval is already gone - return pd.Interval(self.max, self.max, closed="both") - return pd.Interval(dt, end, closed="both") + return Interval(self.max, self.max, closed="both") + return Interval(dt, end, closed="both") @property def is_max_interval(self): diff --git a/rocketry/pybox/query/base.py b/rocketry/pybox/query/base.py index 916a47be..e86a6b39 100644 --- a/rocketry/pybox/query/base.py +++ b/rocketry/pybox/query/base.py @@ -1,6 +1,7 @@ from typing import Iterable, Iterator import datetime -import pandas as pd + +from rocketry.pybox.time.convert import to_datetime, to_timedelta class QueryBase: @@ -105,10 +106,10 @@ def __invert__(self): return Not(self) def _to_comparable(self, left, right): - dt_cls = (datetime.datetime, pd.Timestamp) # Note we test pd.Timestamp due to test mocking + dt_cls = (datetime.datetime,) is_datetime = isinstance(left, dt_cls) or isinstance(right, dt_cls) if is_datetime: - return pd.Timestamp(left), pd.Timestamp(right) + return to_datetime(left), to_datetime(right) else: return left, right diff --git a/rocketry/pybox/time/__init__.py b/rocketry/pybox/time/__init__.py index 1f76e1fd..8d168ee0 100644 --- a/rocketry/pybox/time/__init__.py +++ b/rocketry/pybox/time/__init__.py @@ -1 +1,2 @@ -from .convert import to_nanoseconds, to_timedelta \ No newline at end of file +from .convert import to_nanoseconds, to_timedelta, to_datetime +from .interval import Interval \ No newline at end of file diff --git a/rocketry/pybox/time/convert.py b/rocketry/pybox/time/convert.py index 8d5790be..1b8a8614 100644 --- a/rocketry/pybox/time/convert.py +++ b/rocketry/pybox/time/convert.py @@ -7,6 +7,11 @@ def to_datetime(s): return s elif isinstance(s, str): return string_to_datetime(s) + elif hasattr(s, "timestamp"): + # Is datetime-like. Tests' monkeypatching + # overrides datetime.datetime thus we cannot + # always rely on type + return datetime.datetime.fromtimestamp(s.timestamp()) else: raise TypeError(f"Cannot convert to datetime: {type(s)}") @@ -69,9 +74,17 @@ def get_unit(s): pos += 1 return abbrs[abbr], pos + def get_hhmmss(s): + hh, mm, ss = s.split(":") + return to_nanoseconds(hour=int(hh), minute=int(mm), second=float(ss)) + # https://github.com/pandas-dev/pandas/blob/e8093ba372f9adfe79439d90fe74b0b5b6dea9d6/pandas/_libs/tslibs/timedeltas.pyx#L296 abbrs = { + 'millisecond': 'millisecond', + 'ms': 'millisecond', + 'seconds': 'second', + 'second': 'second', 'sec': 'second', 's': 'second', @@ -106,18 +119,35 @@ def get_unit(s): pos = skip_wordbreak(s) s = s[pos:] + # Example: "- 2.5 days ..." + # Pos: ^ + numb, pos = get_number(s) s = s[pos:] + if s[0] == ":": + # Expecting HH:MM:SS + ns += get_hhmmss(numb + s) + break + + # Example: "- 2.5 days ..." + # Pos: ^ + pos = skip_wordbreak(s) s = s[pos:] + # Example: "- 2.5 days ..." + # Pos: ^ + abbr, pos = get_unit(s) s = s[pos:] ns += to_nanoseconds(**{abbr: float(numb)}) + + if is_negative: + ns = -ns return datetime.timedelta(microseconds=ns / 1000) -def to_nanoseconds(day=0, hour=0, minute=0, second=0, microsecond=0, nanosecond=0) -> int: +def to_nanoseconds(day=0, hour=0, minute=0, second=0, millisecond=0, microsecond=0, nanosecond=0) -> int: "Turn time components to nanoseconds" - return nanosecond + microsecond * 1_000 + second * int(1e+9) + minute * int(6e+10) + hour * int(3.6e+12) + day * int(8.64e+13) \ No newline at end of file + return nanosecond + microsecond * 1_000 + millisecond * 1_000_000 + second * int(1e+9) + minute * int(6e+10) + hour * int(3.6e+12) + day * int(8.64e+13) \ No newline at end of file diff --git a/rocketry/pybox/time/interval.py b/rocketry/pybox/time/interval.py new file mode 100644 index 00000000..15ef5edc --- /dev/null +++ b/rocketry/pybox/time/interval.py @@ -0,0 +1,50 @@ + + +class Interval: + "Mimics pandas.Interval" + + def __init__(self, left, right, closed="right"): + self.left = left + self.right = right + self.closed = closed + + + def overlaps(self, other:'Interval'): + # Simple case: No overlap if: + + # self: |------| + # other: |------| + is_self_on_far_left = self.right < other.left + + # Or: + # self: |------| + # other: |------| + is_self_on_far_right = self.left > other.right + is_not_overlap = is_self_on_far_left or is_self_on_far_right + if is_not_overlap: + return False + + # More complex: + # self: |------? + # other: ?------| + if self.right == other.left: + if self.closed in ('left', 'neither') or other.closed in ('right', 'neither'): + # self: |------* + # other: |------| + + # self: |------| + # other: *------| + return False + + # self: ?------| + # other: |------? + if self.left == other.right: + if self.closed in ('right', 'neither') or other.closed in ('left', 'neither'): + # self: *------| + # other: |------| + + # self: |------| + # other: |------* + return False + + return True \ No newline at end of file diff --git a/rocketry/session.py b/rocketry/session.py index 75d4db5e..3f8aad69 100644 --- a/rocketry/session.py +++ b/rocketry/session.py @@ -10,9 +10,9 @@ from multiprocessing import cpu_count from pathlib import Path import warnings -import pandas as pd from pydantic import BaseModel, PrivateAttr, validator +from rocketry.pybox.time import to_timedelta from rocketry.log.defaults import create_default_handler from typing import TYPE_CHECKING, Callable, ClassVar, Iterable, Dict, List, Optional, Set, Tuple, Type, Union, Any from itertools import chain @@ -70,7 +70,7 @@ def parse_shut_cond(cls, value): @validator('timeout') def parse_timeout(cls, value): if isinstance(value, str): - return pd.Timedelta(value).to_pytimedelta() + return to_timedelta(value).to_pytimedelta() elif isinstance(value, (float, int)): return datetime.timedelta(milliseconds=value * 1000) else: diff --git a/rocketry/test/condition/task/test_time.py b/rocketry/test/condition/task/test_time.py index a0f28ed9..3d0ae514 100644 --- a/rocketry/test/condition/task/test_time.py +++ b/rocketry/test/condition/task/test_time.py @@ -3,8 +3,6 @@ from typing import List, Tuple import pytest -import pandas as pd -from dateutil.tz import tzlocal from rocketry.conditions import ( TaskStarted, @@ -15,17 +13,13 @@ TaskRunning ) +from rocketry.pybox.time.convert import to_datetime from rocketry.time import ( TimeDelta, TimeOfDay ) from rocketry.tasks import FuncTask -def to_epoch(dt): - # Hack as time.tzlocal() does not work for 1970-01-01 - if dt.tz: - dt = dt.tz_convert("utc").tz_localize(None) - return (dt - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s') def setup_task_state(mock_datetime_now, logs:List[Tuple[str, str]], time_after=None, task=None): """A mock up that sets up a task to test the @@ -48,12 +42,9 @@ def setup_task_state(mock_datetime_now, logs:List[Tuple[str, str]], time_after=N execution="main" ) - # pd.Timestamp -> Epoch, https://stackoverflow.com/a/54313505/13696660 - # We also need tz_localize to convert timestamp to localized form (logging thinks the time is local time and convert that to GTM) - for log in logs: log_time, log_action = log[0], log[1] - log_created = to_epoch(pd.Timestamp(log_time, tz=tzlocal())) + log_created = int(to_datetime(log_time).timestamp()) record = logging.LogRecord( # The content here should not matter for task status name='rocketry.core.task', level=logging.INFO, lineno=1, diff --git a/rocketry/test/condition/task/test_time_executable.py b/rocketry/test/condition/task/test_time_executable.py index 49b4f757..215262e5 100644 --- a/rocketry/test/condition/task/test_time_executable.py +++ b/rocketry/test/condition/task/test_time_executable.py @@ -1,13 +1,14 @@ +import datetime import logging import pytest -import pandas as pd from dateutil.tz import tzlocal from rocketry.conditions import ( TaskExecutable, ) +from rocketry.pybox.time.convert import to_datetime from rocketry.time import ( TimeDelta, TimeOfDay @@ -179,7 +180,7 @@ def to_epoch(dt): # Hack as time.tzlocal() does not work for 1970-01-01 if dt.tz: dt = dt.tz_convert("utc").tz_localize(None) - return (dt - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s') + return (dt - datetime.datetime(1970, 1, 1)) // datetime.timedelta(seconds=1) with tmpdir.as_cwd() as old_dir: @@ -192,12 +193,9 @@ def to_epoch(dt): condition = get_condition() - # pd.Timestamp -> Epoch, https://stackoverflow.com/a/54313505/13696660 - # We also need tz_localize to convert timestamp to localized form (logging thinks the time is local time and convert that to GTM) - for log in logs: log_time, log_action = log[0], log[1] - log_created = to_epoch(pd.Timestamp(log_time, tz=tzlocal())) + log_created = to_datetime(log_time).timestamp() record = logging.LogRecord( # The content here should not matter for task status name='rocketry.core.task', level=logging.INFO, lineno=1, @@ -210,7 +208,7 @@ def to_epoch(dt): record.task_name = "the task" task.logger.handle(record) - setattr(task, f'last_{log_action}', pd.Timestamp(log_time)) + setattr(task, f'last_{log_action}', to_datetime(log_time)) mock_datetime_now(time_after) if outcome: diff --git a/rocketry/test/condition/task/test_time_optimized.py b/rocketry/test/condition/task/test_time_optimized.py index 0ba2c8e3..f6a360c0 100644 --- a/rocketry/test/condition/task/test_time_optimized.py +++ b/rocketry/test/condition/task/test_time_optimized.py @@ -3,8 +3,6 @@ from typing import List, Tuple import pytest -import pandas as pd -from dateutil.tz import tzlocal from rocketry.conditions import ( TaskStarted, @@ -16,6 +14,7 @@ TaskRunning ) from rocketry.conditions.task import TaskInacted, TaskTerminated +from rocketry.pybox.time.convert import to_datetime from rocketry.time import ( TimeDelta, TimeOfDay @@ -24,11 +23,6 @@ from .test_time import setup_task_state -def to_epoch(dt): - # Hack as time.tzlocal() does not work for 1970-01-01 - if dt.tz: - dt = dt.tz_convert("utc").tz_localize(None) - return (dt - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s') @pytest.mark.parametrize("cls", [ @@ -65,7 +59,7 @@ def test_logs_not_used_true(session, cls, mock_datetime_now): execution="main" ) for attr in ("last_run", "last_success", "last_fail", "last_inaction", "last_terminate"): - setattr(task, attr, pd.Timestamp("2000-01-01 12:00:00")) + setattr(task, attr, to_datetime("2000-01-01 12:00:00")) cond = cls(task=task) assert cond.observe(session=session) @@ -83,7 +77,7 @@ def test_logs_not_used_true_inside_period(session, cls, mock_datetime_now): execution="main" ) for attr in ("last_run", "last_success", "last_fail", "last_inaction", "last_terminate"): - setattr(task, attr, pd.Timestamp("2000-01-01 12:00:00")) + setattr(task, attr, to_datetime("2000-01-01 12:00:00")) if cls == TaskRunning: cond = cls(task=task) else: @@ -105,7 +99,7 @@ def test_logs_not_used_false_outside_period(session, cls, mock_datetime_now): execution="main" ) for attr in ("last_run", "last_success", "last_fail", "last_inaction", "last_terminate"): - setattr(task, attr, pd.Timestamp("2000-01-01 05:00:00")) + setattr(task, attr, to_datetime("2000-01-01 05:00:00")) cond = cls(task=task, period=TimeOfDay("07:00", "13:00")) mock_datetime_now("2000-01-01 14:00") assert not cond.observe(session=session) diff --git a/rocketry/test/helpers/log_helpers.py b/rocketry/test/helpers/log_helpers.py index ee983a30..e2aa00f9 100644 --- a/rocketry/test/helpers/log_helpers.py +++ b/rocketry/test/helpers/log_helpers.py @@ -1,15 +1,8 @@ -import pandas as pd import logging import sys -def to_epoch(dt): - dt = pd.Timestamp(dt) - # Hack as time.tzlocal() does not work for 1970-01-01 - if dt.tz: - dt = dt.tz_convert("utc").tz_localize(None) - return (dt - pd.Timestamp("1970-01-01")) // pd.Timedelta('1s') - +from rocketry.pybox.time.convert import to_datetime def log_task_record(task, now, action, start_time=None): "Copy of the mechanism of creating an action log" @@ -27,7 +20,7 @@ def log_task_record(task, now, action, start_time=None): "success": "", }[action] - now = pd.Timestamp(now).to_pydatetime() + now = to_datetime(now) if action == "run": start_time = now @@ -35,7 +28,7 @@ def log_task_record(task, now, action, start_time=None): if start_time is None: start_time = task.last_run else: - start_time = pd.Timestamp(start_time).to_pydatetime() + start_time = to_datetime(start_time) record = logging.LogRecord( @@ -44,7 +37,7 @@ def log_task_record(task, now, action, start_time=None): pathname='rocketry\\rocketry\\core\\task\\base.py', msg=msg, args=(), exc_info=exc_info, ) - record.created = int(now.timestamp())# to_epoch(now) + record.created = int(now.timestamp()) record.action = action record.task_name = task.name record.start = start_time diff --git a/rocketry/test/pybox/test_convert.py b/rocketry/test/pybox/test_convert.py new file mode 100644 index 00000000..1f9edddf --- /dev/null +++ b/rocketry/test/pybox/test_convert.py @@ -0,0 +1,27 @@ +from datetime import timedelta +import pytest +from rocketry.pybox.time import to_timedelta, to_datetime, Interval + +@pytest.mark.parametrize("s,expected", + [ + ('00:00:00', timedelta()), + ('06:05:01', timedelta(hours=6, minutes=5, seconds=1)), + ('06:05:01.00003', timedelta(hours=6, minutes=5, seconds=1, microseconds=30)), + ('06:05:01.5', timedelta(hours=6, minutes=5, seconds=1, milliseconds=500)), + + ('1 days 16:05:01.00003', timedelta(days=1, hours=16, minutes=5, seconds=1, microseconds=30)), + + ('10m 20s', timedelta(minutes=10, seconds=20)), + ('1d 5h 10m 20s', timedelta(days=1, hours=5, minutes=10, seconds=20)), + ('1d, 5h, 10m, 20s', timedelta(days=1, hours=5, minutes=10, seconds=20)), + ('1 d 5 h 10 m 20 s', timedelta(days=1, hours=5, minutes=10, seconds=20)), + ("1days 5hours 10minutes 20seconds", timedelta(days=1, hours=5, minutes=10, seconds=20)), + ("1 day 5 hour 10 minute 20 second", timedelta(days=1, hours=5, minutes=10, seconds=20)), + ("1 days 5 hours 10 minutes 20 seconds", timedelta(days=1, hours=5, minutes=10, seconds=20)), + ("-1 days 5 hours 10 minutes 20 seconds", -timedelta(days=1, hours=5, minutes=10, seconds=20)), + ("--1 days 5 hours 10 minutes 20 seconds", timedelta(days=1, hours=5, minutes=10, seconds=20)), + ] +) +def test_timedelta(s, expected): + assert to_timedelta(s) == expected + diff --git a/rocketry/test/session/test_logs.py b/rocketry/test/session/test_logs.py index 54bf1de9..909c93de 100644 --- a/rocketry/test/session/test_logs.py +++ b/rocketry/test/session/test_logs.py @@ -6,13 +6,13 @@ from pydantic import Field, root_validator import pytest -import pandas as pd from redbird.oper import in_, between from redbird.logging import RepoHandler from redbird.repos import MemoryRepo from rocketry.log.log_record import LogRecord, TaskLogRecord, MinimalRecord +from rocketry.pybox.time.convert import to_datetime from rocketry.tasks import FuncTask def create_line_to_startup_file(): @@ -55,28 +55,28 @@ def validate_timestamp(cls, values): ], id="get succees & failure"), pytest.param( - {"timestamp": between(pd.Timestamp("2021-01-01 02:00:00"), pd.Timestamp("2021-01-01 03:00:00"))}, + {"timestamp": between(to_datetime("2021-01-01 02:00:00"), to_datetime("2021-01-01 03:00:00"))}, [ {'task_name': 'task3', 'created': datetime.datetime(2021, 1, 1, 2, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 2, 0, 0), 'message': "Task 'task3' status: 'run'"}, {'task_name': 'task4', 'created': datetime.datetime(2021, 1, 1, 3, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 3, 0, 0), 'message': "Task 'task4' status: 'run'"}, ], - id="get time span (pd.Timestamp)"), + id="get time span (datetime)"), pytest.param( - {"timestamp": between(None, pd.Timestamp("2021-01-01 03:00:00"), none_as_open=True), "action": "run"}, + {"timestamp": between(None, to_datetime("2021-01-01 03:00:00"), none_as_open=True), "action": "run"}, [ {'task_name': 'task1', 'created': datetime.datetime(2021, 1, 1, 0, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 0, 0, 0), 'message': "Task 'task1' status: 'run'"}, {'task_name': 'task2', 'created': datetime.datetime(2021, 1, 1, 1, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 1, 0, 0), 'message': "Task 'task2' status: 'run'"}, {'task_name': 'task3', 'created': datetime.datetime(2021, 1, 1, 2, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 2, 0, 0), 'message': "Task 'task3' status: 'run'"}, {'task_name': 'task4', 'created': datetime.datetime(2021, 1, 1, 3, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 3, 0, 0), 'message': "Task 'task4' status: 'run'"}, ], - id="get time span (pd.Timestamp, open left)"), + id="get time span (datetime, open left)"), pytest.param( - {"timestamp": between(pd.Timestamp("2021-01-01 02:00:00"), None, none_as_open=True), "action": "run"}, + {"timestamp": between(to_datetime("2021-01-01 02:00:00"), None, none_as_open=True), "action": "run"}, [ {'task_name': 'task3', 'created': datetime.datetime(2021, 1, 1, 2, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 2, 0, 0), 'message': "Task 'task3' status: 'run'"}, {'task_name': 'task4', 'created': datetime.datetime(2021, 1, 1, 3, 0, 0).timestamp(), 'action': 'run', 'start': datetime.datetime(2021, 1, 1, 3, 0, 0), 'message': "Task 'task4' status: 'run'"}, ], - id="get time span (pd.Timestamp, open right)"), + id="get time span (datetime, open right)"), pytest.param( {"timestamp": between(None, None, none_as_open=True), "action": "run"}, [ diff --git a/rocketry/test/time/delta/test_construct.py b/rocketry/test/time/delta/test_construct.py index 46af2a4a..e2dbad45 100644 --- a/rocketry/test/time/delta/test_construct.py +++ b/rocketry/test/time/delta/test_construct.py @@ -24,7 +24,7 @@ def test_equal(): assert not (TimeDelta("2 days") == TimeOfDay("10:00", "12:00")) def test_repr(): - assert str(TimeDelta("2 days")) == 'past 2 days 00:00:00' - assert str(TimeDelta(future="2 days")) == 'next 2 days 00:00:00' - assert str(TimeDelta(past="1 days", future="2 days")) == 'past 1 days 00:00:00 to next 2 days 00:00:00' - assert repr(TimeDelta("2 days")) == "TimeDelta(past=Timedelta('2 days 00:00:00'), future=Timedelta('0 days 00:00:00'))" \ No newline at end of file + assert str(TimeDelta("2 days")) == 'past 2 days, 0:00:00' + assert str(TimeDelta(future="2 days")) == 'next 2 days, 0:00:00' + assert str(TimeDelta(past="1 days", future="2 days")) == 'past 1 day, 0:00:00 to next 2 days, 0:00:00' + assert repr(TimeDelta("2 days")) == "TimeDelta(past=datetime.timedelta(days=2), future=datetime.timedelta(0))" \ No newline at end of file diff --git a/rocketry/test/time/delta/test_roll.py b/rocketry/test/time/delta/test_roll.py index e2fdb7a4..3aa429c7 100644 --- a/rocketry/test/time/delta/test_roll.py +++ b/rocketry/test/time/delta/test_roll.py @@ -1,23 +1,23 @@ import pytest -import pandas as pd from rocketry.core.time import ( TimeDelta ) +from rocketry.pybox.time.convert import to_datetime @pytest.mark.parametrize( "dt,past,future,roll_start,roll_end", [ # Regular pytest.param( - pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), None, None, - pd.Timestamp("2020-01-01 10:00:00"), pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), to_datetime("2020-01-01 10:00:00"), id="No roll"), pytest.param( - pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), None, "1:20:30", - pd.Timestamp("2020-01-01 10:00:00"), pd.Timestamp("2020-01-01 11:20:30"), + to_datetime("2020-01-01 10:00:00"), to_datetime("2020-01-01 11:20:30"), id="Regular"), ], ) @@ -34,14 +34,14 @@ def test_rollforward(dt, past, future, roll_start, roll_end): [ # Regular pytest.param( - pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), None, None, - pd.Timestamp("2020-01-01 10:00:00"), pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), to_datetime("2020-01-01 10:00:00"), id="No roll"), pytest.param( - pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 10:00:00"), "1:20:30", None, - pd.Timestamp("2020-01-01 08:39:30"), pd.Timestamp("2020-01-01 10:00:00"), + to_datetime("2020-01-01 08:39:30"), to_datetime("2020-01-01 10:00:00"), id="Regular"), ], ) diff --git a/rocketry/test/time/interval/timeofweek/test_core.py b/rocketry/test/time/interval/timeofweek/test_core.py index be262896..882e9c88 100644 --- a/rocketry/test/time/interval/timeofweek/test_core.py +++ b/rocketry/test/time/interval/timeofweek/test_core.py @@ -1,6 +1,6 @@ import pytest -import pandas as pd +from rocketry.pybox.time.convert import to_datetime from rocketry.time.interval import ( TimeOfWeek @@ -19,12 +19,12 @@ [ # Regular pytest.param( - pd.Timestamp("2024-01-01 00:00:00"), + to_datetime("2024-01-01 00:00:00"), "Mon 00:00:00", 0, id="Beginning"), pytest.param( - pd.Timestamp("2024-01-07 23:59:59.999999000"), + to_datetime("2024-01-07 23:59:59.999999000"), "Sun 23:59:59.999999000", 604799999999000.0, id="Ending"), diff --git a/rocketry/test/time/test_contains.py b/rocketry/test/time/test_contains.py index e8206d7c..d67dbc53 100644 --- a/rocketry/test/time/test_contains.py +++ b/rocketry/test/time/test_contains.py @@ -6,7 +6,7 @@ """ import pytest -import pandas as pd +from rocketry.pybox.time.convert import to_datetime from rocketry.time.interval import ( TimeOfHour, @@ -341,14 +341,14 @@ def _to_pyparams(cases): @_to_pyparams(true_cases) def test_in(dt, start, end, cls, time_point): - dt = pd.Timestamp(dt) + dt = to_datetime(dt) time = cls(start, end, time_point=time_point) assert dt in time @_to_pyparams(false_cases) def test_not(dt, start, end, cls, time_point): - dt = pd.Timestamp(dt) + dt = to_datetime(dt) time = cls(start, end, time_point=time_point) assert dt not in time diff --git a/rocketry/time/interval.py b/rocketry/time/interval.py index 995978f4..9b226413 100644 --- a/rocketry/time/interval.py +++ b/rocketry/time/interval.py @@ -4,11 +4,11 @@ import re import dateutil -import pandas as pd from rocketry.core.time.anchor import AnchoredInterval from rocketry.core.time.base import TimeInterval from rocketry.core.time.utils import timedelta_to_str, to_dict, to_nanoseconds +from rocketry.pybox.time.interval import Interval class TimeOfMinute(AnchoredInterval): @@ -23,7 +23,7 @@ class TimeOfMinute(AnchoredInterval): """ _scope = "minute" - _scope_max = to_nanoseconds(minute=1) - 1 # See: pd.Timedelta(59999999999, unit="ns") + _scope_max = to_nanoseconds(minute=1) - 1 _unit_resolution = to_nanoseconds(second=1) def anchor_str(self, s, **kwargs): @@ -316,9 +316,9 @@ def __init__(self, day, *, start_time=None, end_time=None): def rollback(self, dt): offset = self.offsets[self.day] dt = dt - offset - return pd.Interval( - pd.Timestamp.combine(dt, self.start_time), - pd.Timestamp.combine(dt, self.end_time) + return Interval( + datetime.datetime.combine(dt, self.start_time), + datetime.datetime.combine(dt, self.end_time) ) def rollforward(self, dt):