ML-1020: when filtering by start/end time: start should be > and end …

…<= (#286)

(instead of start >= and end <)

integration/test_filesystems_integration.py

@@ -473,8 +473,8 @@ def test_filter_before_after_partitioned_inner_other_partition(setup_teardown_te
 def test_filter_before_after_partitioned_outer_other_partition(setup_teardown_test):
     columns = ['my_string', 'my_time', 'my_city']
 
-    df = pd.DataFrame([['shining', pd.Timestamp('2020-12-30 08:53:00'), 'ramat gan'],
-                       ['hello', pd.Timestamp('2020-12-31 15:05:00'), 'tel aviv'],
+    df = pd.DataFrame([['shining', pd.Timestamp('2020-12-31 14:00:00'), 'ramat gan'],
+                       ['hello', pd.Timestamp('2020-12-30 08:53:00'), 'tel aviv'],
                        ['beautiful', pd.Timestamp('2020-12-30 09:00:00'), 'haifa'],
                        ['sun', pd.Timestamp('2020-12-29 09:00:00'), 'tel aviv'],
                        ['world', pd.Timestamp('2020-12-30 15:00:45'), 'hod hasharon'],
@@ -500,7 +500,7 @@ def test_filter_before_after_partitioned_outer_other_partition(setup_teardown_te
     expected = [{'my_string': 'beautiful', 'my_time': pd.Timestamp('2020-12-30 09:00:00'), 'my_city': 'haifa'},
                 {'my_string': 'world', 'my_time': pd.Timestamp('2020-12-30 15:00:45'), 'my_city': 'hod hasharon'},
                 {'my_string': 'is', 'my_time': pd.Timestamp('2020-12-31 13:00:56'), 'my_city': 'hod hasharon'},
-                {'my_string': 'shining', 'my_time': pd.Timestamp('2020-12-30 08:53:00'), 'my_city': 'ramat gan'}]
+                {'my_string': 'shining', 'my_time': pd.Timestamp('2020-12-31 14:00:00'), 'my_city': 'ramat gan'}]
 
     assert read_back_result == expected, f"{read_back_result}\n!=\n{expected}"
 
@@ -525,7 +525,7 @@ def test_filter_by_time_non_partioned(setup_teardown_test):
 
     read_back_result = controller.await_termination()
 
-    expected = [{'my_string': 'dina', 'my_time': pd.Timestamp('2019-07-01 00:00:00'), 'my_city': 'tel aviv'}]
+    expected = [{'my_string': 'katya', 'my_time': pd.Timestamp('2020-12-31 14:00:00'), 'my_city': 'hod hasharon'}]
 
     try:
         assert read_back_result == expected, f"{read_back_result}\n!=\n{expected}"

storey/utils.py

@@ -203,8 +203,8 @@ def _find_filter_helper(list_partitions, dtime, sign, first_sign, first_uncommon
     if first_sign:
         # only for the first iteration we need to have ">="/"<=" instead of ">"/"<"
         _create_filter_tuple(dtime, last_partition, first_sign, single_filter)
-        # start needs to be >= and end needs to be "<"
-        if first_sign == ">=":
+        # start needs to be > and end needs to be "<="
+        if first_sign == "<=":
             tuple_last_range = (filter_column, first_sign, dtime)
         else:
             tuple_last_range = (filter_column, sign, dtime)
@@ -216,8 +216,8 @@ def _find_filter_helper(list_partitions, dtime, sign, first_sign, first_uncommon
 
 
 def _get_filters_for_filter_column(start, end, filter_column, side_range):
-    lower_limit_tuple = (filter_column, ">=", start)
-    upper_limit_tuple = (filter_column, "<", end)
+    lower_limit_tuple = (filter_column, ">", start)
+    upper_limit_tuple = (filter_column, "<=", end)
     side_range.append(lower_limit_tuple)
     side_range.append(upper_limit_tuple)