Introduce not given and apply it to dataset's target

giskard/core/core.py

@@ -23,6 +23,37 @@ class Kwargs:
     pass
 
 
+_T = TypeVar("_T")
+
+
+# Sentinel class used until PEP 0661 is accepted
+class NotGiven:
+    """
+    A sentinel singleton class used to distinguish omitted keyword arguments
+    from those passed in with the value None (which may have different behavior).
+
+    For example:
+
+    ```py
+    def get(timeout: Union[int, NotGiven, None] = NotGiven()) -> Response: ...
+
+    get(timout=1) # 1s timeout
+    get(timout=None) # No timeout
+    get() # Default timeout behavior, which may not be statically known at the method definition.
+    ```
+    """
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    def __repr__(self) -> str:
+        return "NOT_GIVEN"
+
+
+NotGivenOr = Union[_T, NotGiven]
+NOT_GIVEN = NotGiven()
+
+
 def _get_plugin_method_full_name(func):
     from giskard.ml_worker.testing.registry.registry import plugins_root
 

giskard/core/dataset_validation.py

@@ -9,7 +9,7 @@
 
 
 def validate_optional_target(ds: Dataset):
-    if ds.target is None:
+    if not ds.is_target_given:
         warning(
             "You did not provide the optional argument 'target'. "
             "'target' is the column name in df corresponding to the actual target variable (ground truth)."

giskard/core/model_validation.py

@@ -12,15 +12,15 @@
 from giskard.datasets.base import Dataset
 from giskard.ml_worker.testing.registry.slicing_function import SlicingFunction
 from giskard.models.base import BaseModel, WrapperModel
+from .dataset_validation import validate_optional_target
 from ..utils import fullname
 from ..utils.analytics_collector import analytics, get_dataset_properties, get_model_properties
-from .dataset_validation import validate_optional_target
 
 
 @configured_validate_arguments
 def validate_model(model: BaseModel, validate_ds: Optional[Dataset] = None, print_validation_message: bool = True):
     try:
-        if model.meta.model_type != SupportedModelTypes.TEXT_GENERATION and validate_ds is not None:
+        if validate_ds is not None:
             validate_optional_target(validate_ds)
         _do_validate_model(model, validate_ds)
     except (ValueError, TypeError) as err:

giskard/datasets/base/__init__.py

@@ -19,7 +19,7 @@
 from giskard.client.giskard_client import GiskardClient
 from giskard.client.io_utils import compress, save_df
 from giskard.client.python_utils import warning
-from giskard.core.core import DatasetMeta, SupportedColumnTypes
+from giskard.core.core import DatasetMeta, SupportedColumnTypes, NOT_GIVEN, NotGivenOr
 from giskard.core.errors import GiskardImportError
 from giskard.core.validation import configured_validate_arguments
 from giskard.ml_worker.testing.registry.slicing_function import SlicingFunction, SlicingFunctionType
@@ -144,7 +144,7 @@ class Dataset(ColumnMetadataMixin):
     """
 
     name: Optional[str]
-    target: Optional[str]
+    _target: NotGivenOr[Optional[str]]
     column_types: Dict[str, str]
     df: pd.DataFrame
     id: uuid.UUID
@@ -156,7 +156,7 @@ def __init__(
         self,
         df: pd.DataFrame,
         name: Optional[str] = None,
-        target: Optional[Hashable] = None,
+        target: NotGivenOr[Optional[Hashable]] = NOT_GIVEN,
         cat_columns: Optional[List[str]] = None,
         column_types: Optional[Dict[Hashable, str]] = None,
         id: Optional[uuid.UUID] = None,
@@ -186,7 +186,7 @@ def __init__(
 
         self.name = name
         self.df = pd.DataFrame(df)
-        self.target = target
+        self._target = target
 
         if validation:
             from giskard.core.dataset_validation import validate_dtypes, validate_target_exists
@@ -229,7 +229,13 @@ def __init__(
 
         logger.info("Your 'pandas.DataFrame' is successfully wrapped by Giskard's 'Dataset' wrapper class.")
 
-        self.data_processor = DataProcessor()
+    @property
+    def is_target_given(self) -> bool:
+        return self._target is not NOT_GIVEN
+
+    @property
+    def target(self) -> Optional[str]:
+        return self._target or None
 
     def add_slicing_function(self, slicing_function: SlicingFunction):
         """

tests/integrations/test_mlflow.py

@@ -79,7 +79,7 @@ def test_errors(dataset_name, model_name, request):
     # dataset type error
     dataset_copy = dataset.copy()
     dataset_copy.df = [[0.6, 0.4]]
-    dataset_copy.target = [1]
+    dataset_copy._target = [1]
 
     with pytest.raises(Exception) as e:
         _evaluate(dataset_copy, model, evaluator_config)

tests/test_dataset.py

@@ -1,19 +1,17 @@
-import pandas as pd
-import numpy as np
-from pydantic import ValidationError
-import pytest
 import uuid
 
+import numpy as np
+import pandas as pd
+import pytest
 import requests_mock
+from pydantic import ValidationError
 
-from giskard.datasets.base import Dataset
-from giskard.core.dataset_validation import validate_optional_target
 from giskard.client.dtos import DatasetMetaInfo
-
+from giskard.core.dataset_validation import validate_optional_target
+from giskard.datasets.base import Dataset
 from tests import utils
 from tests.communications.test_dto_serialization import is_required, get_fields, get_name
 
-
 # FIXME: conflict on `name` between Giskard Hub (@NotBlank) and Python client (optional in DatasetMeta and DatasetMetaInfo)
 MANDATORY_FIELDS = [
     "id",
@@ -57,15 +55,28 @@ def test_factory():
     assert isinstance(my_dataset, Dataset)
 
 
-def test_valid_df_column_types():
-    # Option 0: none of column_types, cat_columns, infer_column_types = True are provided
+def test_validate_optional_target():
     with pytest.warns(
         UserWarning,
         match=r"You did not provide the optional argument 'target'\. 'target' is the column name "
         r"in df corresponding to the actual target variable \(ground truth\)\.",
     ):
         my_dataset = Dataset(valid_df)
         validate_optional_target(my_dataset)
+
+    with pytest.warns(None) as record:
+        my_dataset = Dataset(valid_df, target=None)
+        validate_optional_target(my_dataset)
+
+        my_dataset = Dataset(valid_df, target="text_column")
+        validate_optional_target(my_dataset)
+
+    assert len(record) == 0
+
+
+def test_valid_df_column_types():
+    # Option 0: none of column_types, cat_columns, infer_column_types = True are provided
+    my_dataset = Dataset(valid_df)
     assert my_dataset.column_types == {
         "categorical_column": "category",
         "text_column": "text",
@@ -158,7 +169,6 @@ def test_numeric_column_names():
 
 
 def test_infer_column_types():
-
     # if df_size >= 100     ==> category_threshold = floor(log10(df_size))
     assert Dataset(pd.DataFrame({"f": [1, 2] * 50})).column_types["f"] == "category"
     assert Dataset(pd.DataFrame({"f": ["a", "b"] * 50})).column_types["f"] == "category"
@@ -228,8 +238,9 @@ def test_dataset_meta_info():
     mandatory_field_names = []
     optional_field_names = []
     for name, field in get_fields(klass).items():
-        mandatory_field_names.append(get_name(name, field)) if is_required(field) else \
-            optional_field_names.append(get_name(name, field))
+        mandatory_field_names.append(get_name(name, field)) if is_required(field) else optional_field_names.append(
+            get_name(name, field)
+        )
     assert set(mandatory_field_names) == set(MANDATORY_FIELDS)
     assert set(optional_field_names) == set(OPTIONAL_FIELDS)
 
@@ -242,7 +253,9 @@ def test_fetch_dataset_meta(request):
         with utils.MockedClient(mock_all=False) as (client, mr):
             meta_info = utils.mock_dataset_meta_info(dataset, project_key)
             meta_info.pop(op)
-            mr.register_uri(method=requests_mock.GET, url=utils.get_url_for_dataset(dataset, project_key), json=meta_info)
+            mr.register_uri(
+                method=requests_mock.GET, url=utils.get_url_for_dataset(dataset, project_key), json=meta_info
+            )
 
             # Should not raise
             client.load_dataset_meta(project_key, uuid=str(dataset.id))
@@ -251,7 +264,9 @@ def test_fetch_dataset_meta(request):
         with utils.MockedClient(mock_all=False) as (client, mr):
             meta_info = utils.mock_dataset_meta_info(dataset, project_key)
             meta_info.pop(op)
-            mr.register_uri(method=requests_mock.GET, url=utils.get_url_for_dataset(dataset, project_key), json=meta_info)
+            mr.register_uri(
+                method=requests_mock.GET, url=utils.get_url_for_dataset(dataset, project_key), json=meta_info
+            )
 
             # Should raise due to missing of values
             with pytest.raises(ValidationError):