From e7a1399f3b5e28ac72c9d1dc27dc3d4a3e24a25e Mon Sep 17 00:00:00 2001 From: Franck Charras <29153872+fcharras@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:10:11 +0100 Subject: [PATCH] Add more solvers and dimensions, work around issues with incompatible dimensions in warmup --- benchmarks/ridge/datasets/simulated_blobs.py | 4 +- benchmarks/ridge/objective.py | 6 +- benchmarks/ridge/solvers/cuml.py | 56 ++++++++++-------- benchmarks/ridge/solvers/scikit_learn.py | 14 +++-- .../ridge/solvers/scikit_learn_intelex.py | 58 ++++++++++--------- .../ridge/solvers/sklearn_torch_dispatch.py | 54 +++++++++-------- 6 files changed, 110 insertions(+), 82 deletions(-) diff --git a/benchmarks/ridge/datasets/simulated_blobs.py b/benchmarks/ridge/datasets/simulated_blobs.py index 5d3276a..d850f73 100644 --- a/benchmarks/ridge/datasets/simulated_blobs.py +++ b/benchmarks/ridge/datasets/simulated_blobs.py @@ -11,8 +11,10 @@ class Dataset(BaseDataset): parameters = { "n_samples, n_features": [ (10_000_000, 100), - (10_000_000, 10), + (5_000_000, 100), + (5_000_000, 10), (5000, 10000), + (5000, 5000), ], "n_targets": [1, 10], "dtype": ["float32"], diff --git a/benchmarks/ridge/objective.py b/benchmarks/ridge/objective.py index 9f7043e..3dbfcdb 100644 --- a/benchmarks/ridge/objective.py +++ b/benchmarks/ridge/objective.py @@ -22,13 +22,15 @@ class Objective(BaseObjective): "alpha": [1.0, 1e-10], "fit_intercept": [True], "solver, max_iter, tol": [ - ("svd", None, 1e-4), - ("cholesky", None, 1e-4), + ("svd", None, 0), + ("cholesky", None, 0), ("lsqr", None, 1e-4), ("sparse_cg", None, 1e-4), ("sag", None, 1e-4), ("saga", None, 1e-4), ("lbfgs", None, 1e-4), + ("cg", None, 1e-4), + ("eig", None, None), # Used for scikit-learn-intelex that doesn't # document the underlying solver nor expose the n_iter_ attribute ("DefaultDense", None, 1e-4), diff --git a/benchmarks/ridge/solvers/cuml.py b/benchmarks/ridge/solvers/cuml.py index c64cd41..6245868 100644 --- a/benchmarks/ridge/solvers/cuml.py +++ b/benchmarks/ridge/solvers/cuml.py @@ -19,29 +19,6 @@ class Solver(BaseSolver): stopping_criterion = SingleRunCriterion(1) - def skip(self, **objective_dict): - - X = objective_dict["X"] - if X.dtype == np.float64: - # We haven't came accross cuda devices that doesn't support float64 yet, - # can it happen ? If it happens, the following instruction will fail, - # please enclose it with the appropriate Try/Except to return the - # appropriate skip decision. - cupy.zeros(1, dtype=cupy.float64) - # return True, ( - # f"This {self.device} device has no support for float64 compute" - # ) - - y = objective_dict["y"] - if (y.ndim == 2) and (y.shape[1] > 1): - return True, "Multitarget is not supported." - - solver = objective_dict["solver"] - if solver != "svd": - return True, "Only accepts the svd solver at the moment." - - return False, None - def set_objective( self, X, @@ -68,15 +45,44 @@ def set_objective( self.tol = tol self.random_state = random_state + def skip(self, **objective_dict): + + X = objective_dict["X"] + if X.dtype == np.float64: + # We haven't came accross cuda devices that doesn't support float64 yet, + # can it happen ? If it happens, the following instruction will fail, + # please enclose it with the appropriate Try/Except to return the + # appropriate skip decision. + cupy.zeros(1, dtype=cupy.float64) + # return True, ( + # f"This {self.device} device has no support for float64 compute" + # ) + + y = objective_dict["y"] + if (y.ndim == 2) and (y.shape[1] > 1): + return True, "Multitarget is not supported." + + solver = objective_dict["solver"] + if solver not in ["svd", "cg", "eig"]: + return True, "Only accepts the svd solver at the moment." + + return False, None + def warm_up(self): + n_warmup_samples = 20 + n_warmup_features = 5 sample_weight = self.sample_weight if sample_weight is not None: - sample_weight = sample_weight[:2] + sample_weight = sample_weight[:n_warmup_samples] cuml.Ridge( alpha=self.alpha, fit_intercept=self.fit_intercept, solver=self.solver, - ).fit(self.X[:2], self.y[:2], sample_weight=sample_weight) + ).fit( + self.X[:n_warmup_samples, :n_warmup_features], + self.y[:n_warmup_samples], + sample_weight=sample_weight, + ) def run(self, _): estimator = cuml.Ridge( diff --git a/benchmarks/ridge/solvers/scikit_learn.py b/benchmarks/ridge/solvers/scikit_learn.py index 971a76d..dec7041 100644 --- a/benchmarks/ridge/solvers/scikit_learn.py +++ b/benchmarks/ridge/solvers/scikit_learn.py @@ -38,22 +38,24 @@ def set_objective( def skip(self, **objective_dict): solver = objective_dict["solver"] - if solver in ["sag", "saga", "sparse_cg", "lbfgs"]: + if solver in ["sag", "saga"]: # TODO: investigate ? return True, ( "Preliminary testing show this solver is too slow to have relevance " "in the benchmark." ) - if solver == "DefaultDense": + if solver in ["DefaultDense", "eig", "cg"]: return True, "No support for this solver parameter." return False, None def warm_up(self): + n_warmup_samples = 20 + n_warmup_features = 5 sample_weight = self.sample_weight if sample_weight is not None: - sample_weight = sample_weight[:2] + sample_weight = sample_weight[:20] Ridge( alpha=self.alpha, fit_intercept=self.fit_intercept, @@ -63,7 +65,11 @@ def warm_up(self): solver=self.solver, positive=True if (self.solver == "lbfgs") else False, random_state=self.random_state, - ).fit(self.X[:2], self.y[:2], sample_weight) + ).fit( + self.X[:n_warmup_samples, :n_warmup_features], + self.y[:n_warmup_samples], + sample_weight, + ) def run(self, _): estimator = Ridge( diff --git a/benchmarks/ridge/solvers/scikit_learn_intelex.py b/benchmarks/ridge/solvers/scikit_learn_intelex.py index 02445bb..29cca65 100644 --- a/benchmarks/ridge/solvers/scikit_learn_intelex.py +++ b/benchmarks/ridge/solvers/scikit_learn_intelex.py @@ -31,30 +31,6 @@ class Solver(BaseSolver): stopping_criterion = SingleRunCriterion(1) - def skip(self, **objective_dict): - if self.runtime is not None: - try: - device = dpctl.SyclDevice(f"{self.runtime}:{self.device}") - except Exception: - return ( - True, - f"{self.runtime} runtime not found for device {self.device}", - ) - - X = objective_dict["X"] - if (X.dtype == np.float64) and not device.has_aspect_fp64: - return True, ( - f"This {self.device} device has no support for float64 compute" - ) - - solver = objective_dict["solver"] - - if solver != "DefaultDense": - # TODO: investigate ? - return True, "The only supported solver parameter is DefaultDense." - - return False, None - def set_objective( self, X, @@ -94,10 +70,36 @@ def set_objective( self.tol = tol self.random_state = random_state + def skip(self, **objective_dict): + if self.runtime is not None: + try: + device = dpctl.SyclDevice(f"{self.runtime}:{self.device}") + except Exception: + return ( + True, + f"{self.runtime} runtime not found for device {self.device}", + ) + + X = objective_dict["X"] + if (X.dtype == np.float64) and not device.has_aspect_fp64: + return True, ( + f"This {self.device} device has no support for float64 compute" + ) + + solver = objective_dict["solver"] + + if solver != "DefaultDense": + # TODO: investigate ? + return True, "The only supported solver parameter is DefaultDense." + + return False, None + def warm_up(self): + n_warmup_samples = 20 + n_warmup_features = 5 sample_weight = self.sample_weight if sample_weight is not None: - sample_weight = sample_weight[:2] + sample_weight = sample_weight[:n_warmup_samples] with nullcontext() if (self.runtime is None) else config_context( target_offload=f"{self.runtime}:{self.device}" ): @@ -110,7 +112,11 @@ def warm_up(self): solver="auto", positive=True if (self.solver == "lbfgs") else False, random_state=self.random_state, - ).fit(self.X[:2], self.y[:2], sample_weight) + ).fit( + self.X[:n_warmup_samples, :n_warmup_features], + self.y[:n_warmup_samples], + sample_weight, + ) def run(self, _): with nullcontext() if (self.runtime is None) else config_context( diff --git a/benchmarks/ridge/solvers/sklearn_torch_dispatch.py b/benchmarks/ridge/solvers/sklearn_torch_dispatch.py index c1e4553..26f3194 100644 --- a/benchmarks/ridge/solvers/sklearn_torch_dispatch.py +++ b/benchmarks/ridge/solvers/sklearn_torch_dispatch.py @@ -30,29 +30,6 @@ class Solver(BaseSolver): stopping_criterion = SingleRunCriterion(1) - def skip(self, **objective_dict): - if not Ridge()._get_tags()["array_api_support"]: - return True, ( - "Requires the development branch for Ridge support for Array API." - ) - - try: - torch.zeros(1, dtype=torch.float32, device=self.device) - except Exception: - return True, f"{self.device} compute backend for pytorch not found" - - X = objective_dict["X"] - if (X.dtype == np.float64) and not has_fp64_support(self.device): - return True, ( - f"This {self.device} device has no support for float64 compute" - ) - - solver = objective_dict["solver"] - if solver != "svd": - return True, "Only accepts the svd solver at the moment." - - return False, None - def set_objective( self, X, @@ -82,7 +59,32 @@ def set_objective( self.tol = tol self.random_state = random_state + def skip(self, **objective_dict): + if not Ridge()._get_tags()["array_api_support"]: + return True, ( + "Requires the development branch for Ridge support for Array API." + ) + + try: + torch.zeros(1, dtype=torch.float32, device=self.device) + except Exception: + return True, f"{self.device} compute backend for pytorch not found" + + X = objective_dict["X"] + if (X.dtype == np.float64) and not has_fp64_support(self.device): + return True, ( + f"This {self.device} device has no support for float64 compute" + ) + + solver = objective_dict["solver"] + if solver != "svd": + return True, "Only accepts the svd solver at the moment." + + return False, None + def warm_up(self): + n_warmup_samples = 20 + n_warmup_features = 5 sample_weight = self.sample_weight if sample_weight is not None: sample_weight = sample_weight[:2] @@ -96,7 +98,11 @@ def warm_up(self): solver=self.solver, positive=True if (self.solver == "lbfgs") else False, random_state=self.random_state, - ).fit(self.X[:2], self.y[:2], sample_weight) + ).fit( + self.X[:n_warmup_samples, :n_warmup_features], + self.y[:n_warmup_samples], + sample_weight, + ) def run(self, _): with config_context(array_api_dispatch=True):