-
Notifications
You must be signed in to change notification settings - Fork 200
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #230 from Kiln-AI/evals
Evals! See the commits for more details (over 100). docs: https://docs.getkiln.ai/docs/evaluations
- Loading branch information
Showing
114 changed files
with
11,797 additions
and
1,091 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| - Always assume pydantic 2 (not pydantic 1) | ||
| - Always use pytest for tests | ||
| - The project supports Python 3.10 and above | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,6 +8,7 @@ dependencies = [ | |
| "pillow>=11.0.0", | ||
| "pystray>=0.19.5", | ||
| "pyinstaller==6.11.1", | ||
| "scipy>=1.15.2", | ||
| ] | ||
|
|
||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| import math | ||
| from dataclasses import dataclass | ||
| from typing import List | ||
|
|
||
| from scipy import stats | ||
|
|
||
|
|
||
| @dataclass | ||
| class CorrelationScore: | ||
| measured_score: float | ||
| human_score: float | ||
| normalized_measured_score: float | ||
| normalized_human_score: float | ||
|
|
||
|
|
||
| @dataclass | ||
| class CorrelationResult: | ||
| mean_absolute_error: float | ||
| mean_normalized_absolute_error: float | ||
| mean_squared_error: float | ||
| mean_normalized_squared_error: float | ||
| spearman_correlation: float | None | ||
| pearson_correlation: float | None | ||
| kendalltau_correlation: float | None | ||
|
|
||
|
|
||
| class CorrelationCalculator: | ||
| def __init__(self): | ||
| self.scores: List[CorrelationScore] = [] | ||
|
|
||
| def add_score(self, score: CorrelationScore): | ||
| self.scores.append(score) | ||
|
|
||
| def calculate_correlation(self) -> CorrelationResult: | ||
| if len(self.scores) == 0: | ||
| raise ValueError("No scores to calculate correlation") | ||
|
|
||
| return CorrelationResult( | ||
| mean_absolute_error=self.calculate_mean_absolute_error(), | ||
| mean_normalized_absolute_error=self.calculate_mean_normalized_absolute_error(), | ||
| mean_squared_error=self.calculate_mean_squared_error(), | ||
| mean_normalized_squared_error=self.calculate_mean_normalized_squared_error(), | ||
| spearman_correlation=self.calculate_spearman_correlation(), | ||
| pearson_correlation=self.calculate_pearson_correlation(), | ||
| kendalltau_correlation=self.calculate_kendalltau_correlation(), | ||
| ) | ||
|
|
||
| def calculate_mean_absolute_error(self) -> float: | ||
| total_absolute_error = sum( | ||
| abs(score.measured_score - score.human_score) for score in self.scores | ||
| ) | ||
| return total_absolute_error / len(self.scores) | ||
|
|
||
| def calculate_mean_normalized_absolute_error(self) -> float: | ||
| total_normalized_absolute_error = sum( | ||
| abs(score.normalized_measured_score - score.normalized_human_score) | ||
| for score in self.scores | ||
| ) | ||
| return total_normalized_absolute_error / len(self.scores) | ||
|
|
||
| def calculate_mean_squared_error(self) -> float: | ||
| total_squared_error = sum( | ||
| (score.measured_score - score.human_score) ** 2 for score in self.scores | ||
| ) | ||
| return total_squared_error / len(self.scores) | ||
|
|
||
| def calculate_mean_normalized_squared_error(self) -> float: | ||
| total_normalized_squared_error = sum( | ||
| (score.normalized_measured_score - score.normalized_human_score) ** 2 | ||
| for score in self.scores | ||
| ) | ||
| return total_normalized_squared_error / len(self.scores) | ||
|
|
||
| def calculate_spearman_correlation(self) -> float | None: | ||
| if len(self.scores) < 2: | ||
| # If there is only one pair, no correlation | ||
| return None | ||
| x = [score.measured_score for score in self.scores] | ||
| y = [score.human_score for score in self.scores] | ||
| result = stats.spearmanr(x, y) | ||
| # library doesn't support proper types | ||
| correlation = result.__getattribute__("correlation") | ||
| if math.isnan(correlation) or not isinstance(correlation, float): | ||
| # Very small samples may have a NaN result (unknown correlation) | ||
| return None | ||
| return correlation | ||
|
|
||
| def calculate_pearson_correlation(self) -> float | None: | ||
| if len(self.scores) < 2: | ||
| # If there is only one pair, no correlation | ||
| return None | ||
| x = [score.measured_score for score in self.scores] | ||
| y = [score.human_score for score in self.scores] | ||
| result = stats.pearsonr(x, y) | ||
| if math.isnan(result.correlation): | ||
| # Very small samples may have a NaN result (unknown correlation) | ||
| return None | ||
| return result.correlation | ||
|
|
||
| def calculate_kendalltau_correlation(self) -> float | None: | ||
| if len(self.scores) < 2: | ||
| # If there is only one pair, no correlation | ||
| return None | ||
| x = [score.measured_score for score in self.scores] | ||
| y = [score.human_score for score in self.scores] | ||
| result = stats.kendalltau(x, y) | ||
| if math.isnan(result.correlation): | ||
| # Very small samples may have a NaN result (unknown correlation) | ||
| return None | ||
| return result.correlation |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.