searchindex.js

Search.setIndex({"docnames": ["api_reference/data", "api_reference/index", "api_reference/metrics", "api_reference/metrics_accuracy", "api_reference/metrics_calibration", "api_reference/metrics_scoring_rule", "api_reference/recalibration", "api_reference/utils", "api_reference/viz", "index", "installation", "quick_start"], "filenames": ["api_reference/data.rst", "api_reference/index.rst", "api_reference/metrics.rst", "api_reference/metrics_accuracy.rst", "api_reference/metrics_calibration.rst", "api_reference/metrics_scoring_rule.rst", "api_reference/recalibration.rst", "api_reference/utils.rst", "api_reference/viz.rst", "index.rst", "installation.rst", "quick_start.rst"], "titles": ["data module", "API Reference", "metrics module", "metrics_accuracy module", "metrics_calibration module", "metrics_scoring_rule module", "recalibration module", "utils module", "viz module", "Uncertainty Toolbox Documentation", "Installation", "Quick Start"], "terms": {"thi": [0, 1, 2, 3, 4, 5, 6, 7, 8, 11], "section": [0, 1, 2, 3, 4, 5, 6, 7, 8], "contain": [0, 1, 2, 3, 4, 5, 6, 7, 8], "python": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], "api": [0, 2, 3, 4, 5, 6, 7, 8, 9], "refer": [0, 2, 3, 4, 5, 6, 7, 8, 9], "which": [0, 2, 3, 4, 5, 6, 7, 8], "code": [0, 3, 4, 5, 6, 7, 8, 9], "import": [0, 11], "gener": 0, "synthetic_arange_random": 0, "num_point": 0, "10": [0, 4], "dataset": [0, 2, 3, 4, 5, 6, 8, 11], "evenli": 0, "space": [0, 4], "point": [0, 4, 5, 8], "ident": [0, 4], "function": [0, 4, 6, 7], "some": [0, 2], "random": [0, 8], "return": [0, 2, 3, 4, 5, 6, 7, 8, 9], "predict": [0, 2, 3, 4, 5, 6, 8, 9, 11], "uncertainti": [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11], "given": [0, 4, 5, 8], "standard": [0, 2, 4, 5, 6, 8, 11], "deviat": [0, 2, 4, 5, 6, 8, 11], "from": [0, 4, 5, 8], "hypothet": 0, "model": [0, 4, 6], "along": [0, 8], "true": [0, 2, 3, 4, 5, 6, 7, 8], "input": [0, 6, 7, 8], "x": [0, 8, 11], "output": [0, 6, 8], "y": [0, 8, 11], "paramet": [0, 2, 3, 4, 5, 6, 7, 8], "int": [0, 2, 4, 5, 6, 8], "The": [0, 2, 4, 5, 9], "number": [0, 2, 4, 5, 8], "set": [0, 2, 4, 5, 8, 11], "type": [0, 2, 3, 4, 5, 6, 7, 8], "tupl": [0, 4, 8], "ndarrai": [0, 2, 3, 4, 5, 6, 7, 8], "These": 0, "ar": [0, 4, 5, 7], "valu": [0, 4, 5, 6, 7, 8, 11], "uniform": 0, "nois": 0, "ad": [0, 8], "error": [0, 3, 4, 7], "between": [0, 4], "truth": 0, "plu": 0, "unifom": 0, "synthetic_sine_heteroscedast": [0, 11], "n_point": 0, "sampl": 0, "synthet": 0, "sine": 0, "heteroscedast": 0, "noisi": 0, "can": [0, 6, 10], "us": [0, 2, 5, 6, 7, 8], "train": 0, "assess": [0, 2, 3, 4, 5], "defin": 0, "toolbox": [1, 7, 10, 11], "data": [1, 4, 5, 9, 11], "modul": [1, 9], "metric": [1, 3, 4, 5, 6, 8, 9, 11], "metrics_calibr": [1, 9], "metrics_accuraci": [1, 9], "metrics_scoring_rul": [1, 9], "recalibr": [1, 4, 9], "util": [1, 6, 9], "viz": [1, 9], "main": 2, "qualiti": [2, 3, 4, 5], "quantif": [2, 3, 4, 5, 9], "get_all_accuracy_metr": 2, "y_pred": [2, 3, 4, 5, 6, 8], "y_true": [2, 3, 4, 5, 6, 8], "verbos": [2, 4], "comput": [2, 4, 5, 6, 11], "all": [2, 3, 7, 8, 11], "accuraci": [2, 3], "1d": [2, 3, 4, 5, 6, 7, 8], "arrai": [2, 3, 4, 5, 6, 7, 8], "mean": [2, 3, 4, 5, 6, 8], "held": [2, 3, 4, 5, 8], "out": [2, 3, 4, 5, 8], "label": [2, 3, 4, 5, 8], "bool": [2, 4, 5, 7, 8], "activ": 2, "mode": 2, "dict": [2, 3], "str": [2, 3, 4, 6, 8], "float": [2, 3, 4, 5, 6, 8], "evalu": 2, "relat": 2, "get_all_adversarial_group_calibr": 2, "y_std": [2, 4, 5, 6, 8], "num_bin": [2, 4], "adversari": [2, 4, 8], "group": [2, 4, 8], "calibr": [2, 4, 6, 8, 9], "he": [2, 5], "bin": 2, "discret": [2, 4], "each": [2, 4, 5, 8], "inner": 2, "dictionari": [2, 3], "size": [2, 4, 5, 8], "get_all_average_calibr": 2, "averag": [2, 3, 4], "get_all_metr": [2, 11], "100": [2, 4, 8, 11], "resolut": [2, 5], "99": [2, 5], "scale": [2, 5, 6], "quantil": [2, 4, 5, 6], "whether": [2, 4, 5], "score": [2, 5], "ani": [2, 4, 7, 8], "get_all_scoring_rule_metr": 2, "rule": [2, 5], "get_all_sharpness_metr": 2, "sharp": [2, 4, 8], "involv": [3, 4, 5], "prediction_error_metr": 3, "get": [3, 4, 6], "A": [3, 4, 6], "mae": 3, "root": [3, 4], "squar": [3, 4], "rmse": 3, "median": 3, "absolut": [3, 4, 8], "mdae": 3, "rel": 3, "percent": 3, "differ": 3, "marpd": 3, "r": 3, "2": [3, 4, 8], "r2": 3, "pearson": 3, "": 3, "correl": 3, "coeffici": 3, "corr": 3, "adversarial_group_calibr": 4, "cali_typ": [4, 8], "prop_typ": 4, "interv": [4, 5, 6, 8], "num_group_bin": 4, "draw_with_replac": 4, "fals": [4, 8], "num_trial": 4, "num_group_draw": 4, "measur": 4, "one": [4, 6, 8], "mean_ab": [4, 8], "root_mean_sq": 4, "observ": [4, 6, 8, 11], "proport": [4, 6, 8], "center": [4, 5, 6], "below": 4, "probabl": [4, 5, 6], "0": [4, 5, 6, 8], "1": [4, 6, 8], "draw": 4, "subgroup": 4, "replac": 4, "trial": 4, "estim": [4, 6], "worst": [4, 8], "per": 4, "print": 4, "progress": 4, "statement": 4, "namespac": 4, "an": [4, 6, 11], "get_prediction_interv": 4, "recal_model": 4, "none": [4, 8], "correspond": [4, 11], "For": [4, 10, 11], "specifi": [4, 6], "level": [4, 6], "q": [4, 6], "must": [4, 6], "singleton": 4, "pair": 4, "5": 4, "i": [4, 5, 6, 9, 11], "e": [4, 6, 10], "ha": 4, "nomin": 4, "coverag": [4, 6], "equal": [4, 5], "check": [4, 5, 7], "option": [4, 8], "isotonicregress": [4, 6], "appli": 4, "befor": 4, "lower": [4, 5, 8], "upper": [4, 5, 8], "bound": [4, 5, 8], "get_proportion_in_interv": 4, "fall": [4, 8], "singl": [4, 6, 8], "scalar": [4, 6], "get_proportion_list": 4, "expect": [4, 6, 8], "rang": [4, 8], "here": [4, 11], "vector": [4, 8, 11], "case": 4, "memori": 4, "constraint": 4, "sklearn": [4, 6], "isoton": [4, 6], "regress": [4, 6], "two": 4, "numpi": [4, 7], "get_proportion_lists_vector": [4, 8], "faster": 4, "execut": 4, "suit": [4, 10], "when": 4, "get_proportion_under_quantil": 4, "get_quantil": 4, "achiev": 4, "mean_absolute_calibration_error": 4, "ec": 4, "while": 4, "much": 4, "higher": 4, "requir": [4, 10], "mai": 4, "fail": 4, "run": [4, 10], "larger": 4, "calcul": 4, "miscalibration_area": 4, "miscalibr": [4, 8], "area": [4, 8], "howev": 4, "integr": 4, "taken": [4, 11], "trace": 4, "curv": [4, 8], "In": 4, "limit": 4, "converg": 4, "same": [4, 7], "root_mean_squared_calibration_error": 4, "overal": [4, 5], "confid": [4, 8], "quantifi": 4, "proper": 5, "check_scor": 5, "start_q": 5, "01": 5, "end_q": 5, "neg": 5, "orient": 5, "test": [5, 10], "weight": 5, "over": 5, "list": [5, 8], "scan": 5, "sequenc": 5, "distribut": 5, "start": [5, 9], "end": 5, "smaller": 5, "more": 5, "desir": 5, "crps_gaussian": 5, "continu": 5, "rank": 5, "gaussian": 5, "crp": 5, "heldout": 5, "interval_scor": 5, "start_p": 5, "end_p": 5, "percentil": 5, "where": 5, "p": 5, "amount": 5, "captur": 5, "form": 5, "nll_gaussian": 5, "log": 5, "likelihood": 5, "procedur": 6, "aim": 6, "improv": 6, "predictor": 6, "get_interval_recalibr": 6, "fit": 6, "take": 6, "callabl": 6, "union": [6, 7, 8], "get_q_idx": 6, "exp_prop": [6, 8], "index": 6, "element": [6, 7], "approxim": 6, "get_quantile_recalibr": 6, "get_std_recalibr": 6, "y_mean": 6, "criterion": 6, "ma_cal": 6, "ratio": [6, 8], "uncalibr": 6, "optim": 6, "dure": 6, "rms_cal": 6, "miscal": 6, "iso_rec": 6, "obs_prop": [6, 8], "algorithm": 6, "base": 6, "map": 6, "provid": 6, "necessari": 6, "adjust": 6, "produc": 6, "better": 6, "span": 6, "optimize_recalibration_ratio": 6, "factor": 6, "uniformli": 6, "search": 6, "via": [6, 10], "black": 6, "box": 6, "opt_ratio": 6, "best": 6, "updat": [6, 8, 10], "written": 6, "variou": 7, "within": 7, "assert_is_flat_same_shap": 7, "arg": 7, "length": 7, "noreturn": [7, 8], "flat": 7, "shape": 7, "els": 7, "rais": 7, "assert": 7, "assert_is_posit": 7, "posit": 7, "visual": [8, 9], "filter_subset": 8, "input_list": 8, "n_subset": 8, "keep": 8, "onli": [8, 10], "indic": 8, "plot": 8, "after": 8, "filter": 8, "reduc": 8, "plot_adversarial_group_calibr": 8, "curve_label": 8, "group_siz": 8, "score_mean": 8, "score_stderr": 8, "ax": 8, "vari": 8, "record": 8, "occur": 8, "legend": 8, "devat": 8, "matplotlib": 8, "object": 8, "plot_calibr": 8, "show": 8, "v": 8, "displai": 8, "plot_interv": 8, "ylim": 8, "num_stds_confidence_bound": 8, "versu": 8, "axi": 8, "width": 8, "term": 8, "plot_intervals_ord": 8, "order": 8, "plot_residuals_vs_std": 8, "residu": 8, "plot_sharp": 8, "plot_xi": 8, "xlim": 8, "leg_loc": 8, "3": [8, 10], "dimension": 8, "associ": [8, 11], "band": 8, "locat": 8, "save_figur": 8, "file_nam": 8, "figur": 8, "ext_list": 8, "white_background": 8, "save": 8, "extens": 8, "name": 8, "imag": 8, "file": 8, "string": 8, "denot": 8, "background": 8, "white": 8, "set_styl": 8, "style_str": 8, "default": 8, "style": 8, "update_rc": 8, "key_str": 8, "matplotlibrc": 8, "github": [9, 10], "websit": 9, "homepag": 9, "instal": 9, "quick": 9, "minim": 9, "exampl": [9, 10], "6": 10, "lightweight": 10, "packag": 10, "pip": 10, "full": 10, "latest": 10, "git": 10, "clone": 10, "http": 10, "com": 10, "cd": 10, "To": 10, "verifi": 10, "correct": 10, "you": 10, "sourc": 10, "shell": 10, "run_all_test": 10, "sh": 10, "tutori": 11, "longer": 11, "see": 11, "predictions_std": 11, "respect": 11, "uncertainty_toolbox": 11, "uct": 11, "load": 11}, "objects": {"uncertainty_toolbox": [[0, 0, 0, "-", "data"], [2, 0, 0, "-", "metrics"], [3, 0, 0, "-", "metrics_accuracy"], [4, 0, 0, "-", "metrics_calibration"], [5, 0, 0, "-", "metrics_scoring_rule"], [6, 0, 0, "-", "recalibration"], [7, 0, 0, "-", "utils"], [8, 0, 0, "-", "viz"]], "uncertainty_toolbox.data": [[0, 1, 1, "", "synthetic_arange_random"], [0, 1, 1, "", "synthetic_sine_heteroscedastic"]], "uncertainty_toolbox.metrics": [[2, 1, 1, "", "get_all_accuracy_metrics"], [2, 1, 1, "", "get_all_adversarial_group_calibration"], [2, 1, 1, "", "get_all_average_calibration"], [2, 1, 1, "", "get_all_metrics"], [2, 1, 1, "", "get_all_scoring_rule_metrics"], [2, 1, 1, "", "get_all_sharpness_metrics"]], "uncertainty_toolbox.metrics_accuracy": [[3, 1, 1, "", "prediction_error_metrics"]], "uncertainty_toolbox.metrics_calibration": [[4, 1, 1, "", "adversarial_group_calibration"], [4, 1, 1, "", "get_prediction_interval"], [4, 1, 1, "", "get_proportion_in_interval"], [4, 1, 1, "", "get_proportion_lists"], [4, 1, 1, "", "get_proportion_lists_vectorized"], [4, 1, 1, "", "get_proportion_under_quantile"], [4, 1, 1, "", "get_quantile"], [4, 1, 1, "", "mean_absolute_calibration_error"], [4, 1, 1, "", "miscalibration_area"], [4, 1, 1, "", "root_mean_squared_calibration_error"], [4, 1, 1, "", "sharpness"]], "uncertainty_toolbox.metrics_scoring_rule": [[5, 1, 1, "", "check_score"], [5, 1, 1, "", "crps_gaussian"], [5, 1, 1, "", "interval_score"], [5, 1, 1, "", "nll_gaussian"]], "uncertainty_toolbox.recalibration": [[6, 1, 1, "", "get_interval_recalibrator"], [6, 1, 1, "", "get_q_idx"], [6, 1, 1, "", "get_quantile_recalibrator"], [6, 1, 1, "", "get_std_recalibrator"], [6, 1, 1, "", "iso_recal"], [6, 1, 1, "", "optimize_recalibration_ratio"]], "uncertainty_toolbox.utils": [[7, 1, 1, "", "assert_is_flat_same_shape"], [7, 1, 1, "", "assert_is_positive"]], "uncertainty_toolbox.viz": [[8, 1, 1, "", "filter_subset"], [8, 1, 1, "", "plot_adversarial_group_calibration"], [8, 1, 1, "", "plot_calibration"], [8, 1, 1, "", "plot_intervals"], [8, 1, 1, "", "plot_intervals_ordered"], [8, 1, 1, "", "plot_residuals_vs_stds"], [8, 1, 1, "", "plot_sharpness"], [8, 1, 1, "", "plot_xy"], [8, 1, 1, "", "save_figure"], [8, 1, 1, "", "set_style"], [8, 1, 1, "", "update_rc"]]}, "objtypes": {"0": "py:module", "1": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"]}, "titleterms": {"data": 0, "modul": [0, 2, 3, 4, 5, 6, 7, 8], "uncertainty_toolbox": [0, 2, 3, 4, 5, 6, 7, 8], "api": 1, "refer": 1, "metric": 2, "metrics_accuraci": 3, "metrics_calibr": 4, "metrics_scoring_rul": 5, "recalibr": 6, "util": 7, "viz": 8, "uncertainti": 9, "toolbox": 9, "document": 9, "user": 9, "guid": 9, "instal": 10, "quick": 11, "start": 11, "minim": 11, "exampl": 11}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 6, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 56}})