From fb08716c91c79f550e7192db55d8aead8d7f1bf8 Mon Sep 17 00:00:00 2001 From: Emeli Dral Date: Fri, 17 Feb 2023 15:34:20 +0000 Subject: [PATCH] Version 0.2.5 Updates sample notebooks: fixed evidently import --- .../evidently_metric_presets.ipynb | 71 +-- .../sample_notebooks/evidently_metrics.ipynb | 5 +- .../evidently_test_presets.ipynb | 3 +- .../sample_notebooks/evidently_tests.ipynb | 17 +- .../getting_started_tutorial.ipynb | 564 +++++++++++++++++- src/evidently/_version.py | 2 +- 6 files changed, 592 insertions(+), 70 deletions(-) diff --git a/examples/sample_notebooks/evidently_metric_presets.ipynb b/examples/sample_notebooks/evidently_metric_presets.ipynb index a6e6780d03..1e8aa1a2ab 100644 --- a/examples/sample_notebooks/evidently_metric_presets.ipynb +++ b/examples/sample_notebooks/evidently_metric_presets.ipynb @@ -16,7 +16,6 @@ "try:\n", " import evidently\n", "except:\n", - " !npm install -g yarn\n", " !pip install git+https://github.com/evidentlyai/evidently.git" ] }, @@ -128,6 +127,15 @@ "iris_cur['prediction'] = model.predict(iris_cur[iris_data.feature_names])" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "iris.head()" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -265,64 +273,9 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "
Loading...
\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "multiclass_cat_target_drift_report = Report(metrics=[\n", " TargetDriftPreset(num_stattest='ks', cat_stattest='psi'),\n", @@ -377,7 +330,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.11.0" } }, "nbformat": 4, diff --git a/examples/sample_notebooks/evidently_metrics.ipynb b/examples/sample_notebooks/evidently_metrics.ipynb index ca661940ed..2ff1c0e996 100644 --- a/examples/sample_notebooks/evidently_metrics.ipynb +++ b/examples/sample_notebooks/evidently_metrics.ipynb @@ -16,7 +16,6 @@ "try:\n", " import evidently\n", "except:\n", - " !npm install -g yarn\n", " !pip install git+https://github.com/evidentlyai/evidently.git" ] }, @@ -166,7 +165,7 @@ "#dataset-level metrics\n", "data_drift_dataset_report = Report(metrics=[\n", " DatasetDriftMetric(),\n", - " DataDriftTable(), \n", + " DataDriftTable(num_stattest='kl_div', cat_stattest='psi'), \n", "])\n", "\n", "data_drift_dataset_report.run(reference_data=adult_ref, current_data=adult_cur)\n", @@ -465,7 +464,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.11.0" } }, "nbformat": 4, diff --git a/examples/sample_notebooks/evidently_test_presets.ipynb b/examples/sample_notebooks/evidently_test_presets.ipynb index 39da39f756..c3907f46ef 100644 --- a/examples/sample_notebooks/evidently_test_presets.ipynb +++ b/examples/sample_notebooks/evidently_test_presets.ipynb @@ -18,7 +18,6 @@ "try:\n", " import evidently\n", "except:\n", - " !npm install -g yarn\n", " !pip install git+https://github.com/evidentlyai/evidently.git" ] }, @@ -332,7 +331,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.11.0" } }, "nbformat": 4, diff --git a/examples/sample_notebooks/evidently_tests.ipynb b/examples/sample_notebooks/evidently_tests.ipynb index 8166b4ad90..d51cb16410 100644 --- a/examples/sample_notebooks/evidently_tests.ipynb +++ b/examples/sample_notebooks/evidently_tests.ipynb @@ -18,7 +18,6 @@ "try:\n", " import evidently\n", "except:\n", - " !npm install -g yarn\n", " !pip install git+https://github.com/evidentlyai/evidently.git" ] }, @@ -252,10 +251,20 @@ " TestCorrelationChanges(),\n", "])\n", "\n", - "data_quality_dataset_tests.run(reference_data=adult_ref, current_data=adult_cur)\n", + "data_quality_dataset_tests.run(reference_data=iris_ref, current_data=iris_cur)\n", "data_quality_dataset_tests" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e270bbd", + "metadata": {}, + "outputs": [], + "source": [ + "iris_cur.head()" + ] + }, { "cell_type": "code", "execution_count": null, @@ -322,7 +331,7 @@ "source": [ "#column-level tests\n", "data_drift_column_tests = TestSuite(tests=[\n", - " TestColumnDrift(column_name='education-num')\n", + " TestColumnDrift(column_name='education-num', stattest='psi', stattest_threshold=0.3)\n", "])\n", "\n", "data_drift_column_tests.run(reference_data=adult_ref, current_data=adult_cur)\n", @@ -504,7 +513,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.11.0" } }, "nbformat": 4, diff --git a/examples/sample_notebooks/getting_started_tutorial.ipynb b/examples/sample_notebooks/getting_started_tutorial.ipynb index 5dd57ccd3e..ff07e6c52f 100644 --- a/examples/sample_notebooks/getting_started_tutorial.ipynb +++ b/examples/sample_notebooks/getting_started_tutorial.ipynb @@ -1 +1,563 @@ -{"cells":[{"cell_type":"markdown","metadata":{"id":"630a2437"},"source":["# Getting Started Tutorial"]},{"cell_type":"markdown","metadata":{"id":"3c3dfc08"},"source":["To install Evidently using the pip package manager, run:\n","\n","```$ pip install evidently```\n","\n","\n","If you want to see reports inside a Jupyter notebook, you need to also install the Jupyter nbextension. After installing evidently, run the two following commands in the terminal from the Evidently directory.\n","\n","To install jupyter nbextension, run:\n","\n","```$ jupyter nbextension install --sys-prefix --symlink --overwrite --py evidently```\n","\n","To enable it, run:\n","\n","```$ jupyter nbextension enable evidently --py --sys-prefix```\n","\n","That's it!"]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":165075,"status":"ok","timestamp":1670449882138,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"aec9a1cc","outputId":"3935b908-73f5-436f-9fca-af065e330a75"},"outputs":[{"name":"stdout","output_type":"stream","text":["\u001b[K\u001b[?25h\n","\u003e yarn@1.22.19 preinstall /tools/node/lib/node_modules/yarn\n","\u003e :; (node ./preinstall.js \u003e /dev/null 2\u003e\u00261 || true)\n","\n","/tools/node/bin/yarn -\u003e /tools/node/lib/node_modules/yarn/bin/yarn.js\n","/tools/node/bin/yarnpkg -\u003e /tools/node/lib/node_modules/yarn/bin/yarn.js\n","+ yarn@1.22.19\n","added 1 package in 1.237s\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting git+https://github.com/evidentlyai/evidently.git\n"," Cloning https://github.com/evidentlyai/evidently.git to /tmp/pip-req-build-yp3popjj\n"," Running command git clone -q https://github.com/evidentlyai/evidently.git /tmp/pip-req-build-yp3popjj\n","Collecting dataclasses\u003e=0.6\n"," Downloading dataclasses-0.6-py3-none-any.whl (14 kB)\n","Requirement already satisfied: plotly\u003e=5.5.0 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (5.5.0)\n","Requirement already satisfied: statsmodels\u003e=0.12.2 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (0.12.2)\n","Requirement already satisfied: scikit-learn\u003e=0.24.0 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (1.0.2)\n","Requirement already satisfied: pandas\u003e=1.1.5 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (1.3.5)\n","Requirement already satisfied: numpy\u003e=1.19.5 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (1.21.6)\n","Requirement already satisfied: scipy\u003e=1.5.4 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (1.7.3)\n","Requirement already satisfied: requests\u003e=2.19.0 in /usr/local/lib/python3.8/dist-packages (from evidently==0.2.0) (2.23.0)\n","Collecting PyYAML~=5.1\n"," Downloading PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl (662 kB)\n","\u001b[K |████████████████████████████████| 662 kB 4.0 MB/s \n","\u001b[?25hRequirement already satisfied: python-dateutil\u003e=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas\u003e=1.1.5-\u003eevidently==0.2.0) (2.8.2)\n","Requirement already satisfied: pytz\u003e=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas\u003e=1.1.5-\u003eevidently==0.2.0) (2022.6)\n","Requirement already satisfied: tenacity\u003e=6.2.0 in /usr/local/lib/python3.8/dist-packages (from plotly\u003e=5.5.0-\u003eevidently==0.2.0) (8.1.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from plotly\u003e=5.5.0-\u003eevidently==0.2.0) (1.15.0)\n","Requirement already satisfied: idna\u003c3,\u003e=2.5 in /usr/local/lib/python3.8/dist-packages (from requests\u003e=2.19.0-\u003eevidently==0.2.0) (2.10)\n","Requirement already satisfied: chardet\u003c4,\u003e=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests\u003e=2.19.0-\u003eevidently==0.2.0) (3.0.4)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,\u003c1.26,\u003e=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests\u003e=2.19.0-\u003eevidently==0.2.0) (1.24.3)\n","Requirement already satisfied: certifi\u003e=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests\u003e=2.19.0-\u003eevidently==0.2.0) (2022.9.24)\n","Requirement already satisfied: joblib\u003e=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit-learn\u003e=0.24.0-\u003eevidently==0.2.0) (1.2.0)\n","Requirement already satisfied: threadpoolctl\u003e=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn\u003e=0.24.0-\u003eevidently==0.2.0) (3.1.0)\n","Requirement already satisfied: patsy\u003e=0.5 in /usr/local/lib/python3.8/dist-packages (from statsmodels\u003e=0.12.2-\u003eevidently==0.2.0) (0.5.3)\n","Building wheels for collected packages: evidently\n"," Building wheel for evidently (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for evidently: filename=evidently-0.2.0-py3-none-any.whl size=2038267 sha256=c742e0abccd02ab45663c8a00b5df98aaf95ea5454fd1f289e94d122d5486d62\n"," Stored in directory: /tmp/pip-ephem-wheel-cache-sd5rgvvd/wheels/82/ca/bf/c32566cd2582af83db289709a0c3ed324a309883ef7048a316\n","Successfully built evidently\n","Installing collected packages: PyYAML, dataclasses, evidently\n"," Attempting uninstall: PyYAML\n"," Found existing installation: PyYAML 6.0\n"," Uninstalling PyYAML-6.0:\n"," Successfully uninstalled PyYAML-6.0\n","Successfully installed PyYAML-5.4.1 dataclasses-0.6 evidently-0.2.0\n"]}],"source":["try:\n"," import evidently\n","except:\n"," !npm install -g yarn\n"," !pip install git+https://github.com/evidentlyai/evidently.git"]},{"cell_type":"code","execution_count":2,"metadata":{"executionInfo":{"elapsed":2869,"status":"ok","timestamp":1670449884998,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"0c00061b"},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","\n","from sklearn.datasets import fetch_california_housing\n","\n","from evidently import ColumnMapping\n","\n","from evidently.report import Report\n","from evidently.metrics.base_metric import generate_column_metrics\n","from evidently.metric_preset import DataDriftPreset, TargetDriftPreset, DataQualityPreset, RegressionPreset\n","from evidently.metrics import *\n","\n","from evidently.test_suite import TestSuite\n","from evidently.tests.base_test import generate_column_tests\n","from evidently.test_preset import DataStabilityTestPreset, NoTargetPerformanceTestPreset\n","from evidently.tests import *"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1670449884998,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"56d3e494"},"outputs":[],"source":["import warnings\n","warnings.filterwarnings('ignore')\n","warnings.simplefilter('ignore')"]},{"cell_type":"markdown","metadata":{"id":"1dca5a2c"},"source":["## Load Data"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":3515,"status":"ok","timestamp":1670449888508,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"5c9d9f9e"},"outputs":[],"source":["data = fetch_california_housing(as_frame=True)\n","housing_data = data.frame"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1670449888509,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"60692ed0"},"outputs":[],"source":["housing_data.rename(columns={'MedHouseVal': 'target'}, inplace=True)\n","housing_data['prediction'] = housing_data['target'].values + np.random.normal(0, 5, housing_data.shape[0])"]},{"cell_type":"code","execution_count":6,"metadata":{"executionInfo":{"elapsed":7,"status":"ok","timestamp":1670449888509,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"f48b2f20"},"outputs":[],"source":["reference = housing_data.sample(n=5000, replace=False)\n","current = housing_data.sample(n=5000, replace=False)"]},{"cell_type":"markdown","metadata":{"id":"fedb4612"},"source":["## Report"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":0,"output_embedded_package_id":"1fhn3NeCqspz5iFigvZSIKQiNbXzdQC-d"},"executionInfo":{"elapsed":8243,"status":"ok","timestamp":1670449896746,"user":{"displayName":"Emeli Dral","userId":"07849725042886651837"},"user_tz":0},"id":"bb77cbe1","outputId":"ff3d9b4c-19ff-4f12-c2d3-45cafeb8354c"},"outputs":[{"data":{"text/plain":"Output hidden; open in https://colab.research.google.com to view."},"metadata":{},"output_type":"display_data"}],"source":["report = Report(metrics=[\n"," DataDriftPreset(), \n","])\n","\n","report.run(reference_data=reference, current_data=current)\n","report"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"1bCO6AcggV2v5gw21oLWAGQo5RYeP_yOu"},"id":"7e7de377","outputId":"445a5c02-06b6-4149-a983-b9f022b57dba"},"outputs":[],"source":["report = Report(metrics=[\n"," ColumnSummaryMetric(column_name='AveRooms'),\n"," ColumnQuantileMetric(column_name='AveRooms', quantile=0.25),\n"," ColumnDriftMetric(column_name='AveRooms'),\n"," \n","])\n","\n","report.run(reference_data=reference, current_data=current)\n","report"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"1mYzaITssvBlfDbpJZtSKEGmVxMtUsN0J"},"id":"9e95865e","outputId":"962857b9-c1fb-4186-ad33-673d562ade54"},"outputs":[],"source":["report = Report(metrics=[\n"," generate_column_metrics(ColumnQuantileMetric, parameters={'quantile':0.25}, columns=['AveRooms', 'AveBedrms']),\n","])\n","\n","report.run(reference_data=reference, current_data=current)\n","report"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"1JUEabeT61mW16eviYucIPleIfXVoYdc-"},"id":"fc3f8a8f","outputId":"cc932a1c-8fe0-4ac0-a875-cc25aaa45ac2"},"outputs":[],"source":["report = Report(metrics=[\n"," ColumnSummaryMetric(column_name='AveRooms'),\n"," generate_column_metrics(ColumnQuantileMetric, parameters={'quantile':0.25}, columns='num'),\n"," DataDriftPreset()\n","])\n","\n","report.run(reference_data=reference, current_data=current)\n","report"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"d72ca314"},"outputs":[{"data":{"text/plain":["{'metrics': [{'metric': 'ColumnSummaryMetric',\n"," 'result': {'column_name': 'AveRooms',\n"," 'column_type': 'num',\n"," 'reference_characteristics': {'number_of_rows': 5000,\n"," 'count': 5000,\n"," 'mean': 5.41,\n"," 'std': 2.62,\n"," 'min': 1.13,\n"," 'p25': 4.46,\n"," 'p50': 5.22,\n"," 'p75': 6.01,\n"," 'max': 132.53,\n"," 'unique': 4902,\n"," 'unique_percentage': 98.04,\n"," 'missing': 0,\n"," 'missing_percentage': 0.0,\n"," 'infinite_count': 0,\n"," 'infinite_percentage': 0.0,\n"," 'most_common': 6.0,\n"," 'most_common_percentage': 0.18},\n"," 'current_characteristics': {'number_of_rows': 5000,\n"," 'count': 5000,\n"," 'mean': 5.46,\n"," 'std': 2.76,\n"," 'min': 0.85,\n"," 'p25': 4.44,\n"," 'p50': 5.22,\n"," 'p75': 6.08,\n"," 'max': 132.53,\n"," 'unique': 4904,\n"," 'unique_percentage': 98.08,\n"," 'missing': 0,\n"," 'missing_percentage': 0.0,\n"," 'infinite_count': 0,\n"," 'infinite_percentage': 0.0,\n"," 'most_common': 4.4,\n"," 'most_common_percentage': 0.08}}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'AveBedrms',\n"," 'quantile': 0.25,\n"," 'current': 1.0070422535211268,\n"," 'reference': 1.004086039892326}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'AveOccup',\n"," 'quantile': 0.25,\n"," 'current': 2.44328465580516,\n"," 'reference': 2.4321641663217095}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'AveRooms',\n"," 'quantile': 0.25,\n"," 'current': 4.444507485494254,\n"," 'reference': 4.4573415745290745}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'HouseAge',\n"," 'quantile': 0.25,\n"," 'current': 19.0,\n"," 'reference': 18.0}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'Latitude',\n"," 'quantile': 0.25,\n"," 'current': 33.93,\n"," 'reference': 33.94}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'Longitude',\n"," 'quantile': 0.25,\n"," 'current': -121.83,\n"," 'reference': -121.82}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'MedInc',\n"," 'quantile': 0.25,\n"," 'current': 2.566025,\n"," 'reference': 2.56}},\n"," {'metric': 'ColumnQuantileMetric',\n"," 'result': {'column_name': 'Population',\n"," 'quantile': 0.25,\n"," 'current': 785.75,\n"," 'reference': 795.75}},\n"," {'metric': 'DatasetDriftMetric',\n"," 'result': {'drift_share': 0.5,\n"," 'number_of_columns': 10,\n"," 'number_of_drifted_columns': 0,\n"," 'share_of_drifted_columns': 0.0,\n"," 'dataset_drift': False}},\n"," {'metric': 'DataDriftTable',\n"," 'result': {'number_of_columns': 10,\n"," 'number_of_drifted_columns': 0,\n"," 'share_of_drifted_columns': 0.0,\n"," 'dataset_drift': False,\n"," 'drift_by_columns': {'target': {'column_name': 'target',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.01557681351221734,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'prediction': {'column_name': 'prediction',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.01546798152507848,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'AveBedrms': {'column_name': 'AveBedrms',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.012523411778532187,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'AveOccup': {'column_name': 'AveOccup',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.019119938482313907,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'AveRooms': {'column_name': 'AveRooms',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.020646185722527074,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'HouseAge': {'column_name': 'HouseAge',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.01850666997783238,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'Latitude': {'column_name': 'Latitude',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.010420141936907787,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'Longitude': {'column_name': 'Longitude',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.012766439595767638,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'MedInc': {'column_name': 'MedInc',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.032384989237177586,\n"," 'drift_detected': False,\n"," 'threshold': 0.1},\n"," 'Population': {'column_name': 'Population',\n"," 'column_type': 'num',\n"," 'stattest_name': 'Wasserstein distance (normed)',\n"," 'drift_score': 0.024097319257549068,\n"," 'drift_detected': False,\n"," 'threshold': 0.1}}}}]}"]},"execution_count":11,"metadata":{},"output_type":"execute_result"}],"source":["report.as_dict()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"ab320537"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'{\"version\": \"0.2.0\", \"timestamp\": \"2022-12-07 21:51:33.765568\", \"metrics\": [{\"metric\": \"ColumnSummaryMetric\", \"result\": {\"column_name\": \"AveRooms\", \"column_type\": \"num\", \"reference_characteristics\": {\"number_of_rows\": 5000, \"count\": 5000, \"mean\": 5.41, \"std\": 2.62, \"min\": 1.13, \"p25\": 4.46, \"p50\": 5.22, \"p75\": 6.01, \"max\": 132.53, \"unique\": 4902, \"unique_percentage\": 98.04, \"missing\": 0, \"missing_percentage\": 0.0, \"infinite_count\": 0, \"infinite_percentage\": 0.0, \"most_common\": 6.0, \"most_common_percentage\": 0.18}, \"current_characteristics\": {\"number_of_rows\": 5000, \"count\": 5000, \"mean\": 5.46, \"std\": 2.76, \"min\": 0.85, \"p25\": 4.44, \"p50\": 5.22, \"p75\": 6.08, \"max\": 132.53, \"unique\": 4904, \"unique_percentage\": 98.08, \"missing\": 0, \"missing_percentage\": 0.0, \"infinite_count\": 0, \"infinite_percentage\": 0.0, \"most_common\": 4.4, \"most_common_percentage\": 0.08}}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"AveBedrms\", \"quantile\": 0.25, \"current\": 1.0070422535211268, \"reference\": 1.004086039892326}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"AveOccup\", \"quantile\": 0.25, \"current\": 2.44328465580516, \"reference\": 2.4321641663217095}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"AveRooms\", \"quantile\": 0.25, \"current\": 4.444507485494254, \"reference\": 4.4573415745290745}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"HouseAge\", \"quantile\": 0.25, \"current\": 19.0, \"reference\": 18.0}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"Latitude\", \"quantile\": 0.25, \"current\": 33.93, \"reference\": 33.94}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"Longitude\", \"quantile\": 0.25, \"current\": -121.83, \"reference\": -121.82}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"MedInc\", \"quantile\": 0.25, \"current\": 2.566025, \"reference\": 2.56}}, {\"metric\": \"ColumnQuantileMetric\", \"result\": {\"column_name\": \"Population\", \"quantile\": 0.25, \"current\": 785.75, \"reference\": 795.75}}, {\"metric\": \"DatasetDriftMetric\", \"result\": {\"drift_share\": 0.5, \"number_of_columns\": 10, \"number_of_drifted_columns\": 0, \"share_of_drifted_columns\": 0.0, \"dataset_drift\": false}}, {\"metric\": \"DataDriftTable\", \"result\": {\"number_of_columns\": 10, \"number_of_drifted_columns\": 0, \"share_of_drifted_columns\": 0.0, \"dataset_drift\": false, \"drift_by_columns\": {\"target\": {\"column_name\": \"target\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.01557681351221734, \"drift_detected\": false, \"threshold\": 0.1}, \"prediction\": {\"column_name\": \"prediction\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.01546798152507848, \"drift_detected\": false, \"threshold\": 0.1}, \"AveBedrms\": {\"column_name\": \"AveBedrms\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.012523411778532187, \"drift_detected\": false, \"threshold\": 0.1}, \"AveOccup\": {\"column_name\": \"AveOccup\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.019119938482313907, \"drift_detected\": false, \"threshold\": 0.1}, \"AveRooms\": {\"column_name\": \"AveRooms\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.020646185722527074, \"drift_detected\": false, \"threshold\": 0.1}, \"HouseAge\": {\"column_name\": \"HouseAge\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.01850666997783238, \"drift_detected\": false, \"threshold\": 0.1}, \"Latitude\": {\"column_name\": \"Latitude\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.010420141936907787, \"drift_detected\": false, \"threshold\": 0.1}, \"Longitude\": {\"column_name\": \"Longitude\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.012766439595767638, \"drift_detected\": false, \"threshold\": 0.1}, \"MedInc\": {\"column_name\": \"MedInc\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.032384989237177586, \"drift_detected\": false, \"threshold\": 0.1}, \"Population\": {\"column_name\": \"Population\", \"column_type\": \"num\", \"stattest_name\": \"Wasserstein distance (normed)\", \"drift_score\": 0.024097319257549068, \"drift_detected\": false, \"threshold\": 0.1}}}}]}'"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["report.json()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"baf1fd25"},"outputs":[],"source":["#report.save_html('report.html')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"fd4e9085"},"outputs":[],"source":["#report.save_json('report.json')"]},{"cell_type":"markdown","metadata":{"id":"1ea31ae7"},"source":["## Test Suite "]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"1AJKCnZwYYl2cLoRWtKeqF7iIt6Ruuqz-"},"id":"f2311155","outputId":"825e12fb-8ed6-475e-b046-189225e4cb51"},"outputs":[],"source":["tests = TestSuite(tests=[\n"," TestNumberOfColumnsWithMissingValues(),\n"," TestNumberOfRowsWithMissingValues(),\n"," TestNumberOfConstantColumns(),\n"," TestNumberOfDuplicatedRows(),\n"," TestNumberOfDuplicatedColumns(),\n"," TestColumnsType(),\n"," TestNumberOfDriftedColumns(),\n","])\n","\n","tests.run(reference_data=reference, current_data=current)\n","tests"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"1ohVLBp0ivJltJprlA1F-AVL7TcppIziI"},"id":"d78779bb","outputId":"c000d73a-dee7-42e4-b02f-fd08ea97af4a","scrolled":false},"outputs":[],"source":["suite = TestSuite(tests=[\n"," NoTargetPerformanceTestPreset(),\n","])\n","\n","suite.run(reference_data=reference, current_data=current)\n","suite"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"output_embedded_package_id":"11IMmTb-z3p9uT6Hlhj9DzmCEq_C9nn-6"},"id":"e92a0605","outputId":"4aafbda5-e521-44d5-bda1-fb5b65faff12"},"outputs":[],"source":["suite = TestSuite(tests=[\n"," TestColumnDrift('Population'),\n"," TestMeanInNSigmas('HouseAge'),\n"," NoTargetPerformanceTestPreset(columns=['AveRooms', 'AveBedrms', 'AveOccup'])\n","])\n","\n","suite.run(reference_data=reference, current_data=current)\n","suite"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7df650c7"},"outputs":[],"source":["suite = TestSuite(tests=[\n"," TestColumnDrift('Population'),\n"," TestShareOfOutRangeValues('Population'),\n"," generate_column_tests(TestMeanInNSigmas, columns='num'),\n"," \n","])\n","\n","suite.run(reference_data=reference, current_data=current)\n","suite"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"20da511b","scrolled":true},"outputs":[{"data":{"text/plain":["{'tests': [{'name': 'Drift per Column',\n"," 'description': 'The drift score for the feature **Population** is 0.024. The drift detection method is Wasserstein distance (normed). The drift detection threshold is 0.1.',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_drift',\n"," 'parameters': {'features': {'Population': {'stattest_name': 'Wasserstein distance (normed)',\n"," 'score': 0.024,\n"," 'stattest_threshold': 0.1,\n"," 'data_drift': False}}}},\n"," {'name': 'Share of Out-of-Range Values',\n"," 'description': 'The share of values out of range in the column **Population** is 0.0006 (3 out of 5000). The test threshold is eq=0 ± 1e-12.',\n"," 'status': 'FAIL',\n"," 'group': 'data_quality',\n"," 'parameters': {'condition': {'eq': 0 ± 1e-12},\n"," 'left': None,\n"," 'right': None,\n"," 'share_not_in_range': 0.0006}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **AveBedrms** is 1.1. The expected range is from -0.05 to 2.23',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'AveBedrms',\n"," 'n_sigmas': 2,\n"," 'current_mean': 1.1,\n"," 'reference_mean': 1.09,\n"," 'reference_std': 0.57}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **AveOccup** is 2.99. The expected range is from -35.1 to 41.7',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'AveOccup',\n"," 'n_sigmas': 2,\n"," 'current_mean': 2.99,\n"," 'reference_mean': 3.34,\n"," 'reference_std': 19.2}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **AveRooms** is 5.46. The expected range is from 0.17 to 10.7',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'AveRooms',\n"," 'n_sigmas': 2,\n"," 'current_mean': 5.46,\n"," 'reference_mean': 5.41,\n"," 'reference_std': 2.62}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **HouseAge** is 28.7. The expected range is from 3.42 to 54.2',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'HouseAge',\n"," 'n_sigmas': 2,\n"," 'current_mean': 28.73,\n"," 'reference_mean': 28.82,\n"," 'reference_std': 12.7}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **Latitude** is 35.6. The expected range is from 31.4 to 39.9',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'Latitude',\n"," 'n_sigmas': 2,\n"," 'current_mean': 35.64,\n"," 'reference_mean': 35.63,\n"," 'reference_std': 2.13}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **Longitude** is -120. The expected range is from -124 to -116',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'Longitude',\n"," 'n_sigmas': 2,\n"," 'current_mean': -119.59,\n"," 'reference_mean': -119.58,\n"," 'reference_std': 2.01}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **MedInc** is 3.9. The expected range is from 0.16 to 7.56',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'MedInc',\n"," 'n_sigmas': 2,\n"," 'current_mean': 3.9,\n"," 'reference_mean': 3.86,\n"," 'reference_std': 1.85}},\n"," {'name': 'Mean Value Stability',\n"," 'description': 'The mean value of the column **Population** is 1.42e+03. The expected range is from -728 to 3.58e+03',\n"," 'status': 'SUCCESS',\n"," 'group': 'data_quality',\n"," 'parameters': {'column_name': 'Population',\n"," 'n_sigmas': 2,\n"," 'current_mean': 1418.73,\n"," 'reference_mean': 1424.1,\n"," 'reference_std': 1076.04}}],\n"," 'summary': {'all_passed': False,\n"," 'total_tests': 10,\n"," 'success_tests': 9,\n"," 'failed_tests': 1,\n"," 'by_status': Counter({'SUCCESS': 9, 'FAIL': 1})}}"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["suite.as_dict()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"3c1c476f"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"},"text/plain":["'{\"version\": \"0.2.0\", \"timestamp\": \"2022-12-07 21:51:56.737031\", \"tests\": [{\"name\": \"Drift per Column\", \"description\": \"The drift score for the feature **Population** is 0.024. The drift detection method is Wasserstein distance (normed). The drift detection threshold is 0.1.\", \"status\": \"SUCCESS\", \"group\": \"data_drift\", \"parameters\": {\"features\": {\"Population\": {\"stattest_name\": \"Wasserstein distance (normed)\", \"score\": 0.024, \"stattest_threshold\": 0.1, \"data_drift\": false}}}}, {\"name\": \"Share of Out-of-Range Values\", \"description\": \"The share of values out of range in the column **Population** is 0.0006 (3 out of 5000). The test threshold is eq=0 \\\\u00b1 1e-12.\", \"status\": \"FAIL\", \"group\": \"data_quality\", \"parameters\": {\"condition\": {\"eq\": {\"value\": 0, \"relative\": 1e-06, \"absolute\": 1e-12}}, \"left\": null, \"right\": null, \"share_not_in_range\": 0.0006}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **AveBedrms** is 1.1. The expected range is from -0.05 to 2.23\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"AveBedrms\", \"n_sigmas\": 2, \"current_mean\": 1.1, \"reference_mean\": 1.09, \"reference_std\": 0.57}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **AveOccup** is 2.99. The expected range is from -35.1 to 41.7\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"AveOccup\", \"n_sigmas\": 2, \"current_mean\": 2.99, \"reference_mean\": 3.34, \"reference_std\": 19.2}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **AveRooms** is 5.46. The expected range is from 0.17 to 10.7\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"AveRooms\", \"n_sigmas\": 2, \"current_mean\": 5.46, \"reference_mean\": 5.41, \"reference_std\": 2.62}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **HouseAge** is 28.7. The expected range is from 3.42 to 54.2\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"HouseAge\", \"n_sigmas\": 2, \"current_mean\": 28.73, \"reference_mean\": 28.82, \"reference_std\": 12.7}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **Latitude** is 35.6. The expected range is from 31.4 to 39.9\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"Latitude\", \"n_sigmas\": 2, \"current_mean\": 35.64, \"reference_mean\": 35.63, \"reference_std\": 2.13}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **Longitude** is -120. The expected range is from -124 to -116\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"Longitude\", \"n_sigmas\": 2, \"current_mean\": -119.59, \"reference_mean\": -119.58, \"reference_std\": 2.01}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **MedInc** is 3.9. The expected range is from 0.16 to 7.56\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"MedInc\", \"n_sigmas\": 2, \"current_mean\": 3.9, \"reference_mean\": 3.86, \"reference_std\": 1.85}}, {\"name\": \"Mean Value Stability\", \"description\": \"The mean value of the column **Population** is 1.42e+03. The expected range is from -728 to 3.58e+03\", \"status\": \"SUCCESS\", \"group\": \"data_quality\", \"parameters\": {\"column_name\": \"Population\", \"n_sigmas\": 2, \"current_mean\": 1418.73, \"reference_mean\": 1424.1, \"reference_std\": 1076.04}}], \"summary\": {\"all_passed\": false, \"total_tests\": 10, \"success_tests\": 9, \"failed_tests\": 1, \"by_status\": {\"SUCCESS\": 9, \"FAIL\": 1}}}'"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["suite.json()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"e4225064"},"outputs":[],"source":["#suite.save_html('test_suite.html')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"5fedd579"},"outputs":[],"source":["#suite.save_json('test_suite.json')"]}],"metadata":{"colab":{"name":"","version":""},"kernelspec":{"display_name":"Python 3 (ipykernel)","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.13"}},"nbformat":4,"nbformat_minor":5} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ab9703de", + "metadata": { + "id": "630a2437" + }, + "source": [ + "# Getting Started Tutorial" + ] + }, + { + "cell_type": "markdown", + "id": "edf45dc9", + "metadata": { + "id": "3c3dfc08" + }, + "source": [ + "To install Evidently using the pip package manager, run:\n", + "\n", + "```$ pip install evidently```\n", + "\n", + "\n", + "If you want to see reports inside a Jupyter notebook, you need to also install the Jupyter nbextension. After installing evidently, run the two following commands in the terminal from the Evidently directory.\n", + "\n", + "To install jupyter nbextension, run:\n", + "\n", + "```$ jupyter nbextension install --sys-prefix --symlink --overwrite --py evidently```\n", + "\n", + "To enable it, run:\n", + "\n", + "```$ jupyter nbextension enable evidently --py --sys-prefix```\n", + "\n", + "That's it!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d1e357fe", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 165075, + "status": "ok", + "timestamp": 1670449882138, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "aec9a1cc", + "outputId": "3935b908-73f5-436f-9fca-af065e330a75" + }, + "outputs": [], + "source": [ + "try:\n", + " import evidently\n", + " print(evidently.__version__)\n", + "except:\n", + " !pip install git+https://github.com/evidentlyai/evidently.git" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "588ad0ef", + "metadata": { + "executionInfo": { + "elapsed": 2869, + "status": "ok", + "timestamp": 1670449884998, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "0c00061b" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "from sklearn.datasets import fetch_california_housing\n", + "\n", + "from evidently import ColumnMapping\n", + "\n", + "from evidently.report import Report\n", + "from evidently.metrics.base_metric import generate_column_metrics\n", + "from evidently.metric_preset import DataDriftPreset, TargetDriftPreset, DataQualityPreset, RegressionPreset\n", + "from evidently.metrics import *\n", + "\n", + "from evidently.test_suite import TestSuite\n", + "from evidently.tests.base_test import generate_column_tests\n", + "from evidently.test_preset import DataStabilityTestPreset, NoTargetPerformanceTestPreset, RegressionTestPreset\n", + "from evidently.tests import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65cb6430", + "metadata": { + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1670449884998, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "56d3e494" + }, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "warnings.simplefilter('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "b980a390", + "metadata": { + "id": "1dca5a2c" + }, + "source": [ + "## Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "61cdfdbb", + "metadata": { + "executionInfo": { + "elapsed": 3515, + "status": "ok", + "timestamp": 1670449888508, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "5c9d9f9e" + }, + "outputs": [], + "source": [ + "data = fetch_california_housing(as_frame=True)\n", + "housing_data = data.frame" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "771f0c60", + "metadata": { + "executionInfo": { + "elapsed": 8, + "status": "ok", + "timestamp": 1670449888509, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "60692ed0" + }, + "outputs": [], + "source": [ + "housing_data.rename(columns={'MedHouseVal': 'target'}, inplace=True)\n", + "housing_data['prediction'] = housing_data['target'].values + np.random.normal(0, 5, housing_data.shape[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48d4d376", + "metadata": { + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1670449888509, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "f48b2f20" + }, + "outputs": [], + "source": [ + "reference = housing_data.sample(n=5000, replace=False)\n", + "current = housing_data.sample(n=5000, replace=False)" + ] + }, + { + "cell_type": "markdown", + "id": "0e9ab9e3", + "metadata": { + "id": "fedb4612" + }, + "source": [ + "## Report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ea8aea7", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 0, + "output_embedded_package_id": "1fhn3NeCqspz5iFigvZSIKQiNbXzdQC-d" + }, + "executionInfo": { + "elapsed": 8243, + "status": "ok", + "timestamp": 1670449896746, + "user": { + "displayName": "Emeli Dral", + "userId": "07849725042886651837" + }, + "user_tz": 0 + }, + "id": "bb77cbe1", + "outputId": "ff3d9b4c-19ff-4f12-c2d3-45cafeb8354c" + }, + "outputs": [], + "source": [ + "report = Report(metrics=[\n", + " DataDriftPreset(), \n", + "])\n", + "\n", + "report.run(reference_data=reference, current_data=current)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4284a51", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "1bCO6AcggV2v5gw21oLWAGQo5RYeP_yOu" + }, + "id": "7e7de377", + "outputId": "445a5c02-06b6-4149-a983-b9f022b57dba" + }, + "outputs": [], + "source": [ + "report = Report(metrics=[\n", + " ColumnSummaryMetric(column_name='AveRooms'),\n", + " ColumnQuantileMetric(column_name='AveRooms', quantile=0.25),\n", + " ColumnDriftMetric(column_name='AveRooms'),\n", + " \n", + "])\n", + "\n", + "report.run(reference_data=reference, current_data=current)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "481d4118", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "1mYzaITssvBlfDbpJZtSKEGmVxMtUsN0J" + }, + "id": "9e95865e", + "outputId": "962857b9-c1fb-4186-ad33-673d562ade54" + }, + "outputs": [], + "source": [ + "report = Report(metrics=[\n", + " generate_column_metrics(ColumnQuantileMetric, parameters={'quantile':0.25}, columns=['AveRooms', 'AveBedrms']),\n", + "])\n", + "\n", + "report.run(reference_data=reference, current_data=current)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "98574312", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "1JUEabeT61mW16eviYucIPleIfXVoYdc-" + }, + "id": "fc3f8a8f", + "outputId": "cc932a1c-8fe0-4ac0-a875-cc25aaa45ac2", + "scrolled": true + }, + "outputs": [], + "source": [ + "report = Report(metrics=[\n", + " ColumnSummaryMetric(column_name='AveRooms'),\n", + " generate_column_metrics(ColumnQuantileMetric, parameters={'quantile':0.25}, columns='num'),\n", + " DataDriftPreset()\n", + "])\n", + "\n", + "report.run(reference_data=reference, current_data=current)\n", + "report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4f1e2f4", + "metadata": { + "colab": { + "background_save": true + }, + "id": "d72ca314", + "scrolled": false + }, + "outputs": [], + "source": [ + "report.as_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "911c5f08", + "metadata": { + "colab": { + "background_save": true + }, + "id": "ab320537" + }, + "outputs": [], + "source": [ + "report.json()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e9cc197e", + "metadata": { + "colab": { + "background_save": true + }, + "id": "fd4e9085" + }, + "outputs": [], + "source": [ + "#report.save_json('report.json')" + ] + }, + { + "cell_type": "markdown", + "id": "c30f75c6", + "metadata": { + "id": "1ea31ae7" + }, + "source": [ + "## Test Suite " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "296e2709", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "1AJKCnZwYYl2cLoRWtKeqF7iIt6Ruuqz-" + }, + "id": "f2311155", + "outputId": "825e12fb-8ed6-475e-b046-189225e4cb51" + }, + "outputs": [], + "source": [ + "tests = TestSuite(tests=[\n", + " TestNumberOfColumnsWithMissingValues(),\n", + " TestNumberOfRowsWithMissingValues(),\n", + " TestNumberOfConstantColumns(),\n", + " TestNumberOfDuplicatedRows(),\n", + " TestNumberOfDuplicatedColumns(),\n", + " TestColumnsType(),\n", + " TestNumberOfDriftedColumns(),\n", + "])\n", + "\n", + "tests.run(reference_data=reference, current_data=current)\n", + "tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53904378", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "1ohVLBp0ivJltJprlA1F-AVL7TcppIziI" + }, + "id": "d78779bb", + "outputId": "c000d73a-dee7-42e4-b02f-fd08ea97af4a", + "scrolled": false + }, + "outputs": [], + "source": [ + "suite = TestSuite(tests=[\n", + " NoTargetPerformanceTestPreset(),\n", + "])\n", + "\n", + "suite.run(reference_data=reference, current_data=current)\n", + "suite" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1b883b8", + "metadata": { + "colab": { + "background_save": true, + "output_embedded_package_id": "11IMmTb-z3p9uT6Hlhj9DzmCEq_C9nn-6" + }, + "id": "e92a0605", + "outputId": "4aafbda5-e521-44d5-bda1-fb5b65faff12" + }, + "outputs": [], + "source": [ + "suite = TestSuite(tests=[\n", + " TestColumnDrift('Population'),\n", + " TestMeanInNSigmas('HouseAge'),\n", + " NoTargetPerformanceTestPreset(columns=['AveRooms', 'AveBedrms', 'AveOccup'])\n", + "])\n", + "\n", + "suite.run(reference_data=reference, current_data=current)\n", + "suite" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29efc3ec", + "metadata": { + "id": "7df650c7" + }, + "outputs": [], + "source": [ + "suite = TestSuite(tests=[\n", + " TestNumberOfColumnsWithMissingValues(),\n", + " TestNumberOfRowsWithMissingValues(),\n", + " TestNumberOfConstantColumns(),\n", + " TestNumberOfDuplicatedRows(),\n", + " TestNumberOfDuplicatedColumns(),\n", + " TestColumnsType(),\n", + " TestNumberOfDriftedColumns(),\n", + " TestColumnDrift('Population'),\n", + " TestShareOfOutRangeValues('Population'),\n", + " DataStabilityTestPreset(),\n", + " RegressionTestPreset()\n", + " \n", + "])\n", + "\n", + "suite.run(reference_data=reference, current_data=current)\n", + "suite" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de0dd624", + "metadata": { + "colab": { + "background_save": true + }, + "id": "20da511b", + "scrolled": false + }, + "outputs": [], + "source": [ + "suite.as_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36e53b01", + "metadata": { + "colab": { + "background_save": true + }, + "id": "3c1c476f" + }, + "outputs": [], + "source": [ + "suite.json()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f7dd954", + "metadata": { + "id": "e4225064" + }, + "outputs": [], + "source": [ + "#suite.save_html('test_suite.html')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a7696c9", + "metadata": { + "id": "5fedd579" + }, + "outputs": [], + "source": [ + "#suite.save_json('test_suite.json')" + ] + } + ], + "metadata": { + "colab": { + "name": "", + "version": "" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/evidently/_version.py b/src/evidently/_version.py index d16c36976b..3457b9bba9 100644 --- a/src/evidently/_version.py +++ b/src/evidently/_version.py @@ -1,5 +1,5 @@ #!/usr/bin/env python # coding: utf-8 -version_info = (0, 2, 4) +version_info = (0, 2, 5) __version__ = ".".join(map(str, version_info))