diff --git a/examples/DVCLive-Evidently.ipynb b/examples/DVCLive-Evidently.ipynb
new file mode 100644
index 00000000..171bbd58
--- /dev/null
+++ b/examples/DVCLive-Evidently.ipynb
@@ -0,0 +1,1715 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": [],
+ "toc_visible": true,
+ "authorship_tag": "ABX9TyNJAdha/v4n9zLqIfGakg0E"
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Install Evidently and DVC with DVCLive"
+ ],
+ "metadata": {
+ "id": "WpfOFaqHcnAt"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!pip uninstall -q -y sqlalchemy pyarrow ipython-sql pandas-gbq"
+ ],
+ "metadata": {
+ "id": "BqWpagFPZ45W",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468096427,
+ "user_tz": -120,
+ "elapsed": 2337,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 1,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "DijzqeokW595",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468130037,
+ "user_tz": -120,
+ "elapsed": 33615,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%%capture\n",
+ "!pip install -q dvc==3.25.0 dvclive==3.0.1 evidently==0.4.5 pandas==1.5.3"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Load the data"
+ ],
+ "metadata": {
+ "id": "ZyZ2sX8GcvMU"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!mkdir raw_data && \\\n",
+ " cd raw_data && \\\n",
+ " wget https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip && \\\n",
+ " unzip bike+sharing+dataset.zip"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ZUrB0D59XMDD",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468131788,
+ "user_tz": -120,
+ "elapsed": 1772,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "9f6f5a3c-f856-4d56-a8fb-ec4483ec6127"
+ },
+ "execution_count": 3,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "--2023-10-16 14:55:29-- https://archive.ics.uci.edu/static/public/275/bike+sharing+dataset.zip\n",
+ "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
+ "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
+ "HTTP request sent, awaiting response... 200 OK\n",
+ "Length: unspecified\n",
+ "Saving to: ‘bike+sharing+dataset.zip’\n",
+ "\n",
+ "bike+sharing+datase [ <=> ] 273.43K 443KB/s in 0.6s \n",
+ "\n",
+ "2023-10-16 14:55:30 (443 KB/s) - ‘bike+sharing+dataset.zip’ saved [279992]\n",
+ "\n",
+ "Archive: bike+sharing+dataset.zip\n",
+ " inflating: Readme.txt \n",
+ " inflating: day.csv \n",
+ " inflating: hour.csv \n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import pandas as pd"
+ ],
+ "metadata": {
+ "id": "P3XXcUrQY1EQ",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468132141,
+ "user_tz": -120,
+ "elapsed": 357,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 4,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "df = pd.read_csv(\"raw_data/day.csv\", header=0, sep=',', parse_dates=['dteday'])\n",
+ "df.head()"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 206
+ },
+ "id": "MDK0xkdbYCWg",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468132141,
+ "user_tz": -120,
+ "elapsed": 9,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "ec8d2605-144d-45ff-b442-70ba858a44a3"
+ },
+ "execution_count": 5,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ " instant dteday season yr mnth holiday weekday workingday \\\n",
+ "0 1 2011-01-01 1 0 1 0 6 0 \n",
+ "1 2 2011-01-02 1 0 1 0 0 0 \n",
+ "2 3 2011-01-03 1 0 1 0 1 1 \n",
+ "3 4 2011-01-04 1 0 1 0 2 1 \n",
+ "4 5 2011-01-05 1 0 1 0 3 1 \n",
+ "\n",
+ " weathersit temp atemp hum windspeed casual registered \\\n",
+ "0 2 0.344167 0.363625 0.805833 0.160446 331 654 \n",
+ "1 2 0.363478 0.353739 0.696087 0.248539 131 670 \n",
+ "2 1 0.196364 0.189405 0.437273 0.248309 120 1229 \n",
+ "3 1 0.200000 0.212122 0.590435 0.160296 108 1454 \n",
+ "4 1 0.226957 0.229270 0.436957 0.186900 82 1518 \n",
+ "\n",
+ " cnt \n",
+ "0 985 \n",
+ "1 801 \n",
+ "2 1349 \n",
+ "3 1562 \n",
+ "4 1600 "
+ ],
+ "text/html": [
+ "\n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " instant | \n",
+ " dteday | \n",
+ " season | \n",
+ " yr | \n",
+ " mnth | \n",
+ " holiday | \n",
+ " weekday | \n",
+ " workingday | \n",
+ " weathersit | \n",
+ " temp | \n",
+ " atemp | \n",
+ " hum | \n",
+ " windspeed | \n",
+ " casual | \n",
+ " registered | \n",
+ " cnt | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 2011-01-01 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 6 | \n",
+ " 0 | \n",
+ " 2 | \n",
+ " 0.344167 | \n",
+ " 0.363625 | \n",
+ " 0.805833 | \n",
+ " 0.160446 | \n",
+ " 331 | \n",
+ " 654 | \n",
+ " 985 | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " 2 | \n",
+ " 2011-01-02 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 2 | \n",
+ " 0.363478 | \n",
+ " 0.353739 | \n",
+ " 0.696087 | \n",
+ " 0.248539 | \n",
+ " 131 | \n",
+ " 670 | \n",
+ " 801 | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " 3 | \n",
+ " 2011-01-03 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 0.196364 | \n",
+ " 0.189405 | \n",
+ " 0.437273 | \n",
+ " 0.248309 | \n",
+ " 120 | \n",
+ " 1229 | \n",
+ " 1349 | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " 4 | \n",
+ " 2011-01-04 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 2 | \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 0.200000 | \n",
+ " 0.212122 | \n",
+ " 0.590435 | \n",
+ " 0.160296 | \n",
+ " 108 | \n",
+ " 1454 | \n",
+ " 1562 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " 5 | \n",
+ " 2011-01-05 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 3 | \n",
+ " 1 | \n",
+ " 1 | \n",
+ " 0.226957 | \n",
+ " 0.229270 | \n",
+ " 0.436957 | \n",
+ " 0.186900 | \n",
+ " 82 | \n",
+ " 1518 | \n",
+ " 1600 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 5
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Define column mapping"
+ ],
+ "metadata": {
+ "id": "4a9DrmjyhhEP"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from evidently.pipeline.column_mapping import ColumnMapping"
+ ],
+ "metadata": {
+ "id": "_bkEZuM8gELe",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468132141,
+ "user_tz": -120,
+ "elapsed": 5,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 6,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "data_columns = ColumnMapping()\n",
+ "data_columns.numerical_features = ['weathersit', 'temp', 'atemp', 'hum', 'windspeed']\n",
+ "data_columns.categorical_features = ['holiday', 'workingday']"
+ ],
+ "metadata": {
+ "id": "dLIZqkHAgEuo",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468132141,
+ "user_tz": -120,
+ "elapsed": 5,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 7,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Define what to log"
+ ],
+ "metadata": {
+ "id": "yNBKbk51hpyz"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from evidently.report import Report\n",
+ "from evidently.metric_preset import DataDriftPreset"
+ ],
+ "metadata": {
+ "id": "owblpS3Ahw0o",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468136565,
+ "user_tz": -120,
+ "elapsed": 4428,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 8,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "def eval_drift(reference, production, column_mapping):\n",
+ " data_drift_report = Report(metrics=[DataDriftPreset()])\n",
+ " data_drift_report.run(\n",
+ " reference_data=reference, current_data=production, column_mapping=column_mapping\n",
+ " )\n",
+ " report = data_drift_report.as_dict()\n",
+ "\n",
+ " drifts = []\n",
+ "\n",
+ " for feature in (\n",
+ " column_mapping.numerical_features + column_mapping.categorical_features\n",
+ " ):\n",
+ " drifts.append(\n",
+ " (\n",
+ " feature,\n",
+ " report[\"metrics\"][1][\"result\"][\"drift_by_columns\"][feature][\n",
+ " \"drift_score\"\n",
+ " ],\n",
+ " )\n",
+ " )\n",
+ "\n",
+ " return drifts\n"
+ ],
+ "metadata": {
+ "id": "vRF8PjiYho6z",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468136565,
+ "user_tz": -120,
+ "elapsed": 3,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 9,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Define the comparison windows"
+ ],
+ "metadata": {
+ "id": "4Yhet51mh6Xz"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "#set reference dates\n",
+ "reference_dates = ('2011-01-01 00:00:00','2011-01-28 23:00:00')\n",
+ "\n",
+ "#set experiment batches dates\n",
+ "experiment_batches = [\n",
+ " ('2011-01-01 00:00:00','2011-01-29 23:00:00'),\n",
+ " ('2011-01-29 00:00:00','2011-02-07 23:00:00'),\n",
+ " ('2011-02-07 00:00:00','2011-02-14 23:00:00'),\n",
+ " ('2011-02-15 00:00:00','2011-02-21 23:00:00'),\n",
+ "]"
+ ],
+ "metadata": {
+ "id": "nTq8xUbGh3Ux",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468136565,
+ "user_tz": -120,
+ "elapsed": 3,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 10,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "# Run and log experiments with DVCLive"
+ ],
+ "metadata": {
+ "id": "8lNq9OdniDss"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!git config --global user.email \"you@example.com\"\n",
+ "!git config --global user.name \"Your Name\""
+ ],
+ "metadata": {
+ "id": "zUt5jrVSRIqD",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468136565,
+ "user_tz": -120,
+ "elapsed": 3,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 11,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from dvclive import Live"
+ ],
+ "metadata": {
+ "id": "5Hx1jI9PnT3C",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468137794,
+ "user_tz": -120,
+ "elapsed": 1231,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 12,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "There are two ways to use DVC, put all the drift evaluation steps in one single experiment (corresponding to a git commit), or to save each step as a separate experiment (git commit)"
+ ],
+ "metadata": {
+ "id": "jTsrtISaSF7D"
+ }
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## In one experiment"
+ ],
+ "metadata": {
+ "id": "RGrEbbla30jr"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Setup a git repo with dvc\n",
+ "\n",
+ "%cd /content\n",
+ "!rm -rf experiments && mkdir experiments\n",
+ "%cd experiments\n",
+ "\n",
+ "!git init\n",
+ "!git add .gitignore\n",
+ "!git commit -m \"Init repo\"\n",
+ "!dvc init\n",
+ "!git commit -m \"Init DVC\""
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "ijUf_HhRobl0",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468140631,
+ "user_tz": -120,
+ "elapsed": 2844,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "796d7eec-17dc-40b2-a4c9-5bdcf9184c58"
+ },
+ "execution_count": 13,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "/content\n",
+ "/content/experiments\n",
+ "hint: Using 'master' as the name for the initial branch. This default branch name\n",
+ "hint: is subject to change. To configure the initial branch name to use in all\n",
+ "hint: of your new repositories, which will suppress this warning, call:\n",
+ "hint: \n",
+ "hint: \tgit config --global init.defaultBranch \n",
+ "hint: \n",
+ "hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and\n",
+ "hint: 'development'. The just-created branch can be renamed via this command:\n",
+ "hint: \n",
+ "hint: \tgit branch -m \n",
+ "Initialized empty Git repository in /content/experiments/.git/\n",
+ "fatal: pathspec '.gitignore' did not match any files\n",
+ "On branch master\n",
+ "\n",
+ "Initial commit\n",
+ "\n",
+ "nothing to commit (create/copy files and use \"git add\" to track)\n",
+ "Initialized DVC repository.\n",
+ "\n",
+ "You can now commit the changes to git.\n",
+ "\n",
+ "+---------------------------------------------------------------------+\n",
+ "| |\n",
+ "| DVC has enabled anonymous aggregate usage analytics. |\n",
+ "| Read the analytics documentation (and how to opt-out) here: |\n",
+ "| |\n",
+ "| |\n",
+ "+---------------------------------------------------------------------+\n",
+ "\n",
+ "What's next?\n",
+ "------------\n",
+ "- Check out the documentation: \n",
+ "- Get help and share ideas: \n",
+ "- Star us on GitHub: \n",
+ "[master (root-commit) 9220260] Init DVC\n",
+ " 3 files changed, 6 insertions(+)\n",
+ " create mode 100644 .dvc/.gitignore\n",
+ " create mode 100644 .dvc/config\n",
+ " create mode 100644 .dvcignore\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "with Live(report=\"notebook\") as live:\n",
+ " for date in experiment_batches:\n",
+ " live.log_param(\"begin\", date[0])\n",
+ " live.log_param(\"end\", date[1])\n",
+ "\n",
+ " metrics = eval_drift(\n",
+ " df.loc[df.dteday.between(reference_dates[0], reference_dates[1])],\n",
+ " df.loc[df.dteday.between(date[0], date[1])],\n",
+ " column_mapping=data_columns,\n",
+ " )\n",
+ "\n",
+ " for feature in metrics:\n",
+ " live.log_metric(feature[0], round(feature[1], 3))\n",
+ "\n",
+ " live.next_step()"
+ ],
+ "metadata": {
+ "id": "_h-jGJqPiA30",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468156663,
+ "user_tz": -120,
+ "elapsed": 16055,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 1000
+ },
+ "outputId": "0b949e24-8c53-4765-a8ee-64d002b3801e"
+ },
+ "execution_count": 14,
+ "outputs": [
+ {
+ "output_type": "display_data",
+ "data": {
+ "text/plain": [
+ ""
+ ],
+ "text/markdown": "# DVC Report\n\nparams.yaml\n\n| begin | end |\n|---------------------|---------------------|\n| 2011-02-15 00:00:00 | 2011-02-21 23:00:00 |\n\nmetrics.json\n\n| weathersit | temp | atemp | hum | windspeed | holiday | workingday | step |\n|--------------|--------|---------|-------|-------------|-----------|--------------|--------|\n| 0.231 | 0 | 0 | 0.062 | 0.012 | 0.275 | 0.593 | 3 |\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n"
+ },
+ "metadata": {}
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "To explore the results from CLI:"
+ ],
+ "metadata": {
+ "id": "Pc3jDX1q-y3c"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!dvc plots show"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "6OAsURiL-Ge2",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468158085,
+ "user_tz": -120,
+ "elapsed": 1434,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "0fb47be1-f524-41f1-8c74-d4663d721290"
+ },
+ "execution_count": 15,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "\rReading plot's data from workspace: 0% 0/7 [00:00, ?files/s]\rReading plot's data from workspace: 0% 0/7 [00:00, ?files/s{'info': ''}]\r \rfile:///content/experiments/dvc_plots/index.html\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import IPython\n",
+ "IPython.display.HTML(filename='dvc_plots/index.html')"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 802
+ },
+ "id": "pwdjxeEG-I49",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468158085,
+ "user_tz": -120,
+ "elapsed": 4,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "e01c74f9-45b6-4715-b1bd-2dd20875a421"
+ },
+ "execution_count": 16,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ ""
+ ],
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ " \n",
+ " DVC Plot\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ "\n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "\n",
+ ""
+ ]
+ },
+ "metadata": {},
+ "execution_count": 16
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## In multiple experiments (one per step)"
+ ],
+ "metadata": {
+ "id": "CCdF_ipAIY7k"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Setup a git repo with dvc\n",
+ "\n",
+ "%cd /content\n",
+ "!rm -rf experiments && mkdir experiments\n",
+ "%cd experiments\n",
+ "\n",
+ "!git init\n",
+ "!git add .gitignore\n",
+ "!git commit -m \"Init repo\"\n",
+ "!dvc init\n",
+ "!git commit -m \"Init DVC\""
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "0x81BAI--2Gm",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468159295,
+ "user_tz": -120,
+ "elapsed": 1213,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "7fb22cea-d367-41b0-f27d-a99e9d6081dc"
+ },
+ "execution_count": 17,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "/content\n",
+ "/content/experiments\n",
+ "hint: Using 'master' as the name for the initial branch. This default branch name\n",
+ "hint: is subject to change. To configure the initial branch name to use in all\n",
+ "hint: of your new repositories, which will suppress this warning, call:\n",
+ "hint: \n",
+ "hint: \tgit config --global init.defaultBranch \n",
+ "hint: \n",
+ "hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and\n",
+ "hint: 'development'. The just-created branch can be renamed via this command:\n",
+ "hint: \n",
+ "hint: \tgit branch -m \n",
+ "Initialized empty Git repository in /content/experiments/.git/\n",
+ "fatal: pathspec '.gitignore' did not match any files\n",
+ "On branch master\n",
+ "\n",
+ "Initial commit\n",
+ "\n",
+ "nothing to commit (create/copy files and use \"git add\" to track)\n",
+ "Initialized DVC repository.\n",
+ "\n",
+ "You can now commit the changes to git.\n",
+ "\n",
+ "+---------------------------------------------------------------------+\n",
+ "| |\n",
+ "| DVC has enabled anonymous aggregate usage analytics. |\n",
+ "| Read the analytics documentation (and how to opt-out) here: |\n",
+ "| |\n",
+ "| |\n",
+ "+---------------------------------------------------------------------+\n",
+ "\n",
+ "What's next?\n",
+ "------------\n",
+ "- Check out the documentation: \n",
+ "- Get help and share ideas: \n",
+ "- Star us on GitHub: \n",
+ "[master (root-commit) 469083d] Init DVC\n",
+ " 3 files changed, 6 insertions(+)\n",
+ " create mode 100644 .dvc/.gitignore\n",
+ " create mode 100644 .dvc/config\n",
+ " create mode 100644 .dvcignore\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "from dvclive import Live\n",
+ "\n",
+ "for step, date in enumerate(experiment_batches):\n",
+ " with Live() as live:\n",
+ " live.log_param(\"step\", step)\n",
+ " live.log_param(\"begin\", date[0])\n",
+ " live.log_param(\"end\", date[1])\n",
+ "\n",
+ " metrics = eval_drift(\n",
+ " df.loc[df.dteday.between(reference_dates[0], reference_dates[1])],\n",
+ " df.loc[df.dteday.between(date[0], date[1])],\n",
+ " column_mapping=data_columns,\n",
+ " )\n",
+ "\n",
+ " for feature in metrics:\n",
+ " live.log_metric(feature[0], round(feature[1], 3))\n"
+ ],
+ "metadata": {
+ "id": "VfVLDwfD39qO",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468161649,
+ "user_tz": -120,
+ "elapsed": 2355,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 18,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import dvc.api\n",
+ "\n",
+ "pd.DataFrame(dvc.api.exp_show())"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 238
+ },
+ "id": "ijcN3PaZ6fM0",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468162078,
+ "user_tz": -120,
+ "elapsed": 433,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "outputId": "2d26f834-604f-4e28-8924-f5d97ae92596"
+ },
+ "execution_count": 19,
+ "outputs": [
+ {
+ "output_type": "execute_result",
+ "data": {
+ "text/plain": [
+ " Experiment rev typ Created parent State Executor \\\n",
+ "0 None workspace baseline None None None None \n",
+ "1 None master baseline 02:55 PM None None None \n",
+ "2 elite-mobs e4d6acd branch_commit 02:56 PM None None None \n",
+ "3 buxom-shes 439f6e1 branch_commit 02:56 PM None None None \n",
+ "4 hammy-skip b5b80b5 branch_commit 02:55 PM None None None \n",
+ "5 girly-sere 2ba9568 branch_base 02:55 PM None None None \n",
+ "\n",
+ " weathersit temp atemp hum windspeed holiday workingday step \\\n",
+ "0 0.231 NaN NaN 0.062 0.012 0.275 0.593 3.0 \n",
+ "1 NaN NaN NaN NaN NaN NaN NaN NaN \n",
+ "2 0.231 NaN NaN 0.062 0.012 0.275 0.593 3.0 \n",
+ "3 0.155 0.399 0.537 0.684 0.611 0.588 0.699 2.0 \n",
+ "4 0.985 1.000 1.000 1.000 1.000 0.980 0.851 NaN \n",
+ "5 0.779 0.098 0.107 0.030 0.171 0.545 0.653 1.0 \n",
+ "\n",
+ " begin end \n",
+ "0 2011-02-15 00:00:00 2011-02-21 23:00:00 \n",
+ "1 None None \n",
+ "2 2011-02-15 00:00:00 2011-02-21 23:00:00 \n",
+ "3 2011-02-07 00:00:00 2011-02-14 23:00:00 \n",
+ "4 2011-01-01 00:00:00 2011-01-29 23:00:00 \n",
+ "5 2011-01-29 00:00:00 2011-02-07 23:00:00 "
+ ],
+ "text/html": [
+ "\n",
+ " \n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " Experiment | \n",
+ " rev | \n",
+ " typ | \n",
+ " Created | \n",
+ " parent | \n",
+ " State | \n",
+ " Executor | \n",
+ " weathersit | \n",
+ " temp | \n",
+ " atemp | \n",
+ " hum | \n",
+ " windspeed | \n",
+ " holiday | \n",
+ " workingday | \n",
+ " step | \n",
+ " begin | \n",
+ " end | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " None | \n",
+ " workspace | \n",
+ " baseline | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " 0.231 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 0.062 | \n",
+ " 0.012 | \n",
+ " 0.275 | \n",
+ " 0.593 | \n",
+ " 3.0 | \n",
+ " 2011-02-15 00:00:00 | \n",
+ " 2011-02-21 23:00:00 | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " None | \n",
+ " master | \n",
+ " baseline | \n",
+ " 02:55 PM | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " None | \n",
+ " None | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " elite-mobs | \n",
+ " e4d6acd | \n",
+ " branch_commit | \n",
+ " 02:56 PM | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " 0.231 | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 0.062 | \n",
+ " 0.012 | \n",
+ " 0.275 | \n",
+ " 0.593 | \n",
+ " 3.0 | \n",
+ " 2011-02-15 00:00:00 | \n",
+ " 2011-02-21 23:00:00 | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " buxom-shes | \n",
+ " 439f6e1 | \n",
+ " branch_commit | \n",
+ " 02:56 PM | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " 0.155 | \n",
+ " 0.399 | \n",
+ " 0.537 | \n",
+ " 0.684 | \n",
+ " 0.611 | \n",
+ " 0.588 | \n",
+ " 0.699 | \n",
+ " 2.0 | \n",
+ " 2011-02-07 00:00:00 | \n",
+ " 2011-02-14 23:00:00 | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " hammy-skip | \n",
+ " b5b80b5 | \n",
+ " branch_commit | \n",
+ " 02:55 PM | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " 0.985 | \n",
+ " 1.000 | \n",
+ " 1.000 | \n",
+ " 1.000 | \n",
+ " 1.000 | \n",
+ " 0.980 | \n",
+ " 0.851 | \n",
+ " NaN | \n",
+ " 2011-01-01 00:00:00 | \n",
+ " 2011-01-29 23:00:00 | \n",
+ "
\n",
+ " \n",
+ " 5 | \n",
+ " girly-sere | \n",
+ " 2ba9568 | \n",
+ " branch_base | \n",
+ " 02:55 PM | \n",
+ " None | \n",
+ " None | \n",
+ " None | \n",
+ " 0.779 | \n",
+ " 0.098 | \n",
+ " 0.107 | \n",
+ " 0.030 | \n",
+ " 0.171 | \n",
+ " 0.545 | \n",
+ " 0.653 | \n",
+ " 1.0 | \n",
+ " 2011-01-29 00:00:00 | \n",
+ " 2011-02-07 23:00:00 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n"
+ ]
+ },
+ "metadata": {},
+ "execution_count": 19
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "To explore the results from CLI:"
+ ],
+ "metadata": {
+ "id": "TQE5aBWl-sef"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "!dvc exp show"
+ ],
+ "metadata": {
+ "id": "oZtY-97bQj-Q",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468163295,
+ "user_tz": -120,
+ "elapsed": 1221,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ },
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "14eb8d4c-c9ce-4bb8-caba-42e46d45bb65"
+ },
+ "execution_count": 20,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ " ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── \n",
+ " Experiment Created weathersit temp atemp hum windspeed holiday workingday step begin end \n",
+ " ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── \n",
+ " workspace - 0.231 0 0 0.062 0.012 0.275 0.593 3 2011-02-15 00:00:00 2011-02-21 23:00:00 \n",
+ " master 02:55 PM - - - - - - - - - - \n",
+ " ├── e4d6acd [elite-mobs] 02:56 PM 0.231 0 0 0.062 0.012 0.275 0.593 3 2011-02-15 00:00:00 2011-02-21 23:00:00 \n",
+ " ├── 439f6e1 [buxom-shes] 02:56 PM 0.155 0.399 0.537 0.684 0.611 0.588 0.699 2 2011-02-07 00:00:00 2011-02-14 23:00:00 \n",
+ " ├── b5b80b5 [hammy-skip] 02:55 PM 0.985 1 1 1 1 0.98 0.851 0 2011-01-01 00:00:00 2011-01-29 23:00:00 \n",
+ " └── 2ba9568 [girly-sere] 02:55 PM 0.779 0.098 0.107 0.03 0.171 0.545 0.653 1 2011-01-29 00:00:00 2011-02-07 23:00:00 \n",
+ " ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── \n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [],
+ "metadata": {
+ "id": "QoYexufp-qw2",
+ "executionInfo": {
+ "status": "ok",
+ "timestamp": 1697468163757,
+ "user_tz": -120,
+ "elapsed": 464,
+ "user": {
+ "displayName": "Francesco Motoko",
+ "userId": "00974636158007469548"
+ }
+ }
+ },
+ "execution_count": 20,
+ "outputs": []
+ }
+ ]
+}