dspy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llm import LLMModel\n",
    "\n",
    "model = LLMModel(\n",
    "    provider=\"openai\",\n",
    "    model_name=\"gpt-4o-2024-08-06\",\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "model.invoke(\"\"\"\n",
    "\n",
    "\"\"\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:\t*** In DSPy 2.5, all LM clients except `dspy.LM` are deprecated, underperform, and are about to be deleted. ***\n",
      " \t\tYou are using the client GPT3, which will be removed in DSPy 2.6.\n",
      " \t\tChanging the client is straightforward and will let you use new features (Adapters) that improve the consistency of LM outputs, especially when using chat LMs. \n",
      "\n",
      " \t\tLearn more about the changes and how to migrate at\n",
      " \t\thttps://github.com/stanfordnlp/dspy/blob/main/examples/migration.ipynb\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated Summary: The code defines a function in Python that calculates the nth Fibonacci number using a loop.\n",
      "\n",
      "Evaluation Scores:\n",
      "Accuracy: /10\n",
      "Completeness: 9\n",
      "\n",
      "Completeness Score: 8/10\n",
      "Conciseness: 7/10\n",
      "\n",
      "Rationale: Accuracy Score: 9\n",
      "The summary accurately describes the purpose of the code and how it calculates the nth Fibonacci number using a loop. However, it could be improved by mentioning that the function returns the Fibonacci number.\n",
      "\n",
      "Completeness Score: 8\n",
      "The summary provides a good overview of what the code does, but it could be more detailed by explaining the logic behind the Fibonacci sequence and how the code achieves the calculation.\n",
      "\n",
      "Conciseness Score: 7\n",
      "The summary is concise and to the point, but it could be improved by including more specific details about the code implementation and how it works.\n"
     ]
    }
   ],
   "source": [
    "import dspy\n",
    "from typing import Dict, Any, List\n",
    "from dataclasses import dataclass\n",
    "from datetime import datetime\n",
    "import json\n",
    "import os\n",
    "import pathlib\n",
    "from rich.console import Console\n",
    "from rich.table import Table\n",
    "\n",
    "# Configure LLM\n",
    "lm = dspy.OpenAI(model='gpt-4-turbo-preview', max_tokens=1000)\n",
    "dspy.settings.configure(lm=lm)\n",
    "\n",
    "# Set up logging directory\n",
    "LOG_DIR = pathlib.Path(\"summarizer_logs\")\n",
    "PROMPT_DIR = LOG_DIR / \"prompts\"\n",
    "os.makedirs(LOG_DIR, exist_ok=True)\n",
    "os.makedirs(PROMPT_DIR, exist_ok=True)\n",
    "\n",
    "console = Console()\n",
    "\n",
    "# [Previous code remains the same until train_summarizer function]\n",
    "\n",
    "def save_prompt(prompt_data: Dict, trial_num: int, predictor_num: int):\n",
    "    \"\"\"Save individual prompt data to file.\"\"\"\n",
    "    timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "    filename = PROMPT_DIR / f\"trial_{trial_num}_predictor_{predictor_num}_{timestamp}.json\"\n",
    "    \n",
    "    with open(filename, 'w') as f:\n",
    "        json.dump(prompt_data, f, indent=2)\n",
    "\n",
    "def train_summarizer(trainset, valset=None):\n",
    "    \"\"\"Train the summarizer using MIPRO optimization with detailed logging.\"\"\"\n",
    "    \n",
    "    def metric(example, pred, trace=None):\n",
    "        \"\"\"Metric function for evaluating summaries.\"\"\"\n",
    "        if not pred.summary or not pred.evaluation:\n",
    "            return 0\n",
    "        return pred.evaluation.overall\n",
    "\n",
    "    # Initialize base model\n",
    "    program = CodeSummarizer(max_attempts=2, num_candidates=2)\n",
    "    \n",
    "    # Create logging table\n",
    "    progress_table = Table(title=\"Training Progress\")\n",
    "    progress_table.add_column(\"Trial\")\n",
    "    progress_table.add_column(\"Score\")\n",
    "    progress_table.add_column(\"Status\")\n",
    "    \n",
    "    # Configure optimizer with logging\n",
    "    teleprompter = dspy.teleprompt.MIPROv2(\n",
    "        metric=metric,\n",
    "        num_candidates=5,\n",
    "        init_temperature=0.5,\n",
    "        verbose=True\n",
    "    )\n",
    "    \n",
    "    # Save initial configuration\n",
    "    config_log = {\n",
    "        \"timestamp\": datetime.now().isoformat(),\n",
    "        \"max_attempts\": program.max_attempts,\n",
    "        \"num_candidates\": program.num_candidates,\n",
    "        \"mipro_candidates\": 5,\n",
    "        \"temperature\": 0.5,\n",
    "        \"num_batches\": 20\n",
    "    }\n",
    "    \n",
    "    with open(LOG_DIR / \"config.json\", 'w') as f:\n",
    "        json.dump(config_log, f, indent=2)\n",
    "    \n",
    "    best_score = 0\n",
    "    best_prompts = None\n",
    "    \n",
    "    def log_trial(trial_num, trial_data):\n",
    "        nonlocal best_score, best_prompts\n",
    "        \n",
    "        score = trial_data['score']\n",
    "        is_pruned = trial_data.get('pruned', False)\n",
    "        status = \"PRUNED\" if is_pruned else \"COMPLETED\"\n",
    "        \n",
    "        progress_table.add_row(\n",
    "            str(trial_num),\n",
    "            f\"{score:.3f}\",\n",
    "            status\n",
    "        )\n",
    "        \n",
    "        # Save detailed trial data\n",
    "        trial_log = {\n",
    "            \"trial_num\": trial_num,\n",
    "            \"timestamp\": datetime.now().isoformat(),\n",
    "            \"score\": score,\n",
    "            \"status\": status,\n",
    "            \"predictors\": []\n",
    "        }\n",
    "        \n",
    "        # Log each predictor's prompts\n",
    "        if not is_pruned:\n",
    "            for i, predictor in enumerate(trial_data['program'].predictors()):\n",
    "                predictor_data = {\n",
    "                    \"predictor_num\": i,\n",
    "                    \"type\": predictor.__class__.__name__\n",
    "                }\n",
    "                \n",
    "                if hasattr(predictor, 'extended_signature'):\n",
    "                    predictor_data[\"instructions\"] = predictor.extended_signature.instructions\n",
    "                elif hasattr(predictor, 'signature'):\n",
    "                    predictor_data[\"instructions\"] = predictor.signature.instructions\n",
    "                \n",
    "                trial_log[\"predictors\"].append(predictor_data)\n",
    "                save_prompt(predictor_data, trial_num, i)\n",
    "        \n",
    "        # Save trial log\n",
    "        with open(LOG_DIR / f\"trial_{trial_num}.json\", 'w') as f:\n",
    "            json.dump(trial_log, f, indent=2)\n",
    "        \n",
    "        # Update best score and prompts\n",
    "        if score > best_score and not is_pruned:\n",
    "            best_score = score\n",
    "            best_prompts = trial_log\n",
    "            with open(LOG_DIR / \"best_prompts.json\", 'w') as f:\n",
    "                json.dump(best_prompts, f, indent=2)\n",
    "    \n",
    "    # Compile with progress tracking\n",
    "    optimized_program = teleprompter.compile(\n",
    "        program,\n",
    "        trainset=trainset,\n",
    "        valset=valset,\n",
    "        max_bootstrapped_demos=2,\n",
    "        max_labeled_demos=2,\n",
    "        num_batches=20\n",
    "    )\n",
    "    \n",
    "    # Log all trials\n",
    "    for trial_num, trial_data in optimized_program.trial_logs.items():\n",
    "        log_trial(trial_num, trial_data)\n",
    "    \n",
    "    # Display final progress table\n",
    "    console.print(progress_table)\n",
    "    \n",
    "    # Save final summary\n",
    "    summary = {\n",
    "        \"total_trials\": len(optimized_program.trial_logs),\n",
    "        \"best_score\": best_score,\n",
    "        \"final_score\": list(optimized_program.trial_logs.values())[-1]['score'],\n",
    "        \"timestamp\": datetime.now().isoformat()\n",
    "    }\n",
    "    \n",
    "    with open(LOG_DIR / \"training_summary.json\", 'w') as f:\n",
    "        json.dump(summary, f, indent=2)\n",
    "    \n",
    "    return optimized_program\n",
    "\n",
    "def main():\n",
    "    # [Previous main() code remains the same until training]\n",
    "    \n",
    "    # Train and test optimized model with logging\n",
    "    print(\"\\nTraining optimized model...\")\n",
    "    console.print(\"[bold blue]Starting training with detailed logging...[/bold blue]\")\n",
    "    console.print(f\"Logs will be saved to: {LOG_DIR}\")\n",
    "    \n",
    "    optimized_summarizer = train_summarizer(trainset)\n",
    "    \n",
    "    # Load and display best prompts\n",
    "    with open(LOG_DIR / \"best_prompts.json\", 'r') as f:\n",
    "        best_prompts = json.load(f)\n",
    "    \n",
    "    console.print(\"\\n[bold green]Best Performing Prompts:[/bold green]\")\n",
    "    for predictor in best_prompts[\"predictors\"]:\n",
    "        console.print(f\"\\nPredictor {predictor['predictor_num']}:\")\n",
    "        console.print(predictor['instructions'])\n",
    "    \n",
    "    # [Rest of the main() function remains the same]\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}