Josh/aws updates (#788)

* update bedrock notebooks * update trulens version tags * update lite to express for better eval performance * update feedback def --------- Co-authored-by: Piotr Mardziel <[email protected]>
truera · Jan 10, 2024 · ba83563 · ba83563
1 parent 748400d
commit ba83563
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 28 deletions.
diff --git a/trulens_eval/examples/expositional/models/bedrock.ipynb b/trulens_eval/examples/expositional/models/bedrock.ipynb
@@ -35,7 +35,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# ! pip install trulens_eval==0.18.3 langchain==0.0.305 boto3==1.28.59"
+    "# ! pip install trulens_eval==0.20.2 langchain==0.0.305 boto3==1.28.59"
    ]
   },
   {
@@ -119,9 +119,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Initialize Feedback Function(s)\n",
-    "\n",
-    "TODO: feedback function using bedrock"
+    "### Initialize Feedback Function(s)"
    ]
   },
   {
@@ -145,7 +143,7 @@
     "bedrock = Bedrock(model_id = \"amazon.titan-tg1-large\", region_name=\"us-east-1\")\n",
     "\n",
     "# Define a language match feedback function using HuggingFace.\n",
-    "f_relevance = Feedback(bedrock.relevance).on_input_output()\n",
+    "f_qa_relevance = Feedback(bedrock.relevance_with_cot_reasons, name = \"Answer Relevance\").on_input_output()\n",
     "# By default this will check language match on the main app input and main app\n",
     "# output."
    ]
@@ -166,7 +164,7 @@
    "source": [
     "tru_recorder = TruChain(chain,\n",
     "    app_id='Chain1_ChatApplication',\n",
-    "    feedbacks=[f_relevance])"
+    "    feedbacks=[f_qa_relevance])"
    ]
   },
   {
@@ -224,13 +222,6 @@
    "source": [
     "tru.get_records_and_feedback(app_ids=[])[0] # pass an empty list of app_ids to get all"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -249,7 +240,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/trulens_eval/examples/expositional/models/bedrock_finetuning_experiments.ipynb b/trulens_eval/examples/expositional/models/bedrock_finetuning_experiments.ipynb
@@ -55,7 +55,7 @@
    },
    "outputs": [],
    "source": [
-    "# ! pip install trulens_eval==0.18.3 sagemaker datasets boto3 "
+    "# ! pip install trulens_eval==0.20.2 sagemaker datasets boto3 "
    ]
   },
   {
@@ -433,10 +433,10 @@
     "    pretrained_response = pretrained_predictor.predict(\n",
     "        payload, custom_attributes=\"accept_eula=true\"\n",
     "    )\n",
-    "    responses_before_finetuning.append(pretrained_response[0][\"generation\"])\n",
+    "    responses_before_finetuning.append(pretrained_response[0][\"generated_text\"])\n",
     "    # Please change the following line to \"accept_eula=True\"\n",
     "    finetuned_response = finetuned_predictor.predict(payload, custom_attributes=\"accept_eula=true\")\n",
-    "    responses_after_finetuning.append(finetuned_response[0][\"generation\"])\n",
+    "    responses_after_finetuning.append(finetuned_response[0][\"generated_text\"])\n",
     "\n",
     "\n",
     "try:\n",
@@ -486,7 +486,7 @@
     "    \n",
     "    return pretrained_predictor.predict(\n",
     "        payload, custom_attributes=\"accept_eula=true\"\n",
-    "    )[0][\"generation\"]"
+    "    )[0][\"generated_text\"]"
    ]
   },
   {
@@ -511,7 +511,7 @@
     "    \n",
     "    return finetuned_predictor.predict(\n",
     "        payload, custom_attributes=\"accept_eula=true\"\n",
-    "    )[0][\"generation\"]"
+    "    )[0][\"generated_text\"]"
    ]
   },
   {
@@ -592,7 +592,7 @@
     "from trulens_eval import Bedrock\n",
     "\n",
     "# Initialize Bedrock as feedback function provider\n",
-    "bedrock = Bedrock(model_id = \"amazon.titan-tg1-large\", region_name=\"us-east-1\")\n",
+    "bedrock = Bedrock(model_id = \"amazon.titan-text-express-v1\", region_name=\"us-east-1\")\n",
     "\n",
     "# Create a Feedback object for ground truth similarity\n",
     "ground_truth = GroundTruthAgreement(golden_set, provider = bedrock)\n",
@@ -1286,9 +1286,9 @@
   ],
   "instance_type": "ml.t3.medium",
   "kernelspec": {
-   "display_name": "Python 3 (Data Science 3.0)",
+   "display_name": "vivek_aws",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-310-v1"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -1300,7 +1300,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.6"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

diff --git a/trulens_eval/trulens_eval/feedback/provider/bedrock.py b/trulens_eval/trulens_eval/feedback/provider/bedrock.py
@@ -19,21 +19,19 @@ class Bedrock(LLMProvider):
     endpoint: BedrockEndpoint
 
     def __init__(
-        # rajib76: changed the default model id to the cheaper model
-        # titan-tg1-large is no longer available
         self,
         *args,
-        model_id: str = "amazon.titan-text-lite-v1",
+        model_id: str = "amazon.titan-text-express-v1",
         **kwargs
-        # self, *args, model_id: str = "amazon.titan-tg1-large", **kwargs
+        # self, *args, model_id: str = "amazon.titan-text-express-v1", **kwargs
     ):
         """
         A set of AWS Feedback Functions.
 
         Parameters:
 
         - model_id (str, optional): The specific model id. Defaults to
-          "amazon.titan-tg1-large".
+          "amazon.titan-text-express-v1".
 
         - All other args/kwargs passed to BedrockEndpoint and subsequently
           to boto3 client constructor.