Skip to content

Commit

Permalink
reflexion nb done
Browse files Browse the repository at this point in the history
  • Loading branch information
@alckasoc
alckasoc committed Jun 23, 2024
1 parent ceb41b2 commit 49f57dc
Showing 1 changed file with 246 additions and 3 deletions.
249 changes: 246 additions & 3 deletions notebooks/reflexion.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@
" HOTPOTQA_FEWSHOT_EXAMPLES_COT,\n",
" HOTPOTQA_FEWSHOT_EXAMPLES_REACT,\n",
")\n",
"from agential.cog.prompts.benchmark.fever import (\n",
" FEVER_FEWSHOT_EXAMPLES_COT,\n",
" FEVER_FEWSHOT_EXAMPLES_REACT,\n",
")\n",
"from agential.cog.prompts.benchmark.triviaqa import (\n",
" TRIVIAQA_FEWSHOT_EXAMPLES_COT,\n",
" TRIVIAQA_FEWSHOT_EXAMPLES_REACT,\n",
")\n",
"from agential.cog.prompts.benchmark.ambignq import (\n",
" AMBIGNQ_FEWSHOT_EXAMPLES_COT,\n",
" AMBIGNQ_FEWSHOT_EXAMPLES_REACT,\n",
")\n",
"\n",
"\n",
"import tiktoken\n",
"from langchain_community.docstore.wikipedia import Wikipedia\n",
Expand Down Expand Up @@ -168,7 +181,85 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"question = \"Nikolaj Coster-Waldau worked with the Fox Broadcasting Company.\"\n",
"key = \"REFUTES\"\n",
"\n",
"\n",
"agent = ReflexionCoTAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"fever\"},\n",
" max_reflections=3,\n",
" max_trials=1,\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=FEVER_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_FEVER, \n",
" reflect_examples=FEVER_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_FEVER,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"question = \"Nikolaj Coster-Waldau worked with the Fox Broadcasting Company.\"\n",
"key = \"REFUTES\"\n",
"\n",
"\n",
"agent = ReflexionReActAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"fever\"},\n",
" max_steps=6,\n",
" max_tokens=3896,\n",
" docstore=DocstoreExplorer(Wikipedia()),\n",
" enc=tiktoken.encoding_for_model(\"gpt-3.5-turbo\"),\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=FEVER_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_FEVER, \n",
" reflect_examples=FEVER_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_FEVER,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_steps=6,\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "markdown",
Expand All @@ -182,7 +273,83 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"question = \"When did the simpsons first air on television?\"\n",
"key = \"1989\"\n",
"\n",
"agent = ReflexionCoTAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"ambignq\"},\n",
" max_reflections=3,\n",
" max_trials=1,\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=AMBIGNQ_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_AMBIGNQ, \n",
" reflect_examples=AMBIGNQ_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_AMBIGNQ,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"question = \"When did the simpsons first air on television?\"\n",
"key = \"1989\"\n",
"\n",
"agent = ReflexionReActAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"ambignq\"},\n",
" max_steps=6,\n",
" max_tokens=3896,\n",
" docstore=DocstoreExplorer(Wikipedia()),\n",
" enc=tiktoken.encoding_for_model(\"gpt-3.5-turbo\"),\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=AMBIGNQ_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_AMBIGNQ, \n",
" reflect_examples=AMBIGNQ_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_AMBIGNQ,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_steps=6,\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "markdown",
Expand All @@ -196,7 +363,83 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"question = \"Which American-born Sinclair won the Nobel Prize for Literature in 1930?\"\n",
"key = \"Sinclair Lewis\"\n",
"\n",
"agent = ReflexionCoTAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"triviaqa\"},\n",
" max_reflections=3,\n",
" max_trials=1,\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=TRIVIAQA_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_TRIVIAQA, \n",
" reflect_examples=TRIVIAQA_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_TRIVIAQA,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"question = \"Which American-born Sinclair won the Nobel Prize for Literature in 1930?\"\n",
"key = \"Sinclair Lewis\"\n",
"\n",
"agent = ReflexionReActAgent(\n",
" llm=llm,\n",
" mode={\"qa\": \"triviaqa\"},\n",
" max_steps=6,\n",
" max_tokens=3896,\n",
" docstore=DocstoreExplorer(Wikipedia()),\n",
" enc=tiktoken.encoding_for_model(\"gpt-3.5-turbo\"),\n",
")\n",
"out = agent.generate(\n",
" question=question, \n",
" key=key, \n",
" examples=TRIVIAQA_FEWSHOT_EXAMPLES_COT,\n",
" prompt=REFLEXION_COT_INSTRUCTION_TRIVIAQA, \n",
" reflect_examples=TRIVIAQA_FEWSHOT_EXAMPLES_REFLEXION_COT_REFLECT,\n",
" reflect_prompt=REFLEXION_COT_REFLECT_INSTRUCTION_TRIVIAQA,\n",
" reflect_strategy=\"reflexion\",\n",
" additional_keys={},\n",
" reflect_additional_keys={},\n",
" max_steps=6,\n",
" max_trials=3,\n",
" patience=3,\n",
" reset=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out"
]
},
{
"cell_type": "markdown",
Expand Down

0 comments on commit 49f57dc

Please sign in to comment.