Modify the structure of ActionReturn's result (#102)

* modify the struction of action results

* fix docstrings

---------

Co-authored-by: wangzy <[email protected]>

lagent/actions/builtin_actions.py

@@ -102,7 +102,7 @@ def run(self, response: str) -> ActionReturn:
         action_return = ActionReturn(
             url=None,
             args=dict(text=response),
-            result=dict(text=response),
+            result=[dict(type='text', content=response)],
             type=self.name,
             valid=ActionValidCode.FINISH,
             state=ActionStatusCode.SUCCESS)

lagent/actions/parser.py

@@ -1,7 +1,7 @@
 import json
 import re
 from ast import literal_eval
-from typing import Any, Union
+from typing import Any, List, Union
 
 
 class ParseError(Exception):
@@ -58,20 +58,21 @@ def parse_inputs(self, inputs: str, name: str = 'run') -> dict:
         inputs = {self._api2param[name][0]['name']: inputs}
         return inputs
 
-    def parse_outputs(self, outputs: Any) -> dict:
+    def parse_outputs(self, outputs: Any) -> List[dict]:
         """parser outputs returned by the action
 
         Args:
             outputs (:class:`Any`): raw output of the action
 
         Returns:
-            :class:`dict`: processed output
+            :class:`List[dict]`: processed output of which each member is a 
+                dictionary with two keys - 'type' and 'content'.
         """
         if isinstance(outputs, dict):
             outputs = json.dumps(outputs, ensure_ascii=False)
         elif not isinstance(outputs, str):
             outputs = str(outputs)
-        return {'text': outputs}
+        return [{'type': 'text', 'content': outputs}]
 
 
 class JsonParser(BaseParser):

lagent/agents/autogpt.py

@@ -219,21 +219,20 @@ def format(self, goal: str, inner_history: List[Dict],
             dict(role='user', content=self.triggering_prompt))
         return formatted_data
 
-    def format_response(self, action_return):
+    def format_response(self, action_return) -> dict:
         """format the final response at current step.
 
         Args:
             action_return (ActionReturn): return value of the current action.
 
         Returns:
-            str: the final response at current step.
+            dict: the final response at current step.
         """
         if action_return.state == ActionStatusCode.SUCCESS:
-            response = action_return.result['text']
-            response = f'Command {action_return.type} returned: {response}'
+            response = f'Command {action_return.type} returned: {response.format_result()}'
         else:
             response = action_return.errmsg
-        return response
+        return dict(role='system', content=response)
 
 
 class AutoGPT(BaseAgent):
@@ -277,12 +276,9 @@ def chat(self, goal: str, **kwargs) -> AgentReturn:
                 action, action_input)
             agent_return.actions.append(action_return)
             if action_return.type == self._action_executor.finish_action.name:
-                agent_return.response = action_return.result['text']
+                agent_return.response = action_return.format_result()
                 return agent_return
-            inner_history.append(
-                dict(
-                    role='system',
-                    content=self._protocol.format_response(action_return)))
+            inner_history.append(self._protocol.format_response(action_return))
         agent_return.inner_steps = inner_history
         agent_return.response = default_response
         return agent_return

lagent/agents/react.py

@@ -169,20 +169,22 @@ def parse(
         action_input = arg_match[-1]
         return thought, action.strip(), action_input.strip().strip('"')
 
-    def format_response(self, action_return: ActionReturn) -> str:
+    def format_response(self, action_return: ActionReturn) -> dict:
         """format the final response at current step.
 
         Args:
             action_return (ActionReturn): return value of the current action.
 
         Returns:
-            str: the final response at current step.
+            dict: the final response at current step.
         """
         if action_return.state == ActionStatusCode.SUCCESS:
-            response = action_return.result['text']
+            response = action_return.format_result()
         else:
             response = action_return.errmsg
-        return self.response['begin'] + response + self.response['end']
+        return dict(
+            role='system',
+            content=self.response['begin'] + response + self.response['end'])
 
 
 class ReAct(BaseAgent):
@@ -237,12 +239,9 @@ def chat(self, message: Union[str, dict, List[dict]],
             action_return.thought = thought
             agent_return.actions.append(action_return)
             if action_return.type == self._action_executor.finish_action.name:
-                agent_return.response = action_return.result['text']
+                agent_return.response = action_return.format_result()
                 break
-            inner_history.append(
-                dict(
-                    role='system',
-                    content=self._protocol.format_response(action_return)))
+            inner_history.append(self._protocol.format_response(action_return))
         else:
             agent_return.response = default_response
         agent_return.inner_steps = inner_history[offset:]

lagent/agents/rewoo.py

@@ -191,7 +191,7 @@ def format_solver(
         worker_log = ''
         for thought, action_return in zip(thought_list, action_return_list):
             if action_return.state == ActionStatusCode.SUCCESS:
-                action_resp = action_return.result['text']
+                action_resp = action_return.format_result()
             else:
                 action_resp = action_return.errmsg
             worker_response = self.worker_prompt.format(
@@ -273,7 +273,7 @@ def chat(self, message: Union[str, dict, List[dict]],
             for prev_ptr in prev_ptrs:
                 ptr_num = int(prev_ptr.strip('#E')) - 1  # start from 0
                 actions_input[action_id] = actions_input[action_id].replace(
-                    prev_ptr, action_responses[ptr_num].result['text'])
+                    prev_ptr, action_responses[ptr_num].format_result())
             action_return: ActionReturn = self._action_executor(
                 actions[action_id], actions_input[action_id])
             action_responses.append(action_return)

lagent/agents/stream_agent.py

@@ -177,32 +177,9 @@ def parse(self, message, plugin_executor: ActionExecutor,
                 parameters=dict(command=code))
         return None, message, None
 
-    def format_response(self, action_return, name) -> str:
+    def format_response(self, action_return, name) -> dict:
         if action_return.state == ActionStatusCode.SUCCESS:
-            if isinstance(action_return.result, list):
-                response = []
-                for item in action_return.result:
-                    if item['type'] == 'text':
-                        response.append(item['content'])
-                    else:
-                        response.append(f"[{item['type']}]({item['content']})")
-                response = '\n'.join(response)
-            elif isinstance(action_return.result, dict):
-                response = action_return.result['text']
-                if 'image' in action_return.result:
-                    response += '\n'.join([
-                        f'[image]({im})'
-                        for im in action_return.result['image']
-                    ])
-                if 'audio' in action_return.result:
-                    response += '\n'.join([
-                        f'[audio]({im})'
-                        for im in action_return.result['audio']
-                    ])
-            elif isinstance(action_return.result, str):
-                response = action_return.result
-            else:
-                raise NotImplementedError
+            response = action_return.format_result()
         else:
             response = action_return.errmsg
         content = self.execute['begin'] + response + self.execute['end']
@@ -212,8 +189,7 @@ def format_response(self, action_return, name) -> str:
         elif self.execute.get('belong'):
             return dict(
                 role=self.execute['belong'], content=content, name=name)
-        else:
-            return dict(role=self.execute['role'], content=response, name=name)
+        return dict(role=self.execute['role'], content=response, name=name)
 
 
 class StreamAgent(BaseAgent):

lagent/schema.py

@@ -36,12 +36,23 @@ class ActionReturn:
     args: Optional[dict] = None
     url: Optional[str] = None
     type: Optional[str] = None
-    result: Optional[str] = None
+    result: Optional[List[dict]] = None
     errmsg: Optional[str] = None
     state: Union[ActionStatusCode, int] = ActionStatusCode.SUCCESS
     thought: Optional[str] = None
     valid: Optional[ActionValidCode] = ActionValidCode.OPEN
 
+    def format_result(self) -> str:
+        """Concatenate items in result"""
+        result = []
+        for item in self.result or []:
+            if item['type'] == 'text':
+                result.append(item['content'])
+            else:
+                result.append(f"[{item['type']}]({item['content']})")
+        result = '\n'.join(result)
+        return result
+
 
 # 需要集成int，如此asdict可以把AgentStatusCode 转换成 int
 class AgentStatusCode(int, Enum):
@@ -67,4 +78,4 @@ class AgentReturn:
     actions: List[ActionReturn] = field(default_factory=list)
     response: str = ''
     inner_steps: List = field(default_factory=list)
-    errmsg: Optional[str] = None
+    errmsg: Optional[str] = None