refactor: move runtime/repo info from system message to user message

All-Hands-AI · Jan 15, 2025 · acf0522 · acf0522
1 parent 97e633c
commit acf0522
Show file tree

Hide file tree

Showing 24 changed files with 72 additions and 48 deletions.
diff --git a/config.template.toml b/config.template.toml
@@ -220,8 +220,8 @@ codeact_enable_jupyter = true
 # LLM config group to use
 #llm_config = 'your-llm-config-group'
 
-# Whether to use microagents at all
-#use_microagents = true
+# Whether to use prompt extension (e.g., microagent, repo/runtime info) at all
+#use_prompt_extension = true
 
 # List of microagents to disable
 #disabled_microagents = []

diff --git a/...n/zh-Hans/docusaurus-plugin-content-docs/current/usage/configuration-options.md b/...n/zh-Hans/docusaurus-plugin-content-docs/current/usage/configuration-options.md
@@ -373,7 +373,7 @@ Agent 配置选项在 `config.toml` 文件的 `[agent]` 和 `[agent.<agent_name>
   - 描述: 是否在 action space 中启用 Jupyter
 
 **Microagent 使用**
-- `use_microagents`
+- `use_prompt_extension`
   - 类型: `bool`
   - 默认值: `true`
   - 描述: 是否使用 microagents

diff --git a/docs/modules/usage/configuration-options.md b/docs/modules/usage/configuration-options.md
@@ -332,7 +332,7 @@ The agent configuration options are defined in the `[agent]` and `[agent.<agent_
   - Description: Whether Jupyter is enabled in the action space
 
 ### Microagent Usage
-- `use_microagents`
+- `use_prompt_extension`
   - Type: `bool`
   - Default: `true`
   - Description: Whether to use microagents at all

diff --git a/evaluation/benchmarks/EDA/run_infer.py b/evaluation/benchmarks/EDA/run_infer.py
@@ -76,7 +76,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/agent_bench/run_infer.py b/evaluation/benchmarks/agent_bench/run_infer.py
@@ -60,7 +60,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/aider_bench/run_infer.py b/evaluation/benchmarks/aider_bench/run_infer.py
@@ -68,7 +68,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
 
     # copy 'draft_editor' config if exists
     config_copy = copy.deepcopy(config)

diff --git a/evaluation/benchmarks/biocoder/run_infer.py b/evaluation/benchmarks/biocoder/run_infer.py
@@ -74,7 +74,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/bird/run_infer.py b/evaluation/benchmarks/bird/run_infer.py
@@ -87,7 +87,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/browsing_delegation/run_infer.py b/evaluation/benchmarks/browsing_delegation/run_infer.py
@@ -51,7 +51,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/discoverybench/run_infer.py b/evaluation/benchmarks/discoverybench/run_infer.py
@@ -78,7 +78,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     agent_config = AgentConfig(
         function_calling=False,
         codeact_enable_jupyter=True,

diff --git a/evaluation/benchmarks/gaia/run_infer.py b/evaluation/benchmarks/gaia/run_infer.py
@@ -63,7 +63,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/gorilla/run_infer.py b/evaluation/benchmarks/gorilla/run_infer.py
@@ -56,7 +56,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/gpqa/run_infer.py b/evaluation/benchmarks/gpqa/run_infer.py
@@ -77,7 +77,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/humanevalfix/run_infer.py b/evaluation/benchmarks/humanevalfix/run_infer.py
@@ -98,7 +98,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/logic_reasoning/run_infer.py b/evaluation/benchmarks/logic_reasoning/run_infer.py
@@ -62,7 +62,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/mint/run_infer.py b/evaluation/benchmarks/mint/run_infer.py
@@ -120,7 +120,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/ml_bench/run_infer.py b/evaluation/benchmarks/ml_bench/run_infer.py
@@ -93,7 +93,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/toolqa/run_infer.py b/evaluation/benchmarks/toolqa/run_infer.py
@@ -57,7 +57,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/evaluation/benchmarks/webarena/run_infer.py b/evaluation/benchmarks/webarena/run_infer.py
@@ -78,7 +78,7 @@ def get_config(
     )
     config.set_llm_config(metadata.llm_config)
     agent_config = config.get_agent_config(metadata.agent_class)
-    agent_config.use_microagents = False
+    agent_config.use_prompt_extension = False
     return config
 
 

diff --git a/openhands/agenthub/codeact_agent/codeact_agent.py b/openhands/agenthub/codeact_agent/codeact_agent.py
@@ -111,7 +111,7 @@ def __init__(
                 os.path.dirname(os.path.dirname(openhands.__file__)),
                 'microagents',
             )
-            if self.config.use_microagents
+            if self.config.use_prompt_extension
             else None,
             prompt_dir=os.path.join(os.path.dirname(__file__), 'prompts'),
             disabled_microagents=self.config.disabled_microagents,
@@ -448,6 +448,17 @@ def _get_messages(self, state: State) -> list[Message]:
                 )
             )
 
+        # Repository and runtime info
+        additional_info = self.prompt_manager.get_additional_info()
+        if self.config.use_prompt_extension and additional_info:
+            # only add these if prompt extension is enabled
+            messages.append(
+                Message(
+                    role='user',
+                    content=[TextContent(text=additional_info)],
+                )
+            )
+
         pending_tool_call_action_messages: dict[str, Message] = {}
         tool_call_id_to_message: dict[str, Message] = {}
 

diff --git a/openhands/agenthub/codeact_agent/prompts/system_prompt.j2 b/openhands/agenthub/codeact_agent/prompts/system_prompt.j2
@@ -3,26 +3,4 @@ You are OpenHands agent, a helpful AI assistant that can interact with a compute
 * If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.
 * When configuring git credentials, use "openhands" as the user.name and "[email protected]" as the user.email by default, unless explicitly instructed otherwise.
 * The assistant MUST NOT include comments in the code unless they are necessary to describe non-obvious behavior.
-{{ runtime_info }}
 </IMPORTANT>
-{% if repository_info %}
-<REPOSITORY_INFO>
-At the user's request, repository {{ repository_info.repo_name }} has been cloned to directory {{ repository_info.repo_directory }}.
-</REPOSITORY_INFO>
-{% endif %}
-{% if repository_instructions -%}
-<REPOSITORY_INSTRUCTIONS>
-{{ repository_instructions }}
-</REPOSITORY_INSTRUCTIONS>
-{% endif %}
-{% if runtime_info and runtime_info.available_hosts -%}
-<RUNTIME_INFORMATION>
-The user has access to the following hosts for accessing a web application,
-each of which has a corresponding port:
-{% for host, port in runtime_info.available_hosts.items() -%}
-* {{ host }} (port {{ port }})
-{% endfor %}
-When starting a web server, use the corresponding ports. You should also
-set any options to allow iframes and CORS requests.
-</RUNTIME_INFORMATION>
-{% endif %}
diff --git a/openhands/core/config/agent_config.py b/openhands/core/config/agent_config.py
@@ -17,7 +17,7 @@ class AgentConfig:
         memory_enabled: Whether long-term memory (embeddings) is enabled.
         memory_max_threads: The maximum number of threads indexing at the same time for embeddings.
         llm_config: The name of the llm config to use. If specified, this will override global llm config.
-        use_microagents: Whether to use microagents at all. Default is True.
+        use_prompt_extension: Whether to use prompt extension (e.g., microagents, inject runtime info). Default is True.
         disabled_microagents: A list of microagents to disable. Default is None.
         condenser: Configuration for the memory condenser. Default is NoOpCondenserConfig.
     """
@@ -29,7 +29,7 @@ class AgentConfig:
     memory_enabled: bool = False
     memory_max_threads: int = 3
     llm_config: str | None = None
-    use_microagents: bool = True
+    use_prompt_extension: bool = True
     disabled_microagents: list[str] | None = None
     condenser: CondenserConfig = field(default_factory=NoOpCondenserConfig)  # type: ignore
 

diff --git a/openhands/utils/prompt.py b/openhands/utils/prompt.py
@@ -28,6 +28,34 @@ class RepositoryInfo:
     repo_directory: str | None = None
 
 
+ADDITIONAL_INFO_TEMPLATE = Template(
+    """
+Additional information about the environment:
+{% if repository_info %}
+<REPOSITORY_INFO>
+At the user's request, repository {{ repository_info.repo_name }} has been cloned to directory {{ repository_info.repo_directory }}.
+</REPOSITORY_INFO>
+{% endif %}
+{% if repository_instructions -%}
+<REPOSITORY_INSTRUCTIONS>
+{{ repository_instructions }}
+</REPOSITORY_INSTRUCTIONS>
+{% endif %}
+{% if runtime_info and runtime_info.available_hosts -%}
+<RUNTIME_INFORMATION>
+The user has access to the following hosts for accessing a web application,
+each of which has a corresponding port:
+{% for host, port in runtime_info.available_hosts.items() -%}
+* {{ host }} (port {{ port }})
+{% endfor %}
+When starting a web server, use the corresponding ports. You should also
+set any options to allow iframes and CORS requests.
+</RUNTIME_INFORMATION>
+{% endif %}
+""".strip()
+)
+
+
 class PromptManager:
     """
     Manages prompt templates and micro-agents for AI interactions.
@@ -105,6 +133,13 @@ def _load_template(self, template_name: str) -> Template:
             return Template(file.read())
 
     def get_system_message(self) -> str:
+        return self.system_template.render().strip()
+
+    def get_additional_info(self) -> str:
+        """Gets information about the repository and runtime.
+
+        This is used to inject information about the repository and runtime into the initial user message.
+        """
         repo_instructions = ''
         assert (
             len(self.repo_microagents) <= 1
@@ -115,7 +150,7 @@ def get_system_message(self) -> str:
                 repo_instructions += '\n\n'
             repo_instructions += microagent.content
 
-        return self.system_template.render(
+        return ADDITIONAL_INFO_TEMPLATE.render(
             repository_instructions=repo_instructions,
             repository_info=self.repository_info,
             runtime_info=self.runtime_info,

diff --git a/tests/unit/test_codeact_agent.py b/tests/unit/test_codeact_agent.py
@@ -471,7 +471,7 @@ def test_mock_function_calling():
     llm = Mock()
     llm.is_function_calling_active = lambda: False
     config = AgentConfig()
-    config.use_microagents = False
+    config.use_prompt_extension = False
     agent = CodeActAgent(llm=llm, config=config)
     assert agent.mock_function_calling is True
 
@@ -509,7 +509,7 @@ def test_step_with_no_pending_actions(mock_state: State):
 
     # Create agent with mocked LLM
     config = AgentConfig()
-    config.use_microagents = False
+    config.use_prompt_extension = False
     agent = CodeActAgent(llm=llm, config=config)
 
     # Test step with no pending actions