diff --git a/agent/prototyper.py b/agent/prototyper.py index d3b3355c7..b683a60d8 100644 --- a/agent/prototyper.py +++ b/agent/prototyper.py @@ -14,6 +14,8 @@ """An LLM agent to generate a simple fuzz target prototype that can build. Use it as a usual module locally, or as script in cloud builds. """ +import copy +import os import subprocess as sp import time from datetime import timedelta @@ -21,10 +23,10 @@ import logger from agent.base_agent import BaseAgent +from data_prep import project_targets from data_prep.project_context.context_introspector import ContextRetriever from experiment.benchmark import Benchmark -from llm_toolkit.prompt_builder import (DefaultTemplateBuilder, - PrototyperTemplateBuilder) +from llm_toolkit import prompt_builder from llm_toolkit.prompts import Prompt from results import BuildResult, Result from tool.container_tool import ProjectContainerTool @@ -38,84 +40,86 @@ class Prototyper(BaseAgent): def _initial_prompt(self, results: list[Result]) -> Prompt: """Constructs initial prompt of the agent.""" benchmark = results[-1].benchmark - retriever = ContextRetriever(benchmark) - context_info = retriever.get_context_info() - prompt_builder = PrototyperTemplateBuilder( + + if benchmark.use_project_examples: + project_examples = project_targets.generate_data( + benchmark.project, + benchmark.language, + cloud_experiment_bucket=self.args.cloud_experiment_bucket) + else: + project_examples = [] + + if self.args.context: + retriever = ContextRetriever(benchmark) + context_info = retriever.get_context_info() + else: + context_info = {} + + builder = prompt_builder.PrototyperTemplateBuilder( model=self.llm, benchmark=benchmark, ) - prompt = prompt_builder.build(example_pair=[], - project_context_content=context_info, - tool_guides=self.inspect_tool.tutorial()) - # prompt = prompt_builder.build(example_pair=EXAMPLE_FUZZ_TARGETS.get( - # benchmark.language, []), - # tool_guides=self.inspect_tool.tutorial()) + prompt = builder.build(example_pair=prompt_builder.EXAMPLES.get( + benchmark.file_type.value.lower(), []), + project_example_content=project_examples, + project_context_content=context_info, + tool_guides=self.inspect_tool.tutorial(), + project_dir=self.inspect_tool.project_dir) return prompt - def _update_fuzz_target_and_build_script(self, cur_round: int, response: str, + def _update_fuzz_target_and_build_script(self, response: str, build_result: BuildResult) -> None: """Updates fuzz target and build script in build_result with LLM response. """ fuzz_target_source = self._filter_code( self._parse_tag(response, 'fuzz target')) build_result.fuzz_target_source = fuzz_target_source - if fuzz_target_source: - logger.debug('ROUND %02d Parsed fuzz target from LLM: %s', - cur_round, - fuzz_target_source, - trial=build_result.trial) - else: - logger.error('ROUND %02d No fuzz target source code in conclusion: %s', - cur_round, - response, - trial=build_result.trial) build_script_source = self._filter_code( self._parse_tag(response, 'build script')) # Sometimes LLM adds chronos, which makes no sense for new build scripts. build_result.build_script_source = build_script_source.replace( 'source /src/chronos.sh', '') - if build_script_source: - logger.debug('ROUND %02d Parsed build script from LLM: %s', - cur_round, - build_script_source, - trial=build_result.trial) - else: - logger.debug('ROUND %02d No build script in conclusion: %s', - cur_round, - response, - trial=build_result.trial) def _update_build_result(self, build_result: BuildResult, - compile_process: sp.CompletedProcess, status: bool, - referenced: bool) -> None: + compile_process: sp.CompletedProcess, compiles: bool, + binary_exists: bool, referenced: bool) -> None: """Updates the build result with the latest info.""" - build_result.compiles = status + build_result.compiles = compiles build_result.compile_error = compile_process.stderr build_result.compile_log = self._format_bash_execution_result( compile_process) + build_result.binary_exists = binary_exists build_result.is_function_referenced = referenced - def _validate_fuzz_target_and_build_script(self, cur_round: int, - build_result: BuildResult) -> None: + def _validate_fuzz_target_and_build_script( + self, cur_round: int, build_result: BuildResult + ) -> tuple[Optional[BuildResult], Optional[BuildResult]]: """Validates the new fuzz target and build script.""" # Steps: # 1. Recompile without modifying the build script, in case LLM is wrong. # 2. Recompile with the modified build script, if any. - build_script_source = build_result.build_script_source + build_result_alt = None + if build_result.build_script_source: + build_result_alt = copy.deepcopy(build_result) + logger.info('First compile fuzz target without modifying build script.', + trial=build_result_alt.trial) + build_result_alt.build_script_source = '' + self._validate_fuzz_target_and_build_script_via_compile( + cur_round, build_result_alt) + + # No need to run expensive build_result, when *_alt is perfect. + if build_result_alt and build_result_alt.success: + return build_result_alt, None - logger.info('First compile fuzz target without modifying build script.', + # New fuzz target + has new build.sh. + logger.info('Compile fuzz target with modified build script.', trial=build_result.trial) - build_result.build_script_source = '' self._validate_fuzz_target_and_build_script_via_compile( cur_round, build_result) - if not build_result.success and build_script_source: - logger.info('Then compile fuzz target with modified build script.', - trial=build_result.trial) - build_result.build_script_source = build_script_source - self._validate_fuzz_target_and_build_script_via_compile( - cur_round, build_result) + # Although build_result_alt is not perfect, LLM may still learn from it. + return build_result_alt, build_result def _validate_fuzz_target_references_function( self, compilation_tool: ProjectContainerTool, benchmark: Benchmark, @@ -196,9 +200,182 @@ def _validate_fuzz_target_and_build_script_via_compile( compilation_tool.terminate() self._update_build_result(build_result, compile_process=compile_process, - status=compile_succeed and binary_exists, + compiles=compile_succeed, + binary_exists=binary_exists, referenced=function_referenced) + def _generate_prompt_from_build_result( + self, build_result_alt: Optional[BuildResult], + build_result_ori: Optional[BuildResult], build_result: BuildResult, + prompt: Prompt, cur_round: int) -> tuple[BuildResult, Optional[Prompt]]: + """Selects which build result to use and generates a prompt accordingly.""" + + # Case 1: Successful. + if build_result_alt and build_result_alt.success: + # Preference 1: New fuzz target + default build.sh can compile, save + # binary to expected path, and reference function-under-test. + logger.info( + 'Default /src/build.sh works perfectly, no need for a new ' + 'buid script', + trial=build_result.trial) + logger.info('***** Prototyper succeded in %02d rounds *****', + cur_round, + trial=build_result.trial) + return build_result_alt, None + + if build_result_ori and build_result_ori.success: + # Preference 2: New fuzz target + new build.sh can compile, save + # binary to expected path, and reference function-under-test. + logger.info('***** Prototyper succeded in %02d rounds *****', + cur_round, + trial=build_result.trial) + return build_result_ori, None + + # Case 2: Binary exits, meaning not referencing function-under-test. + function_signature = build_result.benchmark.function_signature + fuzz_target_source = build_result.fuzz_target_source + build_script_source = build_result.build_script_source + compile_log = self.llm.truncate_prompt(build_result.compile_log, + extra_text=prompt.get()).strip() + prompt_text = ( + "The fuzz target's `LLVMFuzzerTestOneInput` did not invoke the " + f'function-under-test `{function_signature}`:\n' + f'\n{fuzz_target_source}\n\n' + '{BUILD_TEXT}\n' + f'\n{compile_log}\n\n' + 'That is NOT enough. YOU MUST MODIFY THE FUZZ TARGET to CALL ' + f'FUNCTION `{function_signature}` **EXPLICITLY OR IMPLICITLY** in ' + '`LLVMFuzzerTestOneInput` to generate a valid fuzz target.\nStudy the ' + 'source code for function usages to know how.\n') + if build_result_alt and build_result_alt.binary_exists: + # Preference 3: New fuzz target + default build.sh can compile and save + # binary to expected path, but does not reference function-under-test. + prompt_text = prompt_text.replace( + '{BUILD_TEXT}', + 'Althoug `/src/build.bk.sh` compiles and saves the binary to the ' + 'correct path:') + # NOTE: Unsafe to say the following, because /src/build.sh may miss a + # library required by the function-under-test, and the fuzz target did not + # invoke the function-under-test either. + # prompt_text += ( + # 'In addition, given the default /src/build.sh works perfectly, you ' + # 'do not have to generate a new build script and can leave ' + # ' empty.') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target. YOU MUST NOT OMIT ANY CODE even if it is the same as before.' + '\n') + prompt.append(prompt_text) + return build_result_alt, prompt + if (build_result_ori and build_result_ori.binary_exists and + not build_result_ori.build_script_source): + # Preference 4.1: New fuzz target + default build.sh can compile and save + # binary to expected path, but does not reference function-under-test. + prompt_text = prompt_text.replace( + '{BUILD_TEXT}', + 'Althoug `/src/build.bk.sh` compiles and saves the binary to the ' + 'correct path:') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target. YOU MUST NOT OMIT ANY CODE even if it is the same as before.' + '\n') + prompt.append(prompt_text) + return build_result_ori, prompt + if build_result_ori and build_result_ori.binary_exists: + # Preference 4.2: New fuzz target + New build.sh can compile and save + # binary to expected path, but does not reference function-under-test. + prompt_text = prompt_text.replace( + '{BUILD_TEXT}', + 'Althoug your build script compiles and saves the binary to the ' + 'correct path:\n' + f'\n{build_script_source}\n\n') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target (and the FULL build script, if any). YOU MUST NOT OMIT ANY ' + 'CODE even if it is the same as before.\n') + prompt.append(prompt_text) + return build_result_ori, prompt + + # Case 3: Compiles, meaning the binary is not saved. + binary_path = os.path.join('/out', build_result.benchmark.target_name) + if (build_result_ori and build_result_ori.compiles and + build_result_ori.build_script_source): + # Preference 5.1: New fuzz target + new build.sh can compile, but does + # not save binary to expected path. + prompt_text = ( + 'The fuzz target and build script compiles successfully, but the ' + 'final fuzz target binary was not saved to the expected path at ' + f'`{binary_path}`.\n' + f'\n{fuzz_target_source}\n\n' + f'\n{build_script_source}\n\n' + f'\n{compile_log}\n\n' + 'YOU MUST MODIFY THE BUILD SCRIPT to ensure the binary is saved to ' + f'{binary_path}.\n') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target (and the FULL build script, if any). YOU MUST NOT OMIT ANY ' + 'CODE even if it is the same as before.\n') + prompt.append(prompt_text) + return build_result_ori, prompt + if (build_result_ori and build_result_ori.compiles and + not build_result_ori.build_script_source): + # Preference 5.2: New fuzz target + default build.sh can compile, but does + # not save binary to expected path, indicating benchmark data error. + logger.error( + 'The human-written build.sh does not save the fuzz target binary to ' + 'expected path /out/%s, indicating incorrect info in benchmark YAML.', + build_result.benchmark.target_name, + trial=build_result.trial) + prompt_text = ( + 'The fuzz target compiles successfully with /src/build.bk.sh, but the' + ' final fuzz target binary was not saved to the expected path at ' + f'`{binary_path}`.\n' + f'\n{fuzz_target_source}\n\n' + f'\n{compile_log}\n\n' + 'YOU MUST MODIFY THE BUILD SCRIPT to ensure the binary is saved to ' + f'{binary_path}.\n') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target (and the FULL build script, if any). YOU MUST NOT OMIT ANY ' + 'CODE even if it is the same as before.\n') + prompt.append(prompt_text) + return build_result_ori, prompt + if build_result_alt and build_result_alt.compiles: + # Preference 6: New fuzz target + default build.sh can compile, but does + # not save binary to expected path, indicating benchmark data error. + logger.error( + 'The human-written build.sh does not save the fuzz target binary to ' + 'expected path /out/%s, indicating incorrect info in benchmark YAML.', + build_result.benchmark.target_name, + trial=build_result.trial) + prompt_text = ( + 'The fuzz target compiles successfully with /src/build.bk.sh, but the' + ' final fuzz target binary was not saved to the expected path at ' + f'`{binary_path}`.\n' + f'\n{fuzz_target_source}\n\n' + f'\n{compile_log}\n\n' + 'YOU MUST MODIFY THE BUILD SCRIPT to ensure the binary is saved to ' + f'{binary_path}.\n') + prompt_text += ( + 'When you have a solution later, make sure you output the FULL fuzz ' + 'target (and the FULL build script, if any). YOU MUST NOT OMIT ANY ' + 'CODE even if it is the same as before.\n') + prompt.append(prompt_text) + return build_result_alt, prompt + + # Preference 7: New fuzz target + both `build.sh`s cannot compile. No need + # to mention the default build.sh. + # return build_result + builder = prompt_builder.PrototyperFixerTemplateBuilder( + model=self.llm, + benchmark=build_result.benchmark, + build_result=build_result, + compile_log=compile_log, + initial=prompt.get()) + prompt = builder.build(example_pair=[], + project_dir=self.inspect_tool.project_dir) + return build_result, prompt + def _container_handle_conclusion(self, cur_round: int, response: str, build_result: BuildResult, prompt: Prompt) -> Optional[Prompt]: @@ -210,69 +387,21 @@ def _container_handle_conclusion(self, cur_round: int, response: str, cur_round, trial=build_result.trial) - self._update_fuzz_target_and_build_script(cur_round, response, build_result) + self._update_fuzz_target_and_build_script(response, build_result) - self._validate_fuzz_target_and_build_script(cur_round, build_result) - if build_result.success: - logger.info('***** Prototyper succeded in %02d rounds *****', - cur_round, - trial=build_result.trial) - return None + build_result_alt, build_result_ori = ( + self._validate_fuzz_target_and_build_script(cur_round, build_result)) - if not build_result.compiles: - compile_log = self.llm.truncate_prompt(build_result.compile_log, - extra_text=prompt.get()).strip() - logger.info('***** Failed to recompile in %02d rounds *****', - cur_round, - trial=build_result.trial) - prompt_text = ( - 'Failed to build fuzz target. Here is the fuzz target, build script, ' - 'compliation command, and other compilation runtime output. Analyze ' - 'the error messages, the fuzz target, and the build script carefully ' - 'to identify the root cause. Avoid making random changes to the fuzz ' - 'target or build script without a clear understanding of the error. ' - 'If necessary, #include necessary headers and #define required macros' - 'or constants in the fuzz target, or adjust compiler flags to link ' - 'required libraries in the build script. After collecting information' - ', analyzing and understanding the error root cause, YOU MUST take at' - ' least one step to validate your theory with source code evidence. ' - 'Only if your theory is verified, respond the revised fuzz target and' - 'build script in FULL.\n' - 'Always try to learn from the source code about how to fix errors, ' - 'for example, search for the key words (e.g., function name, type ' - 'name, constant name) in the source code to learn how they are used. ' - 'Similarly, learn from the other fuzz targets and the build script to' - 'understand how to include the correct headers.\n' - 'Focus on writing a minimum buildable fuzz target that calls the ' - 'target function. We can increase its complexity later, but first try' - 'to make it compile successfully.' - 'If an error happens repeatedly and cannot be fixed, try to ' - 'mitigate it. For example, replace or remove the line.' - f'\n{build_result.fuzz_target_source}\n\n' - f'\n{build_result.build_script_source}\n' - f'\n\n{compile_log}\n\n') - elif not build_result.is_function_referenced: - logger.info( - '***** Fuzz target does not reference function-under-test in %02d ' - 'rounds *****', - cur_round, - trial=build_result.trial) - prompt_text = ( - 'The fuzz target builds successfully, but the target function ' - f'`{build_result.benchmark.function_signature}` was not used by ' - '`LLVMFuzzerTestOneInput` in fuzz target. YOU MUST CALL FUNCTION ' - f'`{build_result.benchmark.function_signature}` INSIDE FUNCTION ' - '`LLVMFuzzerTestOneInput`.') - else: - prompt_text = '' + # Updates build_result with _alt or _ori, depending on their status. + build_result, prompt_final = self._generate_prompt_from_build_result( + build_result_alt, build_result_ori, build_result, prompt, cur_round) - prompt.append(prompt_text) - return prompt + return prompt_final def _container_tool_reaction(self, cur_round: int, response: str, build_result: BuildResult) -> Optional[Prompt]: """Validates LLM conclusion or executes its command.""" - prompt = DefaultTemplateBuilder(self.llm, None).build([]) + prompt = prompt_builder.DefaultTemplateBuilder(self.llm, None).build([]) prompt = self._container_handle_bash_commands(response, self.inspect_tool, prompt) diff --git a/experiment/builder_runner.py b/experiment/builder_runner.py index 542a2ef17..b95d50bf2 100644 --- a/experiment/builder_runner.py +++ b/experiment/builder_runner.py @@ -922,17 +922,15 @@ def build_and_run_cloud( f'--real_project={project_name}', ] - # Temporarily comment out due to error in cached images. - # TODO(dongge): Add this back when the cached image works again. - # if oss_fuzz_checkout.ENABLE_CACHING and ( - # oss_fuzz_checkout.is_image_cached(project_name, 'address') and - # oss_fuzz_checkout.is_image_cached(project_name, 'coverage')): - # logger.info('Using cached image for %s', project_name) - # command.append('--use_cached_image') - - # # Overwrite the Dockerfile to be caching friendly - # oss_fuzz_checkout.rewrite_project_to_cached_project_chronos( - # generated_project) + if oss_fuzz_checkout.ENABLE_CACHING and ( + oss_fuzz_checkout.is_image_cached(project_name, 'address') and + oss_fuzz_checkout.is_image_cached(project_name, 'coverage')): + logger.info('Using cached image for %s', project_name) + command.append('--use_cached_image') + + # Overwrite the Dockerfile to be caching friendly + oss_fuzz_checkout.rewrite_project_to_cached_project_chronos( + generated_project) if cloud_build_tags: command += ['--tags'] + cloud_build_tags diff --git a/experiment/evaluator.py b/experiment/evaluator.py index 752095de6..6c94d705c 100644 --- a/experiment/evaluator.py +++ b/experiment/evaluator.py @@ -306,6 +306,8 @@ def create_ossfuzz_project(self, os.path.basename('agent-build.sh'))) # Add additional statement in dockerfile to overwrite with generated fuzzer + with open(os.path.join(generated_project_path, 'Dockerfile'), 'a') as f: + f.write('\nRUN cp /src/build.sh /src/build.bk.sh\n') with open(os.path.join(generated_project_path, 'Dockerfile'), 'a') as f: f.write('\nCOPY agent-build.sh /src/build.sh\n') diff --git a/experiment/oss_fuzz_checkout.py b/experiment/oss_fuzz_checkout.py index 9621903e3..9dd42c267 100644 --- a/experiment/oss_fuzz_checkout.py +++ b/experiment/oss_fuzz_checkout.py @@ -70,7 +70,7 @@ def _clone_oss_fuzz_repo(): """Clones OSS-Fuzz to |OSS_FUZZ_DIR|.""" clone_command = [ 'git', 'clone', 'https://github.com/google/oss-fuzz', '--depth', '1', - '--branch', 'target-exp-log-account', OSS_FUZZ_DIR + OSS_FUZZ_DIR ] proc = sp.Popen(clone_command, stdout=sp.PIPE, diff --git a/llm_toolkit/prompt_builder.py b/llm_toolkit/prompt_builder.py index 99edd2bae..27cf3f764 100644 --- a/llm_toolkit/prompt_builder.py +++ b/llm_toolkit/prompt_builder.py @@ -28,6 +28,7 @@ from experiment.benchmark import Benchmark, FileType from experiment.fuzz_target_error import SemanticCheckResult from llm_toolkit import models, prompts +from results import BuildResult logger = logging.getLogger(__name__) @@ -546,15 +547,22 @@ class PrototyperTemplateBuilder(DefaultTemplateBuilder): def __init__(self, model: models.LLM, benchmark: Benchmark, - template_dir: str = DEFAULT_TEMPLATE_DIR): - super().__init__(model) - self._template_dir = template_dir + template_dir: str = DEFAULT_TEMPLATE_DIR, + initial: Any = None): + super().__init__(model, benchmark, template_dir, initial) self.agent_templare_dir = AGENT_TEMPLATE_DIR - self.benchmark = benchmark # Load templates. - self.priming_template_file = self._find_template(self.agent_templare_dir, - 'prototyper-priming.txt') + if benchmark.is_c_target: + self.priming_template_file = self._find_template( + self.agent_templare_dir, 'prototyper-priming.c.txt') + elif benchmark.is_cpp_target: + self.priming_template_file = self._find_template( + self.agent_templare_dir, 'prototyper-priming.cpp.txt') + else: + self.problem_template_file = self._find_template( + self.agent_templare_dir, 'prototyper-priming.txt') + self.cpp_priming_filler_file = self._find_template( template_dir, 'cpp-specific-priming-filler.txt') self.problem_template_file = self._find_template(template_dir, @@ -568,11 +576,13 @@ def build(self, example_pair: list[list[str]], project_example_content: Optional[list[list[str]]] = None, project_context_content: Optional[dict] = None, - tool_guides: str = '') -> prompts.Prompt: + tool_guides: str = '', + project_dir: str = '') -> prompts.Prompt: """Constructs a prompt using the templates in |self| and saves it.""" if not self.benchmark: return self._prompt priming = self._format_priming(self.benchmark) + priming = priming.replace('{PROJECT_DIR}', project_dir) final_problem = self.format_problem(self.benchmark.function_signature) final_problem += (f'You MUST call \n' f'{self.benchmark.function_signature}\n' @@ -585,6 +595,54 @@ def build(self, return self._prompt +class PrototyperFixerTemplateBuilder(PrototyperTemplateBuilder): + """Builder specifically targeted C (and excluding C++).""" + + def __init__(self, + model: models.LLM, + benchmark: Benchmark, + build_result: BuildResult, + compile_log: str, + template_dir: str = DEFAULT_TEMPLATE_DIR, + initial: Any = None): + super().__init__(model, benchmark, template_dir, initial) + # Load templates. + self.priming_template_file = self._find_template(self.agent_templare_dir, + 'prototyper-fixing.txt') + self.build_result = build_result + self.compile_log = compile_log + + def build(self, + example_pair: list[list[str]], + project_example_content: Optional[list[list[str]]] = None, + project_context_content: Optional[dict] = None, + tool_guides: str = '', + project_dir: str = '') -> prompts.Prompt: + """Constructs a prompt using the templates in |self| and saves it.""" + del (example_pair, project_example_content, project_context_content, + tool_guides) + if not self.benchmark: + return self._prompt + + if self.build_result.build_script_source: + build_text = (f'\n{self.build_result.build_script_source}\n' + '') + else: + build_text = 'Build script reuses `/src/build.bk.sh`.' + + prompt = self._get_template(self.priming_template_file) + prompt = prompt.replace('{FUZZ_TARGET_SOURCE}', + self.build_result.fuzz_target_source) + prompt = prompt.replace('{BUILD_TEXT}', build_text) + prompt = prompt.replace('{COMPILE_LOG}', self.compile_log) + prompt = prompt.replace('{FUNCTION_SIGNATURE}', + self.benchmark.function_signature) + prompt = prompt.replace('{PROJECT_DIR}', project_dir) + self._prompt.append(prompt) + + return self._prompt + + class DefaultJvmTemplateBuilder(PromptBuilder): """Default builder for JVM projects.""" diff --git a/prompts/agent/prototyper-fixing.txt b/prompts/agent/prototyper-fixing.txt new file mode 100644 index 000000000..94a19ade9 --- /dev/null +++ b/prompts/agent/prototyper-fixing.txt @@ -0,0 +1,16 @@ +Failed to build fuzz target. Here is the fuzz target, build script, compilation command, and compilation output: +\n{FUZZ_TARGET_SOURCE}\n +{BUILD_TEXT} +\n{COMPILE_LOG}\n +YOU MUST first analyze the error messages with the fuzz target and the build script carefully to identify the root cause. +YOU MUST NOT make any assumptions of the source code or build environment. Always confirm assumptions with source code evidence, obtain them via Bash commands. +Once you are absolutely certain of the error root cause, output the FULL SOURCE CODE of the fuzz target (and FULL SOURCE CODE of build script, if /src/build.bk.sh is insufficient). +TIPS: +1. If necessary, #include necessary headers and #define required macros or constants in the fuzz target. +2. Adjust compiler flags to link required libraries in the build script. +3. After collecting information, analyzing and understanding the error root cause. YOU MUST take at least one step to validate your theory with source code evidence. +4. Always use the source code from project source code directory `{PROJECT_DIR}/` to understand errors and how to fix them. For example, search for the key words (e.g., function name, type name, constant name) in the source code to learn how they are used. Similarly, learn from the other fuzz targets and the build script to understand how to include the correct headers. +5. Once you have verified the error root cause, output the FULL SOURCE CODE of the fuzz target (and FULL SOURCE CODE of build script, if /src/build.bk.sh is insufficient). +6. Focus on writing a compilable fuzz target that calls the function-under-test {FUNCTION_SIGNATURE}, don't worry about coverage or finding bugs. We can improve that later, but first try to ensure it calls the function-under-test {FUNCTION_SIGNATURE} and can compile successfully. +7. If an error happens repeatedly and cannot be fixed, try to mitigate it. For example, replace or remove the line. + diff --git a/prompts/agent/prototyper-priming.c.txt b/prompts/agent/prototyper-priming.c.txt new file mode 100644 index 000000000..27540296d --- /dev/null +++ b/prompts/agent/prototyper-priming.c.txt @@ -0,0 +1,141 @@ + +As a security testing engineer, you must write an `int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)` fuzz target in {LANGUAGE}. +Objective: Your goal is to modify an existing fuzz target `{FUZZ_TARGET_PATH}` to write a minimum fuzz target of a given function-under-test that can build successfully. + + + +Follow these steps to write a minimum fuzz target: + +Step 1. Determine the information you need to write an effective fuzz target. +This includes: + * **Source code** of the function under test. + * **Custom Types and Dependencies** definitions and implementations. + * **Initialization and setup** requirements and steps. + * **Build details** and integration steps. + * Valid and edge-case input values. + * Environmental and runtime dependencies. + +Step 2. Collect information using the Bash tool. +Use the bash tool (see section) and follow its rules to gather the necessary information. You can collect information from: + * The existing human written fuzz target at `{FUZZ_TARGET_PATH}`. + * The existing human written build script `/src/build.bk.sh`. + * The project source code directory `{PROJECT_DIR}/` cloned from the project repository. + * Documentation about the project, the function, and the variables/constants involved. + * Environment variables. + * Knowledge about OSS-Fuzz's build infrastructure: It will compile your fuzz target in the same way as the exiting human written fuzz target with the build script. + +Step 3. Analyze the function and its parameters. +Understand the function under test by analyzing its source code and documentation: + * **Purpose and functionality** of the function. + * **Input processing** and internal logic. + * **Dependencies** on other functions or global variables. + * **Error handling** and edge cases. + +Step 4. Understand initialization requirements. +Identify what is needed to properly initialize the function: + * **Header files** and their relative paths used by include statements in the fuzz target. + * **Complex input parameters or objects** initialization. + * **Constructor functions** or initialization routines. + * **Global state** or configuration needs to be set up. + * **Mocking** external dependencies if necessary. + +Step 5. Understand Constraints and edge cases. +For each input parameter, understand: + * Valid ranges and data types. + * Invalid or edge-case values (e.g., zero, NULL, predefined constants, maximum values). + * Special values that trigger different code paths. + +Step 6: Plan Fuzz Target Implementation. +Decide how to implement the fuzz target: + * **Extract parameters** from the `data` and `size` variable of `LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)`. + * Handle fixed-size versus variable-size data. + * **Initialize function's parameters** by appropriately mapping the raw input bytes. + * Ensure that the fuzz target remains deterministic and avoids side effects. + * Avoid `goto` statements. + +Step 7: **Write** the fuzz target code. +Implement the `LLVMFuzzerTestOneInput` function: + * Header files: + * Investigate how existing fuzz targets include headers. + * Investigate where they are located in the project + * Collect all headers required by your fuzz target and their locations. + * Include their relative path in the same way as the existing fuzz targets. + * Macros or Constants: + * Include or define necessary macros or constants. + * Input Handling: + * Check that the input size is sufficient. + * Extract parameters from the input data. + * Handle any necessary conversions or validations. + * Function Invocation: + * Initialize required objects or state. + * Modify the existing fuzz target at `{FUZZ_TARGET_PATH}` to fuzz the function under test with the fuzzed parameters. + * Ensure proper error handling. + * + * Cleanup: + * Free any allocated resources. + * Reset any global state if necessary. + +Step 8 (Optional): **Modify** the Build Script. +Write a new build script only if the existing one (`/src/build.bk.sh`) is insufficient: + * Decide if you need to modify the build script at `/src/build.bk.sh` to successfully build the new fuzz target. + * Include compilation steps for the project under test. + * Include compilation steps for the new fuzz target. + * Specify necessary compiler and linker flags. + * Ensure all dependencies are correctly linked. + +Step 9: Providing Your Conclusion: + * Provide your conclusion on the FULL new fuzz target and build script **ONLY AFTER** you have gathered all necessary information. + * **DO NOT SEND** any other content (e.g., bash tool commands) in the conclusion message. ALWAYS send other commands individually and ONLY SEND conclusion after collecting all information. + * Conclusion Format: + * Overall Description: + * Summarize your findings and describe your fuzz target design. + * Wrap this summary within and tags. + * Modified Fuzz Target: + * Provide the full code of the modified fuzz target. + * Wrap the code within and tags. + * Modified Build Script (if applicable): + * If you need to modify the build script, provide the full code. + * Wrap it within and tags. + * Format Example: + + I determined that the fuzz target needs to include specific header files and adjust the `LLVMFuzzerTestOneInput` function to call the new function-under-test. Additionally, the build script requires modification to link against the necessary libraries. + + + [Your FULL fuzz target code here.] + + + [Your FULL build script code here, if applicable.] + + + + +{TYPE_SPECIFIC_PRIMING} + + +3. Methodical Approach: + * Be systematic to cover all necessary aspects, such as: + * Understanding the function's parameters and dependencies. + * Identifying required header files and libraries. + * Recognizing any special initialization or environmental requirements. +1. Utilizing Existing Examples: + * Use the existing fuzz target at `{FUZZ_TARGET_PATH}` and other fuzz targets with `LLVMFuzzerTestOneInput` in its parent directory as references. + * Pay special attention to: + * How header files are included. + * The structure and content of the `LLVMFuzzerTestOneInput` function. + * Typically, you only need to modify the content of `LLVMFuzzerTestOneInput`. +2. Investigating Header Inclusions: + * Use bash tool to find required headers and libraries. + * Examine library files built by `/src/build.bk.sh` to understand available functions and symbols. +3. Modifying the Build Script (if necessary): + * Modifying `/src/build.bk.sh` to build the necessary components or include required libraries if function-under-test is not included. + * The project's directory may contain a `README.md` with build instructions (e.g., at `/src//README.md` +4. Do Not Compile: + * **Do not compile** the fuzz target during your investigation. + * Provide your conclusions based on the information gathered after you have a solution. +5. Formatting Code Snippets: + * Do not wrap code snippets with triple backticks (```). + * Use the specified XML-style tags for wrapping code and other content. +6. DO NOT send the early: Provide conclusions **only after** gathering all necessary information. +7. Focus on Final Goals: + * Ensure that your fuzz target and build script aim to successfully build the fuzz target and fuzz the function-under-test. + diff --git a/prompts/agent/prototyper-priming.cpp.txt b/prompts/agent/prototyper-priming.cpp.txt new file mode 100644 index 000000000..87ea6a2c2 --- /dev/null +++ b/prompts/agent/prototyper-priming.cpp.txt @@ -0,0 +1,143 @@ + +As a security testing engineer, you must write an `int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)` fuzz target in {LANGUAGE}. +Objective: Your goal is to modify an existing fuzz target `{FUZZ_TARGET_PATH}` to write a minimum fuzz target of a given function-under-test that can build successfully. + + + +Follow these steps to write a minimum fuzz target: + +Step 1. Determine the information you need to write an effective fuzz target. +This includes: + * **Source code** of the function under test. + * **Custom Types and Dependencies** definitions and implementations. + * **Initialization and setup** requirements and steps. + * **Build details** and integration steps. + * Valid and edge-case input values. + * Environmental and runtime dependencies. + +Step 2. Collect information using the Bash tool. +Use the bash tool (see section) and follow its rules to gather the necessary information. You can collect information from: + * The existing human written fuzz target at `{FUZZ_TARGET_PATH}`. + * The existing human written build script `/src/build.bk.sh`. + * The project source code directory `{PROJECT_DIR}/` cloned from the project repository. + * Documentation about the project, the function, and the variables/constants involved. + * Environment variables. + * Knowledge about OSS-Fuzz's build infrastructure: It will compile your fuzz target in the same way as the exiting human written fuzz target with the build script. + +Step 3. Analyze the function and its parameters. +Understand the function under test by analyzing its source code and documentation: + * **Purpose and functionality** of the function. + * **Input processing** and internal logic. + * **Dependencies** on other functions or global variables. + * **Error handling** and edge cases. + +Step 4. Understand initialization requirements. +Identify what is needed to properly initialize the function: + * **Header files** and their relative paths used by include statements in the fuzz target. + * **Complex input parameters or objects** initialization. + * **Constructor functions** or initialization routines. + * **Global state** or configuration needs to be set up. + * **Mocking** external dependencies if necessary. + +Step 5. Understand Constraints and edge cases. +For each input parameter, understand: + * Valid ranges and data types. + * Invalid or edge-case values (e.g., zero, NULL, predefined constants, maximum values). + * Special values that trigger different code paths. + +Step 6: Plan Fuzz Target Implementation. +Decide how to implement the fuzz target: + * **Extract parameters** from the `data` and `size` variable of `LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)`. + * Handle fixed-size versus variable-size data. + * **Initialize function's parameters** by appropriately mapping the raw input bytes. + * Ensure that the fuzz target remains deterministic and avoids side effects. + * Avoid `goto` statements. + +Step 7: **Write** the fuzz target code. +Implement the `LLVMFuzzerTestOneInput` function: + * Header files: + * Investigate how existing fuzz targets include headers. + * Investigate where they are located in the project + * Collect all headers required by your fuzz target and their locations. + * Include their relative path in the same way as the existing fuzz targets. + * Macros or Constants: + * Include or define necessary macros or constants. + * Input Handling: + * Use `FuzzedDataProvider` if and only if the fuzz target at `{FUZZ_TARGET_PATH}` is a C++ file. + * Use `extern "C"` if and only if the fuzz target at `{FUZZ_TARGET_PATH}` is a C++ file. + * Check that the input size is sufficient. + * Extract parameters from the input data. + * Handle any necessary conversions or validations. + * Function Invocation: + * Initialize required objects or state. + * Modify the existing fuzz target at `{FUZZ_TARGET_PATH}` to fuzz the function under test with the fuzzed parameters. + * Ensure proper error handling. + * + * Cleanup: + * Free any allocated resources. + * Reset any global state if necessary. + +Step 8 (Optional): **Modify** the Build Script. +Write a new build script only if the existing one (`/src/build.bk.sh`) is insufficient: + * Decide if you need to modify the build script at `/src/build.bk.sh` to successfully build the new fuzz target. + * Include compilation steps for the project under test. + * Include compilation steps for the new fuzz target. + * Specify necessary compiler and linker flags. + * Ensure all dependencies are correctly linked. + +Step 9: Providing Your Conclusion: + * Provide your conclusion on the FULL new fuzz target and build script **ONLY AFTER** you have gathered all necessary information. + * **DO NOT SEND** any other content (e.g., bash tool commands) in the conclusion message. ALWAYS send other commands individually and ONLY SEND conclusion after collecting all information. + * Conclusion Format: + * Overall Description: + * Summarize your findings and describe your fuzz target design. + * Wrap this summary within and tags. + * Modified Fuzz Target: + * Provide the full code of the modified fuzz target. + * Wrap the code within and tags. + * Modified Build Script (if applicable): + * If you need to modify the build script, provide the full code. + * Wrap it within and tags. + * Format Example: + + I determined that the fuzz target needs to include specific header files and adjust the `LLVMFuzzerTestOneInput` function to call the new function-under-test. Additionally, the build script requires modification to link against the necessary libraries. + + + [Your FULL fuzz target code here.] + + + [Your FULL build script code here, if applicable.] + + + + +{TYPE_SPECIFIC_PRIMING} + + +3. Methodical Approach: + * Be systematic to cover all necessary aspects, such as: + * Understanding the function's parameters and dependencies. + * Identifying required header files and libraries. + * Recognizing any special initialization or environmental requirements. +1. Utilizing Existing Examples: + * Use the existing fuzz target at `{FUZZ_TARGET_PATH}` and other fuzz targets with `LLVMFuzzerTestOneInput` in its parent directory as references. + * Pay special attention to: + * How header files are included. + * The structure and content of the `LLVMFuzzerTestOneInput` function. + * Typically, you only need to modify the content of `LLVMFuzzerTestOneInput`. +2. Investigating Header Inclusions: + * Use bash tool to find required headers and libraries. + * Examine library files built by `/src/build.bk.sh` to understand available functions and symbols. +3. Modifying the Build Script (if necessary): + * Modifying `/src/build.bk.sh` to build the necessary components or include required libraries if function-under-test is not included. + * The project's directory may contain a `README.md` with build instructions (e.g., at `/src//README.md` +4. Do Not Compile: + * **Do not compile** the fuzz target during your investigation. + * Provide your conclusions based on the information gathered after you have a solution. +5. Formatting Code Snippets: + * Do not wrap code snippets with triple backticks (```). + * Use the specified XML-style tags for wrapping code and other content. +6. DO NOT send the early: Provide conclusions **only after** gathering all necessary information. +7. Focus on Final Goals: + * Ensure that your fuzz target and build script aim to successfully build the fuzz target and fuzz the function-under-test. + diff --git a/prompts/agent/prototyper-priming.txt b/prompts/agent/prototyper-priming.txt index 7b51cd8e6..87ea6a2c2 100644 --- a/prompts/agent/prototyper-priming.txt +++ b/prompts/agent/prototyper-priming.txt @@ -18,8 +18,8 @@ This includes: Step 2. Collect information using the Bash tool. Use the bash tool (see section) and follow its rules to gather the necessary information. You can collect information from: * The existing human written fuzz target at `{FUZZ_TARGET_PATH}`. - * The existing human written build script `/src/build.sh`. - * The project source code (in `.`, or `/src//`) clone from the project repository. + * The existing human written build script `/src/build.bk.sh`. + * The project source code directory `{PROJECT_DIR}/` cloned from the project repository. * Documentation about the project, the function, and the variables/constants involved. * Environment variables. * Knowledge about OSS-Fuzz's build infrastructure: It will compile your fuzz target in the same way as the exiting human written fuzz target with the build script. @@ -78,8 +78,8 @@ Implement the `LLVMFuzzerTestOneInput` function: * Reset any global state if necessary. Step 8 (Optional): **Modify** the Build Script. -Write a new build script only if the existing one (`/src/build.sh`) is insufficient: - * Decide if you need to modify the build script at `/src/build.sh` to successfully build the new fuzz target. +Write a new build script only if the existing one (`/src/build.bk.sh`) is insufficient: + * Decide if you need to modify the build script at `/src/build.bk.sh` to successfully build the new fuzz target. * Include compilation steps for the project under test. * Include compilation steps for the new fuzz target. * Specify necessary compiler and linker flags. @@ -127,9 +127,9 @@ Step 9: Providing Your Conclusion: * Typically, you only need to modify the content of `LLVMFuzzerTestOneInput`. 2. Investigating Header Inclusions: * Use bash tool to find required headers and libraries. - * Examine library files built by `/src/build.sh` to understand available functions and symbols. + * Examine library files built by `/src/build.bk.sh` to understand available functions and symbols. 3. Modifying the Build Script (if necessary): - * Modifying `/src/build.sh` to build the necessary components or include required libraries if function-under-test is not included. + * Modifying `/src/build.bk.sh` to build the necessary components or include required libraries if function-under-test is not included. * The project's directory may contain a `README.md` with build instructions (e.g., at `/src//README.md` 4. Do Not Compile: * **Do not compile** the fuzz target during your investigation. diff --git a/prompts/tool/container_tool.txt b/prompts/tool/container_tool.txt index 5f8d8de50..9270442eb 100644 --- a/prompts/tool/container_tool.txt +++ b/prompts/tool/container_tool.txt @@ -92,7 +92,7 @@ Command 5. Check Build Script for Compilation Flags and Libraries: To check which compiler flags and libraries are used in the build script. - cat /src/build.sh + cat /src/build.bk.sh Command 6. Verify Available Libraries: @@ -111,8 +111,7 @@ Command 7. Understand Environment Variables: -1. STRICTLY ONE Bash Command in each message: Do NOT send multiple bash commands in one message. -3. Do Not Compile or Run Code: +1. Do Not Compile or Run Code: * Your investigation is limited to reading and interpreting information using bash commands. diff --git a/results.py b/results.py index 698290e39..ac777f2db 100644 --- a/results.py +++ b/results.py @@ -72,6 +72,7 @@ class BuildResult(Result): compiles: bool # Build success/failure. compile_error: str # Build error message. compile_log: str # Build full output. + binary_exists: bool # Fuzz target binary generated successfully. is_function_referenced: bool # Fuzz target references function-under-test. def __init__(self, @@ -81,6 +82,7 @@ def __init__(self, compiles: bool = False, compile_error: str = '', compile_log: str = '', + binary_exists: bool = False, is_function_referenced: bool = False, fuzz_target_source: str = '', build_script_source: str = '', @@ -91,6 +93,7 @@ def __init__(self, self.compiles = compiles self.compile_error = compile_error self.compile_log = compile_log + self.binary_exists = binary_exists self.is_function_referenced = is_function_referenced def to_dict(self) -> dict: @@ -98,12 +101,13 @@ def to_dict(self) -> dict: 'compiles': self.success, 'compile_error': self.compile_error, 'compile_log': self.compile_log, + 'binary_exists': self.binary_exists, 'is_function_referenced': self.is_function_referenced, } @property def success(self): - return self.compiles and self.is_function_referenced + return self.compiles and self.binary_exists and self.is_function_referenced # TODO: Make this class an attribute of Result, avoid too many attributes in one @@ -132,6 +136,7 @@ def __init__( compiles: bool = False, compile_error: str = '', compile_log: str = '', + binary_exists: bool = False, is_function_referenced: bool = False, crashes: bool = False, # Runtime crash. run_error: str = '', # Runtime crash error message. @@ -151,8 +156,9 @@ def __init__( author: Any = None, chat_history: Optional[dict] = None) -> None: super().__init__(benchmark, trial, work_dirs, compiles, compile_error, - compile_log, is_function_referenced, fuzz_target_source, - build_script_source, author, chat_history) + compile_log, binary_exists, is_function_referenced, + fuzz_target_source, build_script_source, author, + chat_history) self.crashes = crashes self.run_error = run_error self.run_log = run_log diff --git a/stage/execution_stage.py b/stage/execution_stage.py index 353d12327..11e0e6539 100644 --- a/stage/execution_stage.py +++ b/stage/execution_stage.py @@ -133,6 +133,7 @@ def execute(self, result_history: list[Result]) -> Result: compiles=last_result.compiles, compile_error=last_result.compile_error, compile_log=last_result.compile_log, + binary_exists=last_result.binary_exists, is_function_referenced=last_result.is_function_referenced, crashes=run_result.crashes, run_error=run_result.crash_info, @@ -161,6 +162,7 @@ def execute(self, result_history: list[Result]) -> Result: compiles=last_result.compiles, compile_error=last_result.compile_error, compile_log=last_result.compile_log, + binary_exists=last_result.binary_exists, is_function_referenced=last_result.is_function_referenced) return runresult diff --git a/tool/container_tool.py b/tool/container_tool.py index 5976e3ed3..4888f77a3 100644 --- a/tool/container_tool.py +++ b/tool/container_tool.py @@ -29,6 +29,8 @@ def __init__(self, benchmark: Benchmark, name: str = '') -> None: super().__init__(benchmark, name) self.image_name = self._prepare_project_image() self.container_id = self._start_docker_container() + self._backup_default_build_script() + self.project_dir = self._get_project_dir() def tutorial(self) -> str: """Constructs a tool guide tutorial for LLM agents.""" @@ -86,6 +88,23 @@ def _execute_command(self, command: list[str]) -> sp.CompletedProcess: e) return sp.CompletedProcess(command, returncode=1, stdout='', stderr='') + def _backup_default_build_script(self) -> None: + """Creates a copy of the human-written /src/build.sh for LLM to use.""" + backup_command = 'cp /src/build.sh /src/build.bk.sh' + process = self.execute(backup_command) + if process.returncode: + logger.error('Failed to create a backup of /src/build.sh: %s', + self.image_name) + + def _get_project_dir(self) -> str: + """Returns the project-under-test's source code directory.""" + pwd_command = 'pwd' + process = self.execute(pwd_command) + if process.returncode: + logger.error('Failed to get the WORKDIR: %s', self.image_name) + return '' + return process.stdout.strip() + def _start_docker_container(self) -> str: """Runs the project's OSS-Fuzz image as a background container and returns the container ID.""" @@ -112,7 +131,11 @@ def execute(self, command: str) -> sp.CompletedProcess: def compile(self, extra_commands: str = '') -> sp.CompletedProcess: """Compiles the fuzz target.""" command = 'compile > /dev/null' + extra_commands - return self.execute(command) + compile_process = self.execute(command) + # Hide Compilation command so that LLM won't reuse it in the inspection tool + # and be distracted by irrelevant errors, e.g., `build/ already exits`. + compile_process.args = '# Compiles the fuzz target.' + return compile_process def terminate(self) -> bool: """Terminates the container."""