Demonstrate passing "max_tokens" param

context triton-inference-server/server#6864
mkhludnev · Feb 15, 2024 · 257a2a9 · 257a2a9
1 parent 6f0afff
commit 257a2a9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/samples/client.py b/samples/client.py
@@ -90,7 +90,7 @@ async def process_stream(self, prompts, sampling_parameters):
                     self._results_dict[result.get_response().id].append(i)
 
     async def run(self):
-        sampling_parameters = {"temperature": "0.1", "top_p": "0.95"}
+        sampling_parameters = {"temperature": "0.1", "top_p": "0.95", "max_tokens":"100"}
         with open(self._flags.input_prompts, "r") as file:
             print(f"Loading inputs from `{self._flags.input_prompts}`...")
             prompts = file.readlines()