From 9f002bf7ae45c4e1afd5cb35c656af0bb84f1a25 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Tue, 26 Dec 2023 21:30:48 +0900 Subject: [PATCH 1/3] chat: assistant set_wakeup_timer function definition --- .../assistant_setup/delete_wakeup_timers.json | 14 ++++++++++++++ .../assistant_setup/get_wakeup_timers.json | 14 ++++++++++++++ .../assistant_setup/set_wakeup_timer.json | 15 +++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/delete_wakeup_timers.json create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/get_wakeup_timers.json create mode 100644 MAVProxy/modules/mavproxy_chat/assistant_setup/set_wakeup_timer.json diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/delete_wakeup_timers.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/delete_wakeup_timers.json new file mode 100644 index 0000000000..d512ff23bf --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/delete_wakeup_timers.json @@ -0,0 +1,14 @@ +{ + "type": "function", + "function": { + "name": "delete_wakeup_timers", + "description": "Delete all active wakeup timers. You can optionally provide a message parameter to filter which timers will be deleted based on their message. When specifying the message parameter, you can use regular expressions (regex) to match patterns within the timer messages. This is useful when you want to delete timers with specific keywords or patterns in their message. For example, to delete all timers containing the word 'hello', you can use the regex '.*hello.*', where the dot-star (.*) pattern matches any character sequence.", + "parameters": { + "type": "object", + "properties": { + "message": {"type": "string", "description": "wakeup message of timers to be deleted. regex values are accepted."} + }, + "required": [] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/get_wakeup_timers.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_wakeup_timers.json new file mode 100644 index 0000000000..e4b6288cd1 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/get_wakeup_timers.json @@ -0,0 +1,14 @@ +{ + "type": "function", + "function": { + "name": "get_wakeup_timers", + "description": "Retrieves a list of all active wakeup timers. You can optionally provide a message parameter to filter timers by their associated messages. When specifying the message parameter, you can use regular expressions (regex) to match patterns within the timer messages. This is useful when you want to find timers with specific keywords or patterns in their messages. For example, to retrieve all timers containing the word 'hello', you can use the regex '.*hello.*', where the dot-star (.*) pattern matches any character sequence.", + "parameters": { + "type": "object", + "properties": { + "message": {"type": "string", "description": "wakeup message of timers to be retrieved. regex values are accepted."} + }, + "required": [] + } + } +} diff --git a/MAVProxy/modules/mavproxy_chat/assistant_setup/set_wakeup_timer.json b/MAVProxy/modules/mavproxy_chat/assistant_setup/set_wakeup_timer.json new file mode 100644 index 0000000000..63dd326cc4 --- /dev/null +++ b/MAVProxy/modules/mavproxy_chat/assistant_setup/set_wakeup_timer.json @@ -0,0 +1,15 @@ +{ + "type": "function", + "function": { + "name": "set_wakeup_timer", + "description": "Set a timer to wake you up in a specified number of seconds in the future. This allows taking actions in the future. The wakeup message will appear with the user role but will look something like WAKEUP:. Multiple wakeup messages are supported", + "parameters": { + "type": "object", + "properties": { + "seconds": {"type": "number", "description": "number of seconds in the future that the timer will wake you up"}, + "message": {"type": "string", "description": "wakeup message that will be sent to you"} + }, + "required": ["seconds", "message"] + } + } +} From 578bc559dd8f4fe28e76d03fc95f606f07b313c5 Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Tue, 26 Dec 2023 21:31:00 +0900 Subject: [PATCH 2/3] chat: add wakeup timer support --- MAVProxy/modules/mavproxy_chat/chat_openai.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/MAVProxy/modules/mavproxy_chat/chat_openai.py b/MAVProxy/modules/mavproxy_chat/chat_openai.py index 249d9eafb0..19230cfec5 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_openai.py +++ b/MAVProxy/modules/mavproxy_chat/chat_openai.py @@ -10,6 +10,7 @@ from pymavlink import mavutil import time, re from datetime import datetime +from threading import Thread import json import math @@ -30,6 +31,11 @@ def __init__(self, mpstate, status_cb=None, wait_for_command_ack_fn=None): # keep reference to wait_for_command_ack_fn self.wait_for_command_ack_fn = wait_for_command_ack_fn + # wakeup timer array + self.wakeup_schedule = [] + self.thread = Thread(target=self.check_wakeup_timers) + self.thread.start() + # initialise OpenAI connection self.client = None self.assistant = None @@ -271,6 +277,36 @@ def handle_function_call(self, run): output = "set_parameter: failed to set parameter value" print("chat: set_parameter: failed to set parameter value") + # set a wakeup timer + if tool_call.function.name == "set_wakeup_timer": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.set_wakeup_timer(arguments) + except: + output = tool_call.function.name + ": failed" + print("chat: " + output) + + # get wakeup timers + if tool_call.function.name == "get_wakeup_timers": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.get_wakeup_timers(arguments) + except: + output = tool_call.function.name + ": failed" + print("chat: " + output) + + # delete wakeup timers + if tool_call.function.name == "delete_wakeup_timers": + recognised_function = True + try: + arguments = json.loads(tool_call.function.arguments) + output = self.delete_wakeup_timers(arguments) + except: + output = tool_call.function.name + ": failed" + print("chat: " + output) + if not recognised_function: print("chat: handle_function_call: unrecognised function call: " + tool_call.function.name) output = "unrecognised function call: " + tool_call.function.name @@ -538,6 +574,105 @@ def set_parameter(self, arguments): self.mpstate.functions.param_set(param_name, param_value, retries=3) return "set_parameter: parameter value set" + # set a wakeup timer + def set_wakeup_timer(self, arguments): + # check required arguments are specified + seconds = arguments.get("seconds", -1) + if seconds < 0: + return "set_wakeup_timer: seconds not specified" + message = arguments.get("message", None) + if message is None: + return "set_wakeup_timer: message not specified" + + # add timer to wakeup schedule + self.wakeup_schedule.append({"time": time.time() + seconds, "message": message}) + return "set_wakeup_timer: wakeup timer set" + + # get wake timers + def get_wakeup_timers(self, arguments): + # check message argument, default to None meaning all + message = arguments.get("message", None) + + # prepare list of matching timers + matching_timers = [] + + # handle simple case of all timers + if message is None: + matching_timers = self.wakeup_schedule + + # handle regex in message + elif self.contains_regex(message): + message_pattern = re.compile(message, re.IGNORECASE) + for wakeup_timer in self.wakeup_schedule: + if message_pattern.match(wakeup_timer["message"]) is not None: + matching_timers.append(wakeup_timer) + + # handle case of a specific message + else: + for wakeup_timer in self.wakeup_schedule: + if wakeup_timer["message"] == message: + matching_timers.append(wakeup_timer) + + # return matching timers + try: + return json.dumps(matching_timers) + except: + return "get_wakeup_timers: failed to convert wakeup timer list to json" + + # delete wake timers + def delete_wakeup_timers(self, arguments): + # check message argument, default to all + message = arguments.get("message", None) + + # find matching timers + num_timers_deleted = 0 + + # handle simple case of deleting all timers + if message is None: + num_timers_deleted = len(self.wakeup_schedule) + self.wakeup_schedule.clear() + + # handle regex in message + elif self.contains_regex(message): + message_pattern = re.compile(message, re.IGNORECASE) + for wakeup_timer in self.wakeup_schedule: + if message_pattern.match(wakeup_timer["message"]) is not None: + num_timers_deleted = num_timers_deleted + 1 + self.wakeup_schedule.remove(wakeup_timer) + else: + # handle simple case of a single message + for wakeup_timer in self.wakeup_schedule: + if wakeup_timer["message"] == message: + num_timers_deleted = num_timers_deleted + 1 + self.wakeup_schedule.remove(wakeup_timer) + + # return number deleted and remaining + return "delete_wakeup_timers: deleted " + str(num_timers_deleted) + " timers, " + str(len(self.wakeup_schedule)) + " remaining" + + # check if any wakeup timers have expired and send messages if they have + # this function never returns so it should be called from a new thread + def check_wakeup_timers(self): + while True: + # wait for one second + time.sleep(1) + + # check if any timers are set + if len(self.wakeup_schedule) == 0: + continue + + # get current time + now = time.time() + + # check if any timers have expired + for wakeup_timer in self.wakeup_schedule: + if now >= wakeup_timer["time"]: + # send message to assistant + message = "WAKEUP:" + wakeup_timer["message"] + self.send_to_assistant(message) + + # remove from wakeup schedule + self.wakeup_schedule.remove(wakeup_timer) + # wrap latitude to range -90 to 90 def wrap_latitude(self, latitude_deg): if latitude_deg > 90: From 3313e638b3eb4fffe5c62b4c41600fdfc08307cd Mon Sep 17 00:00:00 2001 From: Randy Mackay Date: Wed, 27 Dec 2023 21:07:45 +0900 Subject: [PATCH 3/3] chat: move lock on sending to openai file --- MAVProxy/modules/mavproxy_chat/chat_openai.py | 130 +++++++++--------- MAVProxy/modules/mavproxy_chat/chat_window.py | 55 ++++---- 2 files changed, 93 insertions(+), 92 deletions(-) diff --git a/MAVProxy/modules/mavproxy_chat/chat_openai.py b/MAVProxy/modules/mavproxy_chat/chat_openai.py index 19230cfec5..c864f24077 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_openai.py +++ b/MAVProxy/modules/mavproxy_chat/chat_openai.py @@ -10,7 +10,7 @@ from pymavlink import mavutil import time, re from datetime import datetime -from threading import Thread +from threading import Thread, Lock import json import math @@ -31,6 +31,9 @@ def __init__(self, mpstate, status_cb=None, wait_for_command_ack_fn=None): # keep reference to wait_for_command_ack_fn self.wait_for_command_ack_fn = wait_for_command_ack_fn + # lock to prevent multiple threads sending text to the assistant at the same time + self.send_lock = Lock() + # wakeup timer array self.wakeup_schedule = [] self.thread = Thread(target=self.check_wakeup_timers) @@ -93,71 +96,74 @@ def set_api_key(self, api_key_str): # send text to assistant def send_to_assistant(self, text): - # check connection - if not self.check_connection(): - return "chat: failed to connect to OpenAI" - - # create a new message - input_message = self.client.beta.threads.messages.create( - thread_id=self.assistant_thread.id, - role="user", - content=text - ) - if input_message is None: - return "chat: failed to create input message" - - # create a run - self.run = self.client.beta.threads.runs.create( - thread_id=self.assistant_thread.id, - assistant_id=self.assistant.id - ) - if self.run is None: - return "chat: failed to create run" - - # wait for run to complete - run_done = False - while not run_done: - # wait for one second - time.sleep(0.1) + # get lock + with self.send_lock: + + # check connection + if not self.check_connection(): + return "chat: failed to connect to OpenAI" - # retrieve the run - latest_run = self.client.beta.threads.runs.retrieve( + # create a new message + input_message = self.client.beta.threads.messages.create( thread_id=self.assistant_thread.id, - run_id=self.run.id + role="user", + content=text ) + if input_message is None: + return "chat: failed to create input message" - # check run status - if latest_run.status in ["queued", "in_progress", "cancelling"]: - run_done = False - elif latest_run.status in ["cancelled", "failed", "completed", "expired"]: - run_done = True - elif latest_run.status in ["requires_action"]: - self.handle_function_call(latest_run) - run_done = False - else: - print("chat: unrecognised run status" + latest_run.status) - run_done = True - - # send status to status callback - self.send_status(latest_run.status) - - # retrieve messages on the thread - reply_messages = self.client.beta.threads.messages.list(self.assistant_thread.id, order = "asc", after=input_message.id) - if reply_messages is None: - return "chat: failed to retrieve messages" - - # concatenate all messages into a single reply skipping the first which is our question - reply = "" - need_newline = False - for message in reply_messages.data: - reply = reply + message.content[0].text.value - if need_newline: - reply = reply + "\n" - need_newline = True - - if reply is None or reply == "": - return "chat: failed to retrieve latest reply" - return reply + # create a run + self.run = self.client.beta.threads.runs.create( + thread_id=self.assistant_thread.id, + assistant_id=self.assistant.id + ) + if self.run is None: + return "chat: failed to create run" + + # wait for run to complete + run_done = False + while not run_done: + # wait for one second + time.sleep(0.1) + + # retrieve the run + latest_run = self.client.beta.threads.runs.retrieve( + thread_id=self.assistant_thread.id, + run_id=self.run.id + ) + + # check run status + if latest_run.status in ["queued", "in_progress", "cancelling"]: + run_done = False + elif latest_run.status in ["cancelled", "failed", "completed", "expired"]: + run_done = True + elif latest_run.status in ["requires_action"]: + self.handle_function_call(latest_run) + run_done = False + else: + print("chat: unrecognised run status" + latest_run.status) + run_done = True + + # send status to status callback + self.send_status(latest_run.status) + + # retrieve messages on the thread + reply_messages = self.client.beta.threads.messages.list(self.assistant_thread.id, order = "asc", after=input_message.id) + if reply_messages is None: + return "chat: failed to retrieve messages" + + # concatenate all messages into a single reply skipping the first which is our question + reply = "" + need_newline = False + for message in reply_messages.data: + reply = reply + message.content[0].text.value + if need_newline: + reply = reply + "\n" + need_newline = True + + if reply is None or reply == "": + return "chat: failed to retrieve latest reply" + return reply # handle function call request from assistant # on success this returns the text response that should be sent to the assistant, returns None on failure diff --git a/MAVProxy/modules/mavproxy_chat/chat_window.py b/MAVProxy/modules/mavproxy_chat/chat_window.py index ad25c9116c..df5e790f45 100644 --- a/MAVProxy/modules/mavproxy_chat/chat_window.py +++ b/MAVProxy/modules/mavproxy_chat/chat_window.py @@ -7,16 +7,13 @@ from MAVProxy.modules.lib.wx_loader import wx from MAVProxy.modules.mavproxy_chat import chat_openai, chat_voice_to_text -from threading import Thread, Lock +from threading import Thread class chat_window(): def __init__(self, mpstate, wait_for_command_ack_fn): # keep reference to mpstate self.mpstate = mpstate - # lock to prevent multiple threads sending text to the assistant at the same time - self.send_lock = Lock() - # create chat_openai object self.chat_openai = chat_openai.chat_openai(self.mpstate, self.set_status_text, wait_for_command_ack_fn) @@ -150,32 +147,30 @@ def text_input_change(self, event): # send text to assistant. should be called from a separate thread to avoid blocking def send_text_to_assistant(self): - # get lock - with self.send_lock: - # disable buttons and text input to stop multiple inputs (can't be done from a thread or must use CallAfter) - wx.CallAfter(self.record_button.Disable) - wx.CallAfter(self.text_input.Disable) - wx.CallAfter(self.send_button.Disable) - - # get text from text input and clear text input - send_text = self.text_input.GetValue() - wx.CallAfter(self.text_input.Clear) - - # copy user input text to reply box - orig_text_attr = self.text_reply.GetDefaultStyle() - wx.CallAfter(self.text_reply.SetDefaultStyle, wx.TextAttr(wx.RED)) - wx.CallAfter(self.text_reply.AppendText, send_text + "\n") - - # send text to assistant and place reply in reply box - reply = self.chat_openai.send_to_assistant(send_text) - if reply: - wx.CallAfter(self.text_reply.SetDefaultStyle, orig_text_attr) - wx.CallAfter(self.text_reply.AppendText, reply + "\n\n") - - # reenable buttons and text input (can't be done from a thread or must use CallAfter) - wx.CallAfter(self.record_button.Enable) - wx.CallAfter(self.text_input.Enable) - wx.CallAfter(self.send_button.Enable) + # disable buttons and text input to stop multiple inputs (can't be done from a thread or must use CallAfter) + wx.CallAfter(self.record_button.Disable) + wx.CallAfter(self.text_input.Disable) + wx.CallAfter(self.send_button.Disable) + + # get text from text input and clear text input + send_text = self.text_input.GetValue() + wx.CallAfter(self.text_input.Clear) + + # copy user input text to reply box + orig_text_attr = self.text_reply.GetDefaultStyle() + wx.CallAfter(self.text_reply.SetDefaultStyle, wx.TextAttr(wx.RED)) + wx.CallAfter(self.text_reply.AppendText, send_text + "\n") + + # send text to assistant and place reply in reply box + reply = self.chat_openai.send_to_assistant(send_text) + if reply: + wx.CallAfter(self.text_reply.SetDefaultStyle, orig_text_attr) + wx.CallAfter(self.text_reply.AppendText, reply + "\n\n") + + # reenable buttons and text input (can't be done from a thread or must use CallAfter) + wx.CallAfter(self.record_button.Enable) + wx.CallAfter(self.text_input.Enable) + wx.CallAfter(self.send_button.Enable) # set status text def set_status_text(self, text):