Add Toit language (#6419)

* Add Toit language.

* Move Toit section to correct place.

* Update README.md

---------

Co-authored-by: Colin Seymour <[email protected]>

.gitmodules

@@ -515,6 +515,9 @@
 [submodule "vendor/grammars/hoon-grammar"]
 	path = vendor/grammars/hoon-grammar
 	url = https://github.com/pkova/hoon-grammar.git
+[submodule "vendor/grammars/ide-tools"]
+	path = vendor/grammars/ide-tools
+	url = https://github.com/toitware/ide-tools.git
 [submodule "vendor/grammars/idl.tmbundle"]
 	path = vendor/grammars/idl.tmbundle
 	url = https://github.com/mgalloy/idl.tmbundle

grammars.yml

@@ -430,6 +430,8 @@ vendor/grammars/holyc.tmbundle:
 - source.hc
 vendor/grammars/hoon-grammar:
 - source.hoon
+vendor/grammars/ide-tools:
+- source.toit
 vendor/grammars/idl.tmbundle:
 - source.idl
 - source.idl-dlm

lib/linguist/languages.yml

@@ -7109,6 +7109,14 @@ Thrift:
   - ".thrift"
   ace_mode: text
   language_id: 374
+Toit:
+  type: programming
+  color: "#c2c9fb"
+  extensions:
+  - ".toit"
+  tm_scope: source.toit
+  ace_mode: text
+  language_id: 356554395
 Turing:
   type: programming
   color: "#cf142b"

samples/Toit/chatbot.toit

@@ -0,0 +1,177 @@
+// Copyright (C) 2023 Florian Loitsch
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+import openai
+
+/**
+If there is a gap of more than MAX_GAP between messages, we clear the
+conversation.
+*/
+MAX_GAP ::= Duration --m=3
+/** The maximum number of messages we keep in memory for each chat. */
+MAX_MESSAGES ::= 20
+
+class TimestampedMessage:
+  text/string
+  timestamp/Time
+  is_from_assistant/bool
+
+  constructor --.text --.timestamp --.is_from_assistant:
+
+/**
+A base class for a chat bot.
+
+In addition to implementing the abstract methods $my_name_, $send_message_,
+  subclasses must periodically call $clear_old_messages_. Ideally, this
+  should happen whenever a new event is received from the server.
+
+Typically, a `run` function proceeds in three steps:
+```
+run:
+  while true:
+    message := get_new_message  // From the chat server.
+    clear_old_messages_         // Call to this bot.
+    store_message_ message.text --chat_id=message.chat_id
+    if should_respond:  // This might depend on the message or client.
+      request_response_ message.chat_id
+```
+
+The chat_id is only necessary for bots that can be in multiple channels.
+It's safe to use 0 if the bot doesn't need to keep track of multiple chats.
+
+Once the bot receives a response it calls $send_message_.
+*/
+abstract class ChatBot:
+  // The client is created lazily, to avoid memory pressure during startup.
+  openai_client_/openai.Client? := null
+  openai_key_/string? := ?
+
+  // Maps from chat-id to deque.
+  // Only authenticated chat-ids are in this map.
+  all_messages_/Map := {:}
+
+  /**
+  Creates a new instance of the bot.
+
+  The $max_gap parameter is used to determine if a chat has moved on to
+    a new topic (which leads to a new conversation for the AI bot).
+
+  The $max_messages parameter is used to determine how many messages
+    are kept in memory for each chat.
+  */
+  constructor
+      --openai_key/string
+      --max_gap/Duration=MAX_GAP
+      --max_messages/int=MAX_MESSAGES:
+    openai_key_ = openai_key
+
+  close:
+    if openai_client_:
+      openai_client_.close
+      openai_client_ = null
+      openai_key_ = null
+
+  /** The name of the bot. Sent as a system message. */
+  abstract my_name_ -> string
+
+  /** Sends a message to the given $chat_id. */
+  abstract send_message_ text/string --chat_id/any
+
+  /** Returns the messages for the given $chat_id. */
+  messages_for_ chat_id/any -> Deque:
+    return all_messages_.get chat_id --init=: Deque
+
+  /**
+  Drops old messages from all watched chats.
+  Uses the $MAX_GAP constant to determine if a chat has moved on to
+    a new topic (which leads to a new conversation for the AI bot).
+  */
+  clear_old_messages_:
+    now := Time.now
+    all_messages_.do: | chat_id/any messages/Deque |
+      if messages.is_empty: continue.do
+      last_message := messages.last
+      if (last_message.timestamp.to now) > MAX_GAP:
+        print "Clearing $chat_id"
+        messages.clear
+
+  /**
+  Builds an OpenAI conversation for the given $chat_id.
+
+  Returns a list of $openai.ChatMessage objects.
+  */
+  build_conversation_ chat_id/any -> List:
+    result := [
+      openai.ChatMessage.system "You are contributing to chat of potentially multiple people. Your name is '$my_name_'. Be short.",
+    ]
+    messages := messages_for_ chat_id
+    messages.do: | timestamped_message/TimestampedMessage |
+      if timestamped_message.is_from_assistant:
+        result.add (openai.ChatMessage.assistant timestamped_message.text)
+      else:
+        // We are not combining multiple messages from the user.
+        // Typically, the chat is a back and forth between the user and
+        // the assistant. For memory reasons we prefer to make individual
+        // messages.
+        result.add (openai.ChatMessage.user timestamped_message.text)
+    return result
+
+  /** Stores the $response that the assistant produced in the chat. */
+  store_assistant_response_ response/string --chat_id/any:
+    messages := messages_for_ chat_id
+    messages.add (TimestampedMessage
+      --text=response
+      --timestamp=Time.now
+      --is_from_assistant)
+
+  /**
+  Stores a user-provided $text in the list of messages for the
+    given $chat_id.
+  The $text should contain the name of the author.
+  */
+  store_message_ text/string --chat_id/any --timestamp/Time=Time.now -> none:
+    messages := messages_for_ chat_id
+    // Drop messages if we have too many of them.
+    if messages.size >= MAX_MESSAGES:
+        messages.remove_first
+
+    new_timestamped_message := TimestampedMessage
+        // We store the user with the message.
+        // This is mainly so we don't need to create a new string
+        // when we create the conversation.
+        --text=text
+        --timestamp=timestamp
+        --is_from_assistant=false
+    messages.add new_timestamped_message
+
+  /**
+  Sends a response to the given $chat_id.
+  */
+  send_response_ chat_id/any:
+    if not openai_client_:
+      if not openai_key_: throw "Closed"
+      openai_client_ = openai.Client --key=openai_key_
+
+    conversation := build_conversation_ chat_id
+    response := openai_client_.complete_chat
+        --conversation=conversation
+        --max_tokens=300
+    store_assistant_response_ response --chat_id=chat_id
+    send_message_ response --chat_id=chat_id

samples/Toit/logger.toit

@@ -0,0 +1,150 @@
+// Copyright (C) 2023 Florian Loitsch
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+import certificate_roots
+import gpio
+import monitor
+import uart
+import supabase
+
+// After 5s offload the data. Even if the device is not quiet.
+MAX_OFFLOAD_DELAY ::= Duration --s=5
+// Offload if we accumulate more than 2kb of data.
+MAX_BUFFERED_DATA ::= 2000
+// Offload if we have not received any data for 500ms.
+MAX_QUIET_FOR_OFFLOAD ::= Duration --ms=500
+
+LOGS_TABLE ::= "logs"
+
+class LogForwarder:
+  pin_/gpio.Pin
+  port_/uart.Port
+  buffered_/List := [] // Of ByteArray.
+  buffered_size_/int := 0
+  offload_task_/Task? := null
+  upload_/Lambda
+
+  constructor pin_number/int --upload/Lambda:
+    pin_ = gpio.Pin pin_number
+    port_ = uart.Port --rx=pin_ --tx=null --baud_rate=115200
+    upload_ = upload
+    offload_task_ = task::
+      offload_
+
+  close:
+    if offload_task_:
+      offload_task_.cancel
+      port_.close
+      pin_.close
+      offload_task_ = null
+
+  listen:
+    while true:
+      chunk := port_.read
+      buffer_ chunk
+
+  buffer_ data/ByteArray:
+    print "Received $data.to_string_non_throwing"
+    buffered_.add data
+    buffered_size_ += data.size
+    if buffered_size_ > MAX_BUFFERED_DATA:
+      offload_
+
+  offload_:
+    last_offload := Time.now
+    while true:
+      last_message := Time.now
+      old_size := buffered_size_
+      while (Duration.since last_message) < MAX_QUIET_FOR_OFFLOAD:
+        sleep --ms=20
+
+        if buffered_size_ == 0:
+          // Reset the timer.
+          last_offload = Time.now
+          last_message = last_offload
+          continue
+
+        if (Duration.since last_offload) > MAX_OFFLOAD_DELAY:
+          break
+
+        if buffered_size_ == old_size:
+          continue
+
+        if buffered_size_ > MAX_BUFFERED_DATA:
+          print "too much data"
+          break
+
+        last_message = Time.now
+        old_size = buffered_size_
+
+      print "Offloading"
+      total := ByteArray buffered_size_
+      offset := 0
+      buffered_.do:
+        total.replace offset it
+        offset += it.size
+      to_upload := total.to_string_non_throwing
+      buffered_.clear
+      buffered_size_ = 0
+      print "Uploading: $to_upload"
+      upload_.call to_upload
+
+main
+    --supabase_project/string
+    --supabase_anon/string
+    --device_id/string
+    --pin_rx1/int
+    --pin_rx2/int?:
+
+  client/supabase.Client? := null
+  forwarder1/LogForwarder? := null
+  forwarder2/LogForwarder? := null
+
+  while true:
+    // Trying to work around https://github.com/toitlang/pkg-http/issues/89
+    catch --trace:
+      client = supabase.Client.tls
+          --host="$(supabase_project).supabase.co"
+          --anon=supabase_anon
+          --root_certificates=[certificate_roots.BALTIMORE_CYBERTRUST_ROOT]
+
+      mutex := monitor.Mutex
+
+      offload := :: | uart_pin/int data/string |
+        mutex.do:
+          client.rest.insert --no-return_inserted LOGS_TABLE {
+            "device_id": device_id,
+            "uart_pin": uart_pin,
+            "data": data,
+          }
+
+      if pin_rx2:
+        task::
+          forwarder2 = LogForwarder pin_rx2 --upload=:: | data/string |
+            offload.call pin_rx2 data
+          forwarder2.listen
+
+      forwarder1 = LogForwarder pin_rx1 --upload=:: | data/string |
+        offload.call pin_rx1 data
+      forwarder1.listen
+
+    if forwarder1: forwarder1.close
+    if forwarder2: forwarder2.close
+    if client: client.close