Semantic search added

assets/js/micro.js

@@ -11,23 +11,21 @@ export default {
       blue = ["bg-blue-500", "hover:bg-blue-700"],
       pulseGreen = ["bg-green-500", "hover:bg-green-700", "animate-pulse"];
 
-
     _this = this;
 
     // Adding event listener for "click" event
     recordButton.addEventListener("click", () => {
-
       // Check if it's recording.
       // If it is, we stop the record and update the elements.
       if (mediaRecorder && mediaRecorder.state === "recording") {
         mediaRecorder.stop();
+        // audioChunks.getAudioTracks()[0].stop();
         text.textContent = "Record";
-      } 
+      }
 
       // Otherwise, it means the user wants to start recording.
       else {
         navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
-
           // Instantiate MediaRecorder
           mediaRecorder = new MediaRecorder(stream);
           mediaRecorder.start();
@@ -39,7 +37,7 @@ export default {
 
           // Add "dataavailable" event handler
           mediaRecorder.addEventListener("dataavailable", (event) => {
-            audioChunks.push(event.data);
+            event.data.size > 0 && audioChunks.push(event.data);
           });
 
           // Add "stop" event handler for when the recording stops.
@@ -57,4 +55,4 @@ export default {
       }
     });
   },
-};
+};

lib/app/application.ex

@@ -7,7 +7,6 @@ defmodule App.Application do
 
   @impl true
   def start(_type, _args) do
-
     App.Models.verify_and_download_models()
 
     children = [
@@ -18,17 +17,18 @@ defmodule App.Application do
       # Start the PubSub system
       {Phoenix.PubSub, name: App.PubSub},
       # Nx serving for the embedding
-      # App.TextEmbedding,
+      App.TextEmbedding,
+      App.KnnIndex,
 
       # Nx serving for Speech-to-Text
       {Nx.Serving,
-      serving:
-        if Application.get_env(:app, :use_test_models) == true do
-          App.Models.audio_serving_test()
-        else
-          App.Models.audio_serving()
-        end,
-      name: Whisper},
+       serving:
+         if Application.get_env(:app, :use_test_models) == true do
+           App.Models.audio_serving_test()
+         else
+           App.Models.audio_serving()
+         end,
+       name: Whisper},
       # Nx serving for image classifier
       {Nx.Serving,
        serving:
@@ -39,7 +39,7 @@ defmodule App.Application do
          end,
        name: ImageClassifier},
       {GenMagic.Server, name: :gen_magic},
-      
+
       # Adding a supervisor
       {Task.Supervisor, name: App.TaskSupervisor},
       # Start the Endpoint (http/https)

lib/app/hnswlib_index.ex

@@ -0,0 +1,48 @@
+defmodule App.HnswlibIndex do
+  use Ecto.Schema
+
+  alias App.HnswlibIndex
+  alias App.Repo
+
+  require Logger
+
+  schema "hnswlib_index" do
+    field(:file, :binary)
+  end
+
+  def changeset(struct \\ %__MODULE__{}, params \\ %{}) do
+    struct
+    |> Ecto.Changeset.cast(params, [:id, :file])
+    |> Ecto.Changeset.validate_required([:id])
+  end
+
+  def save() do
+    path = App.KnnIndex.get_index_path()
+    file = File.read!(path)
+
+    Repo.get!(HnswlibIndex, 1)
+    |> HnswlibIndex.changeset(%{file: file})
+    |> Repo.update()
+  end
+
+  def maybe_load_index_from_db(space, dim, max_elements) do
+    Repo.get_by(HnswlibIndex, id: 1)
+    |> case do
+      nil ->
+        # create a singleton row
+        HnswlibIndex.changeset(%__MODULE__{}, %{id: 1})
+        |> Repo.insert()
+
+        Logger.info("New Index")
+        HNSWLib.Index.new(space, dim, max_elements)
+
+      index_file ->
+        Logger.info("Loading Index from DB")
+        path = App.KnnIndex.get_index_path()
+        # save on disk
+        File.write!(path, index_file.file)
+        # load the index file
+        HNSWLib.Index.load_index(space, dim, path)
+    end
+  end
+end

lib/app/image.ex

@@ -8,26 +8,53 @@ defmodule App.Image do
     field(:width, :integer)
     field(:url, :string)
     field(:height, :integer)
+    field(:idx, :integer)
+    field(:sha1, :string)
 
     timestamps(type: :utc_datetime)
   end
 
   def changeset(image, params \\ %{}) do
     image
-    |> Ecto.Changeset.cast(params, [:url, :description, :width, :height])
+    |> Ecto.Changeset.cast(params, [:url, :description, :width, :height, :idx, :sha1])
     |> Ecto.Changeset.validate_required([:url, :description, :width, :height])
+    |> Ecto.Changeset.unique_constraint(:sha1, name: :images_sha1_index)
   end
 
   @doc """
   Uploads the given image to S3
   and adds the image information to the database.
   """
   def insert(image) do
-    %Image{}
-    |> changeset(image)
-    |> Repo.insert!()
+    {:ok,
+     %Image{}
+     |> changeset(image)
+     |> Repo.insert!()}
   end
 
+  def check_sha1(sha1) do
+    App.Repo.get_by(App.Image, %{sha1: sha1})
+    |> case do
+      nil ->
+        :ok
+
+      _ ->
+        nil
+    end
+  end
+
+  # def check_sha(image) do
+  #   {:ok,
+  #    App.Repo.get_by(App.Image, %{sha1: image.sha1})
+  #    |> case do
+  #      nil ->
+  #        image
+
+  #      _ ->
+  #        nil
+  #    end}
+  # end
+
   @doc """
   Uploads the given image to S3.
   Returns {:ok, response} if the upload is successful.

lib/app/knn_index.ex

@@ -0,0 +1,42 @@
+defmodule App.KnnIndex do
+  use GenServer
+
+  @indexes "indexes.bin"
+  @upload_dir Application.app_dir(:app, ["priv", "static", "uploads"])
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, {}, name: __MODULE__)
+  end
+
+  def init(_) do
+    File.mkdir_p!(@upload_dir)
+
+    path = get_index_path()
+    space = :cosine
+    dim = 384
+    max_elements = 200
+
+    require Logger
+
+    case File.exists?(path) do
+      false ->
+        {:ok, _index} = App.HnswlibIndex.maybe_load_index_from_db(space, dim, max_elements)
+
+      true ->
+        Logger.info("Existing Index")
+        {:ok, _index} = HNSWLib.Index.load_index(space, dim, path)
+    end
+  end
+
+  def get_index_path do
+    Path.join([@upload_dir, @indexes])
+  end
+
+  def load_index do
+    GenServer.call(__MODULE__, :load)
+  end
+
+  def handle_call(:load, _from, state) do
+    {:reply, state, state}
+  end
+end

lib/app/text_embedding.ex

@@ -0,0 +1,38 @@
+defmodule App.TextEmbedding do
+  use GenServer
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, {}, name: __MODULE__)
+  end
+
+  def init(_) do
+    model_info = nil
+    tokenizer = nil
+    {:ok, {model_info, tokenizer}, {:continue, :load}}
+  end
+
+  def handle_continue(:load, {_, _}) do
+    transformer = "sentence-transformers/paraphrase-MiniLM-L6-v2"
+
+    {:ok, %{model: _model, params: _params} = model_info} =
+      Bumblebee.load_model({:hf, transformer})
+
+    {:ok, tokenizer} =
+      Bumblebee.load_tokenizer({:hf, transformer})
+
+    require Logger
+    Logger.info("Transformer loaded")
+    {:noreply, {model_info, tokenizer}}
+  end
+
+  # called in Liveview `mount`
+  def serve() do
+    GenServer.call(__MODULE__, :serve)
+  end
+
+  def handle_call(:serve, _from, {model_info, tokenizer} = state) do
+    embedding_serving = Bumblebee.Text.TextEmbedding.text_embedding(model_info, tokenizer)
+
+    {:reply, embedding_serving, state}
+  end
+end