From 25ada3c4120997a3241ff16fed49ac87c564e86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Wed, 6 Mar 2024 01:43:46 +0000 Subject: [PATCH 01/10] chore: Organizing the view. --- assets/js/app.js | 20 ++ lib/app_web/live/page_live.html.heex | 389 +++++++++++++++------------ 2 files changed, 230 insertions(+), 179 deletions(-) diff --git a/assets/js/app.js b/assets/js/app.js index 9334602..2e3ad14 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -89,6 +89,26 @@ let liveSocket = new LiveSocket("/live", Socket, { params: { _csrf_token: csrfToken }, }); +// Toggles to show upload or semantic search containers +// JavaScript to toggle visibility and styles +document.getElementById('upload_option').addEventListener('click', function() { + document.getElementById('upload_container').style.display = 'block'; // Show Element 1 + document.getElementById('search_container').style.display = 'none'; // Hide Element 2 + document.getElementById('upload_option').classList.replace('bg-gray-200', 'bg-blue-500'); + document.getElementById('upload_option').classList.replace('text-black', 'text-white'); + document.getElementById('search_option').classList.replace('bg-blue-500', 'bg-gray-200'); + document.getElementById('search_option').classList.replace('text-white', 'text-black'); +}); + +document.getElementById('search_option').addEventListener('click', function() { + document.getElementById('upload_container').style.display = 'none'; // Hide Element 1 + document.getElementById('search_container').style.display = 'block'; // Show Element 2 + document.getElementById('upload_option').classList.replace('bg-blue-500', 'bg-gray-200'); + document.getElementById('upload_option').classList.replace('text-white', 'text-black'); + document.getElementById('search_option').classList.replace('bg-gray-200', 'bg-blue-500'); + document.getElementById('search_option').classList.replace('text-black', 'text-white'); +}); + // Show progress bar on live navigation and form submits topbar.config({ barColors: { 0: "#29d" }, shadowColor: "rgba(0, 0, 0, .3)" }); window.addEventListener("phx:page-loading-start", (_info) => topbar.show(300)); diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index a54ecfa..a807e8f 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -4,109 +4,136 @@
-

+ + +

- - 🔥 LiveView - - + - - 🐝 Bumblebee - - -

-

- Caption your image! -

-

- Upload your own image (up to 5MB) and perform image captioning with - Elixir + > + 🔥 LiveView - ! -

-

- Powered with + + - HuggingFace🤗 + > + 🐝 Bumblebee - transformer models, - you can run this project locally and perform machine learning tasks with a handful lines of code. +

-
-
-
- -
-
-
- - <%= if @image_preview_base64 do %> -
-
-
- -
-
-

- Semantic search using an audio -

-
-

- Please record a phrase. You can listen to your audio. It will be transcripted automatically into a text and appear below. The semantic search for matching images will then run automatically and the found image appear below. -

-
-
- <.live_file_input upload={@uploads.speech} class="hidden" /> - -
-
-

- -

-
-
- Transcription: - - <%= if @transcription do %> - <%= @transcription %> - <% else %> - Waiting for audio input. - <% end %> -
-
-
-
-
- found_image -
-
-
-
- - <%!-- <%= if @display_list? do %> --%> -
-

- Examples -

-
-
- <%= for example_img <- @example_list do %> - - <%= if example_img.predicting? == true do %> -
- spinner - Loading... -
- <% else %> -
- -

- <%= example_img.label %> + +
+ Description: + + + <%= if @label do %> + <%= @label %> + <% else %> + Waiting for image input. + <% end %> +
+ + + <%!-- <%= if @display_list? do %> --%> +
+

+ Examples

+
+
+ <%= for example_img <- @example_list do %> + + <%= if example_img.predicting? == true do %> +
+ spinner + Loading... +
+ <% else %> +
+ +

+ <%= example_img.label %> +

+
+ <% end %> + <% end %> +
+
- <% end %> - <% end %> + <%!-- <% end %> --%> +

+ + +
+

+ Semantic search using an audio +

+
+

+ Please record a phrase. You can listen to your audio. It will be transcripted automatically into a text and appear below. The semantic search for matching images will then run automatically and the found image appear below. +

+
+
+ <.live_file_input upload={@uploads.speech} class="hidden" /> + +
+ +

+ +

+ +
+ Transcription: + + <%= if @transcription do %> + <%= @transcription %> + <% else %> + Waiting for audio input. + <% end %> +
+ +
+
+
+ found_image +
+
+
+ +
- <%!-- <% end %> --%>
From 1566c8251f107d43dc26e7bb322919187a296043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Wed, 6 Mar 2024 02:42:47 +0000 Subject: [PATCH 02/10] ui: Fixing loading prediction label and spinner. --- README.md | 18 +++++----- lib/app_web/live/page_live.ex | 8 ++--- lib/app_web/live/page_live.html.heex | 54 +++++++++++++++++----------- 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 0d6109b..1eb17b0 100644 --- a/README.md +++ b/README.md @@ -3425,12 +3425,14 @@ so this part of your code will shrink to: ```elixir - - -<%= if @label do %> - <%= @label %> +<%= if @upload_running? do %> + <% else %> - Waiting for image input. + <%= if @label do %> + <%= @label %> + <% else %> + Waiting for image input. + <% end %> <% end %> ``` @@ -5341,9 +5343,9 @@ def handle_progress(:image_list, entry, socket) when entry.done? do )} # Otherwise, if there was an error uploading the image, we log the error and show it to the person. - %{error: errors} -> - Logger.warning("⚠️ Error uploading image. #{inspect(errors)}") - {:noreply, push_event(socket, "toast", %{message: "Image couldn't be uploaded to S3"})} + %{error: error} -> + Logger.warning("⚠️ Error uploading image. #{inspect(error)}") + {:noreply, push_event(socket, "toast", %{message: "Image couldn't be uploaded to S3.\n#{error}"})} end end ``` diff --git a/lib/app_web/live/page_live.ex b/lib/app_web/live/page_live.ex index 6f93222..85f9a40 100644 --- a/lib/app_web/live/page_live.ex +++ b/lib/app_web/live/page_live.ex @@ -199,10 +199,10 @@ defmodule AppWeb.PageLive do )} # Otherwise, if there was an error uploading the image, we log the error and show it to the person. - %{error: errors} -> - Logger.warning("⚠️ Error uploading image. #{inspect(errors)}") - {:noreply, push_event(socket, "toast", %{message: "Image couldn't be uploaded to S3"})} - end + %{error: error} -> + Logger.warning("⚠️ Error uploading image. #{inspect(error)}") + {:noreply, push_event(socket, "toast", %{message: "Image couldn't be uploaded to S3.\n#{error}"})} + end end # This function is called whenever a user records their voice. diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index a807e8f..9fbbaf4 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -50,22 +50,31 @@
-

+

Caption your image!

-

- Upload your own image (up to 5MB) and perform image captioning with - - Elixir - - ! -

-

+

+
+ +
+
+

+ Upload your own image (up to 5MB) and perform image captioning with + + Elixir + + ! +

+
+
+

Powered with -

+
-
- Description: +
+ Description: - - <%= if @label do %> - <%= @label %> + <%= if @upload_running? do %> + <% else %> - Waiting for image input. + <%= if @label do %> + <%= @label %> + <% else %> + Waiting for image input. + <% end %> <% end %>
From df6007f22bb4ce52ff23047ffd17a3fcc93ba5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Wed, 6 Mar 2024 03:08:55 +0000 Subject: [PATCH 03/10] ui: Refactoring examples. --- lib/app_web/live/page_live.html.heex | 44 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index 9fbbaf4..fe0b540 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -186,36 +186,36 @@
- <%!-- <%= if @display_list? do %> --%> -
-

+ <%= if @display_list? do %> +
+

Examples

-
+
<%= for example_img <- @example_list do %> - - <%= if example_img.predicting? == true do %> -
- spinner - Loading... -
- <% else %> -
- -

- <%= example_img.label %> -

-
- <% end %> + + <%= if example_img.predicting? == true do %> +
+ spinner + Loading... +
+ <% else %> +
+ +

+ <%= example_img.label %> +

+
+ <% end %> <% end %>
- <%!-- <% end %> --%> + <% end %>
From f24e42923303ebc792062f1be64a1d9322ec80b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Wed, 6 Mar 2024 03:41:27 +0000 Subject: [PATCH 04/10] ui: Refactoring audio transcription part. --- README.md | 13 +++---- lib/app_web/live/page_live.ex | 8 ++--- lib/app_web/live/page_live.html.heex | 52 ++++++++++++++++++++-------- 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1eb17b0..1e1ea8c 100644 --- a/README.md +++ b/README.md @@ -5949,13 +5949,14 @@ and update it as so: class="flex mt-2 space-x-1.5 items-center font-bold text-gray-900 text-xl" > Transcription: - - <%= if @transcription do %> - <%= @transcription %> + <%= if @audio_running? do %> + <% else %> - Waiting for audio input. + <%= if @transcription do %> + <%= @transcription %> + <% else %> + Waiting for audio input. + <% end %> <% end %>


diff --git a/lib/app_web/live/page_live.ex b/lib/app_web/live/page_live.ex index 85f9a40..67b2e39 100644 --- a/lib/app_web/live/page_live.ex +++ b/lib/app_web/live/page_live.ex @@ -244,7 +244,7 @@ defmodule AppWeb.PageLive do def handle_progress(_, _, socket), do: {:noreply, socket} @doc """ - Called in `handle_progress` to Handle the upload to the bucket and returns the format `{:ok, map}` or {:postpone, message}` + Called in `handle_progress` to handle the upload to the bucket and returns the format `{:ok, map}` or {:postpone, message}` as demanded by the signature of callback function used `consume_uploaded_entry` """ def handle_upload({:ok, %{path: path, tensor: tensor, image_info: image_info} = map}) @@ -275,10 +275,8 @@ defmodule AppWeb.PageLive do end @doc """ - Every time an `async task` is created, this function is called. - We destructure the output of the task and update the socket assigns. - - This function handles both the image that is uploaded by the user and the example images. + This function is invoked after the async task for embedding model is completed. + It retrieves the embedding, normalizes it and knn-searches the embedding in our database. """ @impl true def handle_info({ref, %{chunks: [%{text: text}]} = _result}, %{assigns: assigns} = socket) diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index fe0b540..ad4109e 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -55,8 +55,8 @@

-
@@ -223,12 +223,29 @@

Semantic search using an audio

-
-

- Please record a phrase. You can listen to your audio. It will be transcripted automatically into a text and appear below. The semantic search for matching images will then run automatically and the found image appear below. + +

+
+ + + +
+
+

+ Record a phrase or some key words. + We'll detect them and semantically search it in our database of images! +

+
+
+

+ After recording your audio, you can listen to it. It will be transcripted automatically into text and appear below. +

+

+ Semantic search will automatically kick in and the resulting image will show below.

-
-
+ + + <.live_file_input upload={@uploads.speech} class="hidden" />
- + +
+ <.live_file_input upload={@uploads.speech} class="hidden" /> + +
- -

- -

+ +

+ +

- -
- Transcription: - <%= if @audio_running? do %> - - <% else %> - <%= if @transcription do %> - <%= @transcription %> + +
+ Transcription: + <%= if @audio_running? do %> + <% else %> - Waiting for audio input. + <%= if @transcription do %> + <%= @transcription %> + <% else %> + Waiting for audio input. + <% end %> <% end %> - <% end %> -
+
- -
-
-
- found_image + +
+
+
+ found_image +
+ <%= @audio_search_result.description %>
- <%= @audio_search_result.description %> -
+
+
From 2c69348f75539df4b5c115f3e221cd864c402322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Fri, 8 Mar 2024 19:30:31 +0000 Subject: [PATCH 06/10] fix: Toggling styles work and overriding on larger screens. --- assets/css/app.css | 7 ++++++ assets/js/app.js | 37 +++++++++++++++++++--------- lib/app_web/live/page_live.html.heex | 10 +++----- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/assets/css/app.css b/assets/css/app.css index b0e8da4..9c81d72 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -17,3 +17,10 @@ body { height: 100vh; width: 100vw; } + +/* Override the container displays to both be shown on larger screens */ +@media (min-width: 1024px) { + #upload_container, #search_container { + display: block !important; /* Override any inline styles */ + } +} \ No newline at end of file diff --git a/assets/js/app.js b/assets/js/app.js index 2e3ad14..a93f0c4 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -92,23 +92,36 @@ let liveSocket = new LiveSocket("/live", Socket, { // Toggles to show upload or semantic search containers // JavaScript to toggle visibility and styles document.getElementById('upload_option').addEventListener('click', function() { - document.getElementById('upload_container').style.display = 'block'; // Show Element 1 - document.getElementById('search_container').style.display = 'none'; // Hide Element 2 - document.getElementById('upload_option').classList.replace('bg-gray-200', 'bg-blue-500'); - document.getElementById('upload_option').classList.replace('text-black', 'text-white'); - document.getElementById('search_option').classList.replace('bg-blue-500', 'bg-gray-200'); - document.getElementById('search_option').classList.replace('text-white', 'text-black'); + document.getElementById('upload_container').style.display = 'block'; + document.getElementById('search_container').style.display = 'none'; + + document.getElementById('upload_option').classList.replace('bg-white', 'bg-blue-500'); + document.getElementById('upload_option').classList.replace('text-gray-900', 'text-white'); + document.getElementById('upload_option').classList.replace('hover:bg-gray-50', 'hover:bg-blue-600'); + document.getElementById('upload_option').getElementsByTagName('svg')[0].classList.replace('text-gray-400', 'text-white'); + + document.getElementById('search_option').classList.replace('bg-blue-500', 'bg-white'); + document.getElementById('search_option').classList.replace('text-white', 'text-gray-900'); + document.getElementById('search_option').classList.replace('hover:bg-blue-600', 'hover:bg-gray-50'); + document.getElementById('search_option').getElementsByTagName('svg')[0].classList.replace('text-white', 'text-gray-400'); }); document.getElementById('search_option').addEventListener('click', function() { - document.getElementById('upload_container').style.display = 'none'; // Hide Element 1 - document.getElementById('search_container').style.display = 'block'; // Show Element 2 - document.getElementById('upload_option').classList.replace('bg-blue-500', 'bg-gray-200'); - document.getElementById('upload_option').classList.replace('text-white', 'text-black'); - document.getElementById('search_option').classList.replace('bg-gray-200', 'bg-blue-500'); - document.getElementById('search_option').classList.replace('text-black', 'text-white'); + document.getElementById('upload_container').style.display = 'none'; + document.getElementById('search_container').style.display = 'block'; + + document.getElementById('search_option').classList.replace('bg-white', 'bg-blue-500'); + document.getElementById('search_option').classList.replace('text-gray-900', 'text-white'); + document.getElementById('search_option').classList.replace('hover:bg-gray-50', 'hover:bg-blue-600'); + document.getElementById('search_option').getElementsByTagName('svg')[0].classList.replace('text-gray-400', 'text-white'); + + document.getElementById('upload_option').classList.replace('bg-blue-500', 'bg-white'); + document.getElementById('upload_option').classList.replace('text-white', 'text-gray-900'); + document.getElementById('upload_option').classList.replace('hover:bg-blue-600', 'hover:bg-gray-50'); + document.getElementById('upload_option').getElementsByTagName('svg')[0].classList.replace('text-white', 'text-gray-400'); }); + // Show progress bar on live navigation and form submits topbar.config({ barColors: { 0: "#29d" }, shadowColor: "rgba(0, 0, 0, .3)" }); window.addEventListener("phx:page-loading-start", (_info) => topbar.show(300)); diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index 3d5d8c0..a62cfb3 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -27,7 +27,7 @@

-
+
- -
Element 1 Content
- -
-
+

Caption your image!

@@ -221,7 +217,7 @@
-
+

...or search it!

From e256d2bcdaf81083850cc6dee89c8f03cb031fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lui=CC=81s=20Arteiro?= Date: Fri, 8 Mar 2024 19:32:23 +0000 Subject: [PATCH 07/10] fix: Toggling now properly works responsively. --- lib/app_web/live/page_live.html.heex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/app_web/live/page_live.html.heex b/lib/app_web/live/page_live.html.heex index a62cfb3..4c01839 100644 --- a/lib/app_web/live/page_live.html.heex +++ b/lib/app_web/live/page_live.html.heex @@ -29,8 +29,8 @@
-
-
+ @@ -3172,13 +3183,18 @@ and all of the code inside the [`_comparison`](./_comparison/) folder. +
## 🔍 Semantic search -> Imagine a person wants to see an image that was uploaded -> under a certain theme. -> One way to solve this problem is to perform a **_full-text_ search query** on specific words among these image captions. +> In this section, we will focus on implementing a +> **_full-text_ search query** through the captions of the images. +> At the end of this, +> you'll be able to transcribe audio, +> create embeddings from the audio transcription +> and search the closest related image. +