Merge pull request #43 from ndrean/audio-transcription

#18 1) adding audio transcription

.github/workflows/ci.yml

@@ -27,16 +27,22 @@ jobs:
     - name: Set up Elixir
       uses: erlef/setup-beam@v1
       with:
-        elixir-version: '1.14.2' # Define the elixir version [required]
-        otp-version: '25.1.2' # Define the OTP version [required]
+        elixir-version: '1.16.0' # Define the elixir version [required]
+        otp-version: '26.2.1' # Define the OTP version [required]
+
+    - name: Installing ffmpeg
+      uses: FedericoCarboni/setup-ffmpeg@v3
+
     - name: Restore dependencies cache
       uses: actions/cache@v2
       with:
         path: deps
         key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
         restore-keys: ${{ runner.os }}-mix-
+
     - name: Install dependencies
       run: mix deps.get
+
     - name: Run Tests
       run: mix coveralls.json
       env:

assets/js/app.js

@@ -20,10 +20,11 @@ import "phoenix_html";
 // Establish Phoenix Socket and LiveView configuration.
 import { Socket } from "phoenix";
 import { LiveSocket } from "phoenix_live_view";
-import Toastify from 'toastify-js'
+import Toastify from "toastify-js";
+import Audio from "./micro.js";
 import topbar from "../vendor/topbar";
 
-let Hooks = {};
+let Hooks = { Audio };
 
 // Hook to track inactivity
 Hooks.ActivityTracker = {
@@ -36,7 +37,7 @@ Hooks.ActivityTracker = {
     let processHasBeenSent = false;
 
     // We use the `mounted()` context to push the event. This is used in the `setTimeout` function below.
-    let ctx = this
+    let ctx = this;
 
     // Function to reset the timer
     function resetInactivityTimer() {
@@ -66,22 +67,27 @@ Hooks.ActivityTracker = {
 // Hook to show message toast
 Hooks.MessageToaster = {
   mounted() {
-    this.handleEvent('toast', (payload) => {
+    this.handleEvent("toast", (payload) => {
       Toastify({
         text: payload.message,
         gravity: "bottom",
         position: "right",
         style: {
           background: "linear-gradient(to right, #f27474, #ed87b5)",
         },
-        duration: 4000
-        }).showToast();        
-    })
-  }
-}
+        duration: 4000,
+      }).showToast();
+    });
+  },
+};
 
-let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content");
-let liveSocket = new LiveSocket("/live", Socket, { hooks: Hooks, params: { _csrf_token: csrfToken } });
+let csrfToken = document
+  .querySelector("meta[name='csrf-token']")
+  .getAttribute("content");
+let liveSocket = new LiveSocket("/live", Socket, {
+  hooks: Hooks,
+  params: { _csrf_token: csrfToken },
+});
 
 // Show progress bar on live navigation and form submits
 topbar.config({ barColors: { 0: "#29d" }, shadowColor: "rgba(0, 0, 0, .3)" });

assets/js/micro.js

@@ -0,0 +1,60 @@
+export default {
+  mounted() {
+    let mediaRecorder,
+      audioChunks = [];
+
+    // Defining the elements and styles to be used during recording
+    // and shown on the HTML.
+    const recordButton = document.getElementById("record"),
+      audioElement = document.getElementById("audio"),
+      text = document.getElementById("text"),
+      blue = ["bg-blue-500", "hover:bg-blue-700"],
+      pulseGreen = ["bg-green-500", "hover:bg-green-700", "animate-pulse"];
+
+
+    _this = this;
+
+    // Adding event listener for "click" event
+    recordButton.addEventListener("click", () => {
+
+      // Check if it's recording.
+      // If it is, we stop the record and update the elements.
+      if (mediaRecorder && mediaRecorder.state === "recording") {
+        mediaRecorder.stop();
+        text.textContent = "Record";
+      } 
+
+      // Otherwise, it means the user wants to start recording.
+      else {
+        navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
+
+          // Instantiate MediaRecorder
+          mediaRecorder = new MediaRecorder(stream);
+          mediaRecorder.start();
+
+          // And update the elements
+          recordButton.classList.remove(...blue);
+          recordButton.classList.add(...pulseGreen);
+          text.textContent = "Stop";
+
+          // Add "dataavailable" event handler
+          mediaRecorder.addEventListener("dataavailable", (event) => {
+            audioChunks.push(event.data);
+          });
+
+          // Add "stop" event handler for when the recording stops.
+          mediaRecorder.addEventListener("stop", () => {
+            const audioBlob = new Blob(audioChunks);
+            // the source of the audio element
+            audioElement.src = URL.createObjectURL(audioBlob);
+
+            _this.upload("speech", [audioBlob]);
+            audioChunks = [];
+            recordButton.classList.remove(...pulseGreen);
+            recordButton.classList.add(...blue);
+          });
+        });
+      }
+    });
+  },
+};

lib/app/application.ex

@@ -7,6 +7,7 @@ defmodule App.Application do
 
   @impl true
   def start(_type, _args) do
+
     App.Models.verify_and_download_models()
 
     children = [
@@ -16,16 +17,29 @@ defmodule App.Application do
       App.Repo,
       # Start the PubSub system
       {Phoenix.PubSub, name: App.PubSub},
+      # Nx serving for the embedding
+      # App.TextEmbedding,
+
+      # Nx serving for Speech-to-Text
+      {Nx.Serving,
+      serving:
+        if Application.get_env(:app, :use_test_models) == true do
+          App.Models.audio_serving_test()
+        else
+          App.Models.audio_serving()
+        end,
+      name: Whisper},
       # Nx serving for image classifier
       {Nx.Serving,
        serving:
          if Application.get_env(:app, :use_test_models) == true do
-           App.Models.serving_test()
+           App.Models.caption_serving_test()
          else
-           App.Models.serving()
+           App.Models.caption_serving()
          end,
        name: ImageClassifier},
       {GenMagic.Server, name: :gen_magic},
+
       # Adding a supervisor
       {Task.Supervisor, name: App.TaskSupervisor},
       # Start the Endpoint (http/https)