feat: sentiment

packages/backend/src/evaluators/pii.ts

@@ -58,7 +58,6 @@ export async function evaluate(run: Run, params: Params) {
     error: errorPIIs,
   }
 
-  await sleep(1000)
 
   // TODO: zod for languages, SHOLUD NOT INGEST IN DB IF NOT CORRECT FORMAT
   return PIIs

packages/backend/src/evaluators/sentiment.ts

@@ -1,22 +1,69 @@
+import { callML } from "@/src/utils/ml"
 import { Run } from "shared"
-import { lastMsg } from "../checks"
-import openai from "@/src/utils/openai"
-import lunary from "lunary"
 
-export async function evaluate(run: Run) {
-  const input = lastMsg(run.input)
+// TOOD: refacto this with all the other parsing function already in use
+function parseMessages(messages: unknown) {
+  if (!messages) {
+    return [""]
+  }
+  if (typeof messages === "string" && messages.length) {
+    return [messages]
+  }
+
+  if (messages === "__NOT_INGESTED__") {
+    return [""]
+  }
+
+  if (Array.isArray(messages)) {
+    let contentArray = []
+    for (const message of messages) {
+      let content = message.content || message.text
+      if (typeof content === "string" && content.length) {
+        contentArray.push(content)
+      } else {
+        contentArray.push(JSON.stringify(message))
+      }
+    }
+    return contentArray
+  }
+
+  if (typeof messages === "object") {
+    return [JSON.stringify(messages)]
+  }
 
-  const template = await lunary.renderTemplate("sentiment", {
-    input,
-  })
+  return [""]
+}
+
+export async function evaluate(run: Run) {
+  console.log("SENTIMENT")
+  const input = parseMessages(run.input)
+  const output = parseMessages(run.output)
+  const error = parseMessages(run.error)
 
-  const res = await openai.chat.completions.create(template)
+  const [inputSentiment, outputSentiment] = await Promise.all([
+    analyzeSentiment(input),
+    analyzeSentiment(output),
+  ])
 
-  const output = res.choices[0]?.message?.content
+  const sentiments = {
+    input: inputSentiment,
+    output: outputSentiment,
+    error: error.map(e => 0) 
+  }
 
-  if (!output) ""
 
-  const result = parseFloat(output.toLowerCase().trim())
+  // TODO: zod for languages, SHOLUD NOT INGEST IN DB IF NOT CORRECT FORMAT
+  return sentiments 
+}
 
-  return result
+// TODO: type
+async function analyzeSentiment(
+  texts: string[],
+): Promise<any> {
+  try {
+    return callML("sentiment", { texts })
+  } catch (error) {
+    console.error(error)
+    console.log(texts)
+  }
 }

packages/backend/src/jobs/realtime-evaluators.ts

@@ -6,7 +6,7 @@ import { RealtimeEvaluator } from "shared/evaluators"
 import { sleep } from "../utils/misc"
 import evaluators from "../evaluators"
 
-const RUNS_BATCH_SIZE = 30
+const RUNS_BATCH_SIZE = 30 
 
 async function runEvaluator(evaluator: RealtimeEvaluator, run: Run) {
   try {
@@ -72,6 +72,7 @@ async function evaluatorJob() {
       evaluator e 
     where
       mode = 'realtime'
+      and id = '9d8f756f-5ae9-42e0-b1a0-dc45efc8b0b2'
     order by 
       random()
   `
@@ -85,7 +86,7 @@ async function evaluatorJob() {
       console.log(
         `Skipping Real-time Evaluator ${evaluator.id} (${i} / ${evaluators.length})`,
       )
-      await sleep(500)
+      // await sleep(500)
       continue
     }