diff --git a/apps/nextjs/animations/Comp.json b/apps/nextjs/animations/Comp.json new file mode 100644 index 000000000..626ee8150 --- /dev/null +++ b/apps/nextjs/animations/Comp.json @@ -0,0 +1 @@ +{"nm":"Comp 1","ddd":0,"h":300,"w":300,"meta":{"g":"LottieFiles AE 1.0.0","tc":"#00000000"},"layers":[{"ty":0,"nm":"Pre-comp 1","sr":1,"st":0,"op":211,"ip":0,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[150,150,0],"ix":1},"s":{"a":0,"k":[41,41,100],"ix":6},"sk":{"a":0,"k":0},"p":{"a":0,"k":[150,150,0],"ix":2},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"w":300,"h":300,"refId":"comp_0","ind":1}],"v":"4.8.0","fr":60,"op":41,"ip":9,"assets":[{"nm":"","id":"comp_0","layers":[{"ty":4,"nm":"ball4","sr":1,"st":25,"op":71,"ip":40,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":223,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":25},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":40},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":55},{"s":[129.5],"t":70}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":1},{"ty":4,"nm":"ball 4","sr":1,"st":20,"op":66,"ip":35,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":150,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":20},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":35},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":50},{"s":[129.5],"t":65}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":2},{"ty":4,"nm":"ball 3","sr":1,"st":15,"op":61,"ip":30,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":77,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":15},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":30},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":45},{"s":[129.5],"t":60}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":3},{"ty":4,"nm":"ball3","sr":1,"st":-6,"op":40,"ip":9,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":223,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":-6},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":9},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":24},{"s":[129.5],"t":39}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":4},{"ty":4,"nm":"ball 2","sr":1,"st":-11,"op":35,"ip":4,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":150,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":-11},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":4},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":19},{"s":[129.5],"t":34}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":5},{"ty":4,"nm":"ball 1","sr":1,"st":-16,"op":30,"ip":-1,"hd":false,"ddd":0,"bm":0,"hasMask":false,"ao":0,"ks":{"a":{"a":0,"k":[0,0,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6},"sk":{"a":0,"k":0},"p":{"s":true,"x":{"a":0,"k":77,"ix":3},"y":{"a":1,"k":[{"o":{"x":0.534,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":-16},{"o":{"x":0.514,"y":0},"i":{"x":0.424,"y":1},"s":[129.5],"t":-1},{"o":{"x":0.054,"y":0},"i":{"x":0.465,"y":1},"s":[169.5],"t":14},{"s":[129.5],"t":29}],"ix":4},"z":{"a":0,"k":0}},"r":{"a":0,"k":0,"ix":10},"sa":{"a":0,"k":0},"o":{"a":0,"k":100,"ix":11}},"ef":[],"shapes":[{"ty":"gr","bm":0,"hd":false,"mn":"ADBE Vector Group","nm":"Group 1","ix":1,"cix":2,"np":2,"it":[{"ty":"sh","bm":0,"hd":false,"mn":"ADBE Vector Shape - Group","nm":"Path 1","ix":1,"d":1,"ks":{"a":0,"k":{"c":true,"i":[[0,-14.359],[14.359,0],[0,14.359],[-14.359,0]],"o":[[0,14.359],[-14.359,0],[0,-14.359],[14.359,0]],"v":[[26,0],[0,26],[-26,0],[0,-26]]},"ix":2}},{"ty":"fl","bm":0,"hd":false,"mn":"ADBE Vector Graphic - Fill","nm":"Fill 1","c":{"a":0,"k":[0.9176,0.4824,0.0431],"ix":4},"r":1,"o":{"a":0,"k":100,"ix":5}},{"ty":"tr","a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"sk":{"a":0,"k":0,"ix":4},"p":{"a":0,"k":[0,0],"ix":2},"r":{"a":0,"k":0,"ix":6},"sa":{"a":0,"k":0,"ix":5},"o":{"a":0,"k":100,"ix":7}}]}],"ind":6}]}]} \ No newline at end of file diff --git a/apps/nextjs/animations/loading.json b/apps/nextjs/animations/loading.json new file mode 100644 index 000000000..f174dbafe --- /dev/null +++ b/apps/nextjs/animations/loading.json @@ -0,0 +1 @@ +{"v":"5.5.5","fr":30,"ip":0,"op":90,"w":300,"h":150,"nm":"Loading-2","ddd":0,"assets":[],"layers":[{"ddd":0,"ind":1,"ty":4,"nm":"icon 2","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":39,"s":[-90]},{"t":79,"s":[270]}],"ix":10},"p":{"a":0,"k":[150,75,0],"ix":2},"a":{"a":0,"k":[53,53,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-15.464],[15.464,0],[0,15.464],[-15.464,0]],"o":[[0,15.464],[-15.464,0],[0,-15.464],[15.464,0]],"v":[[28,0],[0,28],[-28,0],[0,-28]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.9607843137254902,0.6509803921568628,0.13725490196078433,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":10,"ix":5},"lc":2,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[53,53],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":51,"s":[0]},{"t":79,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":39,"s":[0]},{"t":64,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":39,"op":79,"st":39,"bm":0},{"ddd":0,"ind":2,"ty":4,"nm":"icon","sr":1,"ks":{"o":{"a":0,"k":100,"ix":11},"r":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":0,"s":[-90]},{"t":40,"s":[270]}],"ix":10},"p":{"a":0,"k":[150,75,0],"ix":2},"a":{"a":0,"k":[53,53,0],"ix":1},"s":{"a":0,"k":[100,100,100],"ix":6}},"ao":0,"shapes":[{"ty":"gr","it":[{"ind":0,"ty":"sh","ix":1,"ks":{"a":0,"k":{"i":[[0,-15.464],[15.464,0],[0,15.464],[-15.464,0]],"o":[[0,15.464],[-15.464,0],[0,-15.464],[15.464,0]],"v":[[28,0],[0,28],[-28,0],[0,-28]],"c":true},"ix":2},"nm":"Path 1","mn":"ADBE Vector Shape - Group","hd":false},{"ty":"st","c":{"a":0,"k":[0.9607843137254902,0.6509803921568628,0.13725490196078433,1],"ix":3},"o":{"a":0,"k":100,"ix":4},"w":{"a":0,"k":10,"ix":5},"lc":2,"lj":1,"ml":10,"bm":0,"nm":"Stroke 1","mn":"ADBE Vector Graphic - Stroke","hd":false},{"ty":"tr","p":{"a":0,"k":[53,53],"ix":2},"a":{"a":0,"k":[0,0],"ix":1},"s":{"a":0,"k":[100,100],"ix":3},"r":{"a":0,"k":0,"ix":6},"o":{"a":0,"k":100,"ix":7},"sk":{"a":0,"k":0,"ix":4},"sa":{"a":0,"k":0,"ix":5},"nm":"Transform"}],"nm":"Group 1","np":2,"cix":2,"bm":0,"ix":1,"mn":"ADBE Vector Group","hd":false},{"ty":"tm","s":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":12,"s":[0]},{"t":40,"s":[100]}],"ix":1},"e":{"a":1,"k":[{"i":{"x":[0.667],"y":[1]},"o":{"x":[0.333],"y":[0]},"t":0,"s":[0]},{"t":25,"s":[100]}],"ix":2},"o":{"a":0,"k":0,"ix":3},"m":1,"ix":2,"nm":"Trim Paths 1","mn":"ADBE Vector Filter - Trim","hd":false}],"ip":0,"op":40,"st":0,"bm":0}],"markers":[]} \ No newline at end of file diff --git a/apps/nextjs/app/api/voice2measurements/route.ts b/apps/nextjs/app/api/voice2measurements/route.ts new file mode 100644 index 000000000..bbccdbd46 --- /dev/null +++ b/apps/nextjs/app/api/voice2measurements/route.ts @@ -0,0 +1,22 @@ +import { NextRequest, NextResponse } from 'next/server'; +import {handleError} from "@/lib/errorHandler"; +import { haveConversation } from '@/lib/conversation2measurements'; +import { text2measurements } from '@/lib/text2measurements'; + +export async function POST(request: NextRequest) { + let { statement, utcDateTime, timeZoneOffset, text, previousStatements, previousQuestions } = await request.json(); + + if(!statement){statement = text;} + //TODO: replace previous statements properly + try { + //const measurements = await text2measurements(statement, utcDateTime, timeZoneOffset); + //haveConversation + //input: statement, utcDateTime, timeZoneOffset, previousStatements) + //output: questionForUser, measurements + const { questionForUser } = await haveConversation(statement, utcDateTime, timeZoneOffset, previousStatements, previousQuestions); + const measurements = text2measurements(statement, utcDateTime, timeZoneOffset); + return NextResponse.json({ success: true, question:questionForUser }); + } catch (error) { + return handleError(error, "voice2measurements") + } +} diff --git a/apps/nextjs/app/dashboard/newvoice2measurements/page.tsx b/apps/nextjs/app/dashboard/newvoice2measurements/page.tsx new file mode 100644 index 000000000..ab622f368 --- /dev/null +++ b/apps/nextjs/app/dashboard/newvoice2measurements/page.tsx @@ -0,0 +1,297 @@ +'use client'; +import Button from "@/components/buttons/Button"; +import { SendQuery } from "@/core/services/GPTService"; +import useSharedStore from "@/core/store/SharedStore"; +import { SuggestionItem } from "@/core/types/types"; +import { useCallback, useEffect, useState } from "react"; +import "regenerator-runtime/runtime"; +import { MdSettingsVoice,MdOutlineSettingsVoice } from "react-icons/md"; +import SpeechRecognition, { useSpeechRecognition } from 'react-speech-recognition'; +import Lottie from "lottie-react"; +import loadingAnimation from "@/animations/loading.json"; +import List from "@/components/list/List"; +import { useSpeechSynthesis } from 'react-speech-kit'; +import { getTimeZoneOffset, getUtcDateTime } from '@/lib/dateTimeWithTimezone'; + +import {Shell} from "@/components/layout/shell"; +import {DashboardHeader} from "@/components/pages/dashboard/dashboard-header"; +import {Icons} from "@/components/icons"; + + + +function isNumber(numStr: string) { + + return !isNaN(Number(numStr)); +} + +export default function Home() { + + const [ifStarted, setIfStarted] = useState(false); + const [ifThinking, setIfThinking] = useState(false); + + const { speak, voices } = useSpeechSynthesis(); + + const { + transcript, + listening, + resetTranscript + } = useSpeechRecognition(); + + type Message = { + type: 'user' | 'response' | 'loading'; + text: string; + }; + const conversation: Message[] = useSharedStore((state) => state.conversation); + const response: string = useSharedStore((state) => state.response); + const suggestions: SuggestionItem[] = useSharedStore((state) => state.suggestions); + + const [input, setInput] = useState(''); + const [messages, setMessages] = useState([]); + const [isLoading, setIsLoading] = useState(false); + + let previousStatements: string = useSharedStore((state) => state.previousStatements); + let previousQuestions: string = useSharedStore((state) => state.previousQuestions); + + const sendMessage = async (input: string) => { + if (input.trim()) { + const response = await fetch('/api/voice2measurements', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + text: input, + timeZoneOffset: getTimeZoneOffset(), + utcDateTime: getUtcDateTime(), + previousStatements: previousStatements, + previousQuestions: previousQuestions + }), + }); + const data = await response.json(); + + //const parsedData = JSON.parse(data.question); + + // Extract the question from the array + //const question = parsedData.questions[0].question; + + console.log(data.question); + //console.log("PRINTING DATA",data.question); + speak({ text: data.question, voice: voices[0] }); + let ps = previousStatements + if(!previousStatements){ + ps = ""; + } + ps += input; + ps += "\n"; + useSharedStore.setState({ previousStatements: ps}); + + let pq = previousQuestions + if(!previousQuestions){ + pq = ""; + } + pq += data.question; + pq += "\n"; + useSharedStore.setState({ previousQuestions: pq}); + + setIfThinking(false); + } + }; + + async function startConversation () { + + if (!ifStarted) { + //sendConversationToGPT(conversation,true); + setIfStarted(true); + } + } + + function formatPassage(passage: string) { + + const processed = passage.replaceAll("\n","\n
"); + return processed; + } + + function processResponse(gptResponse: string): SuggestionItem[] { + + // to get the list of suggestions returned (they are all separated by an \n) + const suggestions: string[] = gptResponse.split("\n"); + const processed: SuggestionItem[] = []; + + suggestions.forEach((eachSuggestion: string, index: number) => { + + // we do not want the 1st index + // because it is not a suggestion + if (index !== 0) { + + // here we are getting the 1st character of this string + const firstChar: string = eachSuggestion[0]; + + // we then check if the first character is a number + if (isNumber(firstChar)) { + + // if it is a suggestion, we get the place's name with the number in front of it + let splitToken: string = " - "; + if (eachSuggestion.indexOf(":") !== -1) { splitToken = ":"; } + const splitSuggestion: string[] = eachSuggestion.split(splitToken); + const placeNameWithNumber: string = splitSuggestion[0]; + + // we can get the place name from above. + // but it could be followed by a . or a ) + let token: string = "."; + if (placeNameWithNumber.indexOf(")") !== -1) { token = ")"; } + + const splitPlaceName: string[] = placeNameWithNumber.split(token); + const placeName: string = splitPlaceName[1]; + + const placeDesc: string = splitSuggestion[1]; + + const newSuggestion: SuggestionItem = { + id: crypto.randomUUID(), + placeName: placeName, + description: placeDesc + }; + + processed.push(newSuggestion); + } + } + }); + + return processed; + } + + const sendConversationToGPT = useCallback(async (updatedConversations: Message[], ifStart: boolean) => { + + const gptResponse: string = await SendQuery(updatedConversations); + let displayText: string = gptResponse; + + displayText = formatPassage(gptResponse); + + useSharedStore.setState({ response: displayText }); + + // using text-to-speech to have the app speak out the response from GPT + speak({ text: gptResponse, voice: voices[0] }); + + setIfThinking(false); + + if (!ifStart) { + + const suggestions = processResponse(gptResponse); + useSharedStore.setState({ suggestions: suggestions }); + } + + },[speak, voices]); + + function startListening() { + + SpeechRecognition.startListening({ continuous: false }); + } + + function stopListening() { + + SpeechRecognition.stopListening(); + } + + const continueConversation = useCallback( + (text: string) => + { + + setIfThinking(true); + + useSharedStore.setState({ response: ""}); + + // create new GPT message based on the current transcript + const newMessage: Message = { + role: "user", + content: text + }; + + resetTranscript(); + + // updating the conversation in the store + const messages = conversation; + messages.push(newMessage); + useSharedStore.setState({ conversation: messages}); + + // and send the updated conversation to the GPT API service + //sendConversationToGPT(messages, false); //TODO: double check + sendMessage(text); + },[conversation, resetTranscript, sendConversationToGPT]); + + useEffect(() => { + + const timeOut = setTimeout(() => { + if (transcript.length > 0) { + continueConversation(transcript); + } + }, 2500); + + return (()=>{ + clearTimeout(timeOut); + }) + + }, [continueConversation, transcript]); + + return ( +
+
+ { + !ifThinking && +
+

Voice 2 Measurements

+
+ } + +
+ + { + !ifStarted ? <> +
+
+ Use your voice to converse in this GPT-powered app to get lifestyle and health suggestions! +
+
+
+
+ + : + <> + { + ifThinking && +
+ +
+ } + { + (!ifThinking && suggestions.length > 0) ? + <> + + + : +
+ } + { + (!ifThinking && suggestions.length == 0) && + + } + + } +
+ ) +} diff --git a/apps/nextjs/app/dashboard/voice2measurements/page.tsx b/apps/nextjs/app/dashboard/voice2measurements/page.tsx new file mode 100644 index 000000000..2ca452ba4 --- /dev/null +++ b/apps/nextjs/app/dashboard/voice2measurements/page.tsx @@ -0,0 +1,88 @@ +"use client" + +import React, { useState } from 'react'; +import {Shell} from "@/components/layout/shell"; +import {DashboardHeader} from "@/components/pages/dashboard/dashboard-header"; +import {Icons} from "@/components/icons"; +import {getTimeZoneOffset, getUtcDateTime} from "@/lib/dateTimeWithTimezone"; + +// Define a type for the message objects +type Message = { + type: 'user' | 'response' | 'loading'; + text: string; +}; + +const App: React.FC = () => { + const [input, setInput] = useState(''); + const [messages, setMessages] = useState([]); + const [isLoading, setIsLoading] = useState(false); + + const sendMessage = async () => { + if (input.trim()) { + + setMessages([...messages, { type: 'user', text: input }, { type: 'loading', text: 'Loading...' }]); + setIsLoading(true); + const response = await fetch('/api/voice2measurements', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + text: input, + timeZoneOffset: getTimeZoneOffset(), + utcDateTime: getUtcDateTime(), + }), + }); + const data = await response.json(); + setMessages(prevMessages => prevMessages + .filter(msg => msg.type !== 'loading') + .concat({ type: 'response', text: JSON.stringify(data) })); + setIsLoading(false); + setInput(''); + } + }; + + const handleInputChange = (e: React.ChangeEvent) => { + setInput(e.target.value); + }; + + const handleKeyPress = (e: React.KeyboardEvent) => { + if (e.key === 'Enter') { + sendMessage(); + } + }; + + return ( + + +
+
+ {messages.map((msg, index) => ( +
+ {msg.type === 'loading' ? (
+
) : msg.text} +
+ ))} +
+
+ + +
+
+ + ); +} + +export default App; diff --git a/apps/nextjs/components/buttons/Button.tsx b/apps/nextjs/components/buttons/Button.tsx new file mode 100644 index 000000000..60d94fee0 --- /dev/null +++ b/apps/nextjs/components/buttons/Button.tsx @@ -0,0 +1,28 @@ +'use client'; + +import { useState } from "react"; + +export interface ButtonProps { + + title: string; + callback: () => void; +} + +export default function Button (props: ButtonProps) { + + const [title, setTitle] = useState(props.title); + + const onSubmit = (e: any) => { + + e.preventDefault(); + props.callback(); + } + + return ( +
+ +
+ ) +} \ No newline at end of file diff --git a/apps/nextjs/components/list/List.tsx b/apps/nextjs/components/list/List.tsx new file mode 100644 index 000000000..3d203a55e --- /dev/null +++ b/apps/nextjs/components/list/List.tsx @@ -0,0 +1,24 @@ +import { SuggestionItem } from "@/core/types/types"; +import ListItem from "./ListItem"; + +export interface ListProps { + + listItems: SuggestionItem[]; +} + +export default function List(props: ListProps) { + + return (
+ { + props.listItems.map((each: SuggestionItem, index:number) => { + return + }) + } +
); +} \ No newline at end of file diff --git a/apps/nextjs/components/list/ListItem.tsx b/apps/nextjs/components/list/ListItem.tsx new file mode 100644 index 000000000..1f76c2fec --- /dev/null +++ b/apps/nextjs/components/list/ListItem.tsx @@ -0,0 +1,50 @@ +import Routes from "@/core/routes/routes"; +import { GetImage } from "@/core/services/ImageService"; +import template from "just-template"; +import { useEffect, useState } from "react"; + +export interface ListItemProps { + + id: number; + listImageURL: string; + listTitle: string; + listDesc: string; +} + +export default function ListItem(props: ListItemProps) { + + const [imageLink, setImageLink] = useState(""); + + useEffect(() => { + + retrieveImage(); + + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + async function retrieveImage() { + + const url = template(Routes.GetImage, { + GOOGLE_CUSTOM_SEARCH_KEY: process.env.GOOGLE_CUSTOM_SEARCH_KEY || "", + GOOGLE_SEARCH_ENGINE_ID: process.env.GOOGLE_SEARCH_ENGINE_ID || "", + SEARCH_TERM: props.listTitle + }) + const linkToImage: string = await GetImage(url); + setImageLink(linkToImage); + } + + return (
+
+ {/* eslint-disable-next-line @next/next/no-img-element */} + {props.listTitle} +
+
+
+ {(props.id + 1 + ")")}{props.listTitle} +
+
+ { props.listDesc } +
+
+
); +} \ No newline at end of file diff --git a/apps/nextjs/config/links.ts b/apps/nextjs/config/links.ts index 8806bd87a..0079912be 100644 --- a/apps/nextjs/config/links.ts +++ b/apps/nextjs/config/links.ts @@ -40,6 +40,16 @@ export const dashboardLinks: Navigation = { href: "/dashboard/text2measurements", icon: "write", }, + { + title: "Voice 2 Measurements", + href: "/dashboard/voice2measurements", + icon: "write", + }, + { + title: "New page : Voice 2 Measurements", + href: "/dashboard/newvoice2measurements", + icon: "write", + }, { title: "Profile", href: "/dashboard/settings", diff --git a/apps/nextjs/lib/conversation2measurements.ts b/apps/nextjs/lib/conversation2measurements.ts index 51f642012..b2e98d4dc 100644 --- a/apps/nextjs/lib/conversation2measurements.ts +++ b/apps/nextjs/lib/conversation2measurements.ts @@ -1,11 +1,12 @@ import {Measurement} from "@/types/models/Measurement"; -import {textCompletion} from "@/lib/llm"; +import {textCompletion4oMini, textCompletion} from "@/lib/llm"; import {convertToLocalDateTime, getUtcDateTime} from "@/lib/dateTimeWithTimezone"; import {text2measurements} from "@/lib/text2measurements"; // IMPORTANT! Set the runtime to edge export const runtime = 'edge'; +//TODO: add previousQuestions as argument as well export function conversation2MeasurementsPrompt(statement: string, utcDateTime: string | null | undefined, timeZoneOffset: number | null | undefined, @@ -49,7 +50,9 @@ Your responses should be in JSON format and have 2 properties called data and me ${previousStatements ? `The following are the previous statements: ${previousStatements}` : ''} + // Use the current local datetime ${localDateTime} to determine startDateLocal. If specified, also determine startTimeLocal, endDateLocal, and endTimeLocal or just leave them null.\`\`\` +If there is no time or date in user's answer, output local datetime. The following is a user request: """ ${statement} @@ -78,7 +81,7 @@ export async function conversation2measurements(statement: string, return measurements; } -export async function getNextQuestion(currentStatement: string, previousStatements: string | null | undefined): Promise { +export async function getNextQuestion(currentStatement: string, previousStatements: string | null | undefined, previousQuestions:string | null | undefined): Promise { let promptText = ` You are a robot designed to collect diet, treatment, and symptom data from the user. @@ -90,26 +93,29 @@ Immediately begin asking the user the following questions Also, after asking each question and getting a response, check if there's anything else the user want to add to the first question response. For instance, after getting a response to "What did you eat today?", your next question should be, "Did you eat anything else today?". If they respond in the negative, move on to the next question. +Make sure to only return exactly one followup question as plain text. + Here is the current user statement: ${currentStatement} Here are the previous statements in the conversation: ${previousStatements} + These are the questions already asked, so don't ask these questions again: ${previousQuestions} `; - return await textCompletion(promptText, "text"); + return await textCompletion4oMini(promptText, "text"); } export async function haveConversation(statement: string, utcDateTime: string, timeZoneOffset: number, - previousStatements: string | null | undefined): Promise<{ + previousStatements: string | null | undefined, + previousQuestions: string | null | undefined): Promise<{ questionForUser: string; - measurements: Measurement[] }> { - let questionForUser = await getNextQuestion(statement, previousStatements); - const measurements = await text2measurements(statement, utcDateTime, timeZoneOffset); + let questionForUser = await getNextQuestion(statement, previousStatements, previousQuestions); + console.log(questionForUser); + return { questionForUser, - measurements } } diff --git a/apps/nextjs/lib/dateTimeWithTimezone.ts b/apps/nextjs/lib/dateTimeWithTimezone.ts index 919ab5533..83ee95f06 100644 --- a/apps/nextjs/lib/dateTimeWithTimezone.ts +++ b/apps/nextjs/lib/dateTimeWithTimezone.ts @@ -22,21 +22,10 @@ export function getUtcDateTimeWithTimezone(): string { */ export function convertToUTC(localDateTime: string, timezoneOffsetInMinutes: number): string { // Convert the localDateTime string to a Date object + console.log("local date time being used "+ localDateTime); const localDate = new Date(localDateTime); - - // Get the local time zone offset in milliseconds - const localOffset = localDate.getTimezoneOffset() * 60 * 1000; - - // Calculate the UTC time in milliseconds - const utcTime = localDate.getTime() + localOffset; - - // Adjust for the provided timezone offset - const adjustedTime = utcTime + (timezoneOffsetInMinutes * 60 * 1000); - - // Create a new Date object using the adjusted UTC time - const utcDate = new Date(adjustedTime); - - return utcDate.toUTCString(); + console.log("date being used "+ localDate); + return new Date(localDate.getTime() + timezoneOffsetInMinutes * 60000).toISOString(); } export function throwErrorIfDateInFuture(utcDateTime: string) { @@ -79,6 +68,6 @@ export function convertToLocalDateTime( utcDateTime: string | number | Date, timeZoneOffsetInMinutes: number): string { const utcDate = new Date(utcDateTime); - const localDate = new Date(utcDate.getTime() + timeZoneOffsetInMinutes * 60 * 1000); + const localDate = new Date(utcDate.getTime() - timeZoneOffsetInMinutes * 60000); return localDate.toISOString(); } diff --git a/apps/nextjs/lib/llm.ts b/apps/nextjs/lib/llm.ts index 78d262e53..1a47f04a8 100644 --- a/apps/nextjs/lib/llm.ts +++ b/apps/nextjs/lib/llm.ts @@ -33,10 +33,39 @@ export async function textCompletion(promptText: string, returnType: "text" | "j return response.choices[0].message.content; } +export async function textCompletion4oMini(promptText: string, returnType: "text" | "json_object"): Promise { + + // Ask OpenAI for a streaming chat completion given the prompt + const response = await openai.chat.completions.create({ + model: 'gpt-4o-mini', + stream: false, + //max_tokens: 150, + messages: [ + { + role: "system", + content: `You are a helpful assistant that has conversations with the user` + }, + { + role: "user", // user = the dFDA app + content: promptText + }, + + ], + response_format: { type: returnType }, + }); + + if(!response.choices[0].message.content) { + throw new Error('No content in response'); + } + + return response.choices[0].message.content; +} + export async function getDateTimeFromStatementInUserTimezone(statement: string, utcDateTime: string, timeZoneOffset: number): Promise { - const localDateTime = convertToLocalDateTime(utcDateTime, timeZoneOffset); + const localDateTime = convertToLocalDateTime(utcDateTime, timeZoneOffset); //utcDateTime; + console.log("--------local datetime " + localDateTime); const promptText = ` estimate the date and time of the user statement based on the user's current date and time ${localDateTime} and the following user statement: diff --git a/apps/nextjs/package.json b/apps/nextjs/package.json index 93582daee..89697f566 100644 --- a/apps/nextjs/package.json +++ b/apps/nextjs/package.json @@ -69,6 +69,8 @@ "gray-matter": "^4.0.3", "highcharts": "^11.4.1", "highcharts-react-official": "^3.2.1", + "just-template": "^2.2.0", + "lottie-react": "^2.4.0", "lucide-react": "^0.259.0", "next": "14.1.0", "next-auth": "^4.24.5", @@ -91,7 +93,10 @@ "react-hot-toast": "^2.4.1", "react-icons": "^5.0.1", "react-markdown": "^8.0.7", + "react-speech-kit": "^3.0.1", + "react-speech-recognition": "^3.10.0", "recharts": "^2.12.0", + "regenerator-runtime": "^0.13.11", "remark": "^15.0.1", "remark-html": "^16.0.1", "replicate": "^0.12.3", @@ -115,6 +120,8 @@ "@types/jest": "^29.5.12", "@types/react": "18.2.55", "@types/react-calendar-heatmap": "^1.6.7", + "@types/react-speech-recognition": "^3.9.1", + "@types/regenerator-runtime": "^0.13.1", "eslint": "8.56.0", "eslint-config-next": "14.1.0", "jest": "^29.7.0",