Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
output is json but is not parsed, only returns text
- Loading branch information
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| import { useState, useEffect, useRef } from 'react'; | ||
| import { vertexAI } from '../../services/firebase.js'; | ||
| import { getGenerativeModel } from 'firebase/vertexai'; | ||
| import Collapsible from '../Collapsible'; | ||
|
|
||
| const AIChatPanel = ({ scene }) => { | ||
| const [messages, setMessages] = useState([]); | ||
| const [input, setInput] = useState(''); | ||
| const [isLoading, setIsLoading] = useState(false); | ||
| const chatContainerRef = useRef(null); | ||
| const modelRef = useRef(null); | ||
|
|
||
| useEffect(() => { | ||
| console.log('AIChatPanel mounted'); | ||
| console.log('Scene available:', !!scene); | ||
| const initializeAI = async () => { | ||
| try { | ||
| console.log('Initializing Vertex AI'); | ||
| modelRef.current = getGenerativeModel(vertexAI, { | ||
| model: 'gemini-1.5-flash' | ||
| }); | ||
| console.log('Vertex AI initialized successfully'); | ||
| } catch (error) { | ||
| console.error('Error initializing Vertex AI:', error); | ||
| } | ||
| }; | ||
|
|
||
| initializeAI(); | ||
| }, []); | ||
|
|
||
| const handleSendMessage = async () => { | ||
| if (!input.trim() || !modelRef.current) return; | ||
|
|
||
| setIsLoading(true); | ||
| const userMessage = { role: 'user', content: input }; | ||
| setMessages((prev) => [...prev, userMessage]); | ||
| setInput(''); | ||
|
|
||
| try { | ||
| // Get the current scene state | ||
| const sceneState = scene.current | ||
| ? scene.current.getAttribute('managed-street') | ||
| : null; | ||
|
|
||
| const prompt = ` | ||
| Context: You are a 3D street scene assistant. The current scene has the following state: | ||
| ${JSON.stringify(sceneState, null, 2)} | ||
| User request: ${input} | ||
| Please provide suggestions for modifying the scene. Format your response as JSON when suggesting specific changes. | ||
| `; | ||
|
|
||
| const result = await modelRef.current.generateContent(prompt); | ||
| const response = await result.response; | ||
| const aiMessage = { role: 'assistant', content: response.text() }; | ||
|
|
||
| setMessages((prev) => [...prev, aiMessage]); | ||
| } catch (error) { | ||
| console.error('Error generating response:', error); | ||
| setMessages((prev) => [ | ||
| ...prev, | ||
| { | ||
| role: 'assistant', | ||
| content: 'Sorry, I encountered an error. Please try again.' | ||
| } | ||
| ]); | ||
| } finally { | ||
| setIsLoading(false); | ||
| } | ||
| }; | ||
|
|
||
| useEffect(() => { | ||
| if (chatContainerRef.current) { | ||
| chatContainerRef.current.scrollTop = | ||
| chatContainerRef.current.scrollHeight; | ||
| } | ||
| }, [messages]); | ||
| return ( | ||
| <div className="chat-panel-container"> | ||
| <Collapsible defaultCollapsed={false}> | ||
| <div>AI Scene Assistant</div> | ||
| <div className="chat-panel"> | ||
| <div ref={chatContainerRef} className="chat-messages"> | ||
| {messages.map((message, index) => ( | ||
| <div key={index} className={`chat-message ${message.role}`}> | ||
| {message.content} | ||
| </div> | ||
| ))} | ||
| {isLoading && <div className="loading-indicator">Thinking...</div>} | ||
| </div> | ||
| <div className="chat-input"> | ||
| <input | ||
| type="text" | ||
| value={input} | ||
| onChange={(e) => setInput(e.target.value)} | ||
| onKeyPress={(e) => e.key === 'Enter' && handleSendMessage()} | ||
| placeholder="Ask about the scene..." | ||
| /> | ||
| <button onClick={handleSendMessage} disabled={isLoading}> | ||
| Send | ||
| </button> | ||
| </div> | ||
| </div> | ||
| </Collapsible> | ||
| </div> | ||
| ); | ||
| }; | ||
|
|
||
| export default AIChatPanel; | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| import { createContext, useContext, useState, useCallback } from 'react'; | ||
| import AIChatService from '../services/aiChatService'; | ||
|
|
||
| const AIChatContext = createContext(null); | ||
|
|
||
| export const AIChatProvider = ({ children, firebaseApp }) => { | ||
| const [chatService] = useState(() => new AIChatService(firebaseApp)); | ||
| const [messages, setMessages] = useState([]); | ||
| const [isProcessing, setIsProcessing] = useState(false); | ||
|
|
||
| const sendMessage = useCallback( | ||
| async (message, sceneState) => { | ||
| setIsProcessing(true); | ||
| try { | ||
| const response = await chatService.generateResponse( | ||
| message, | ||
| sceneState | ||
| ); | ||
| const parsedResponse = chatService.parseResponse(response); | ||
|
|
||
| setMessages((prev) => [ | ||
| ...prev, | ||
| { role: 'user', content: message }, | ||
| { role: 'assistant', content: response, parsed: parsedResponse } | ||
| ]); | ||
|
|
||
| return parsedResponse; | ||
| } catch (error) { | ||
| console.error('Error in chat:', error); | ||
| setMessages((prev) => [ | ||
| ...prev, | ||
| { role: 'user', content: message }, | ||
| { | ||
| role: 'assistant', | ||
| content: 'Sorry, I encountered an error. Please try again.' | ||
| } | ||
| ]); | ||
| throw error; | ||
| } finally { | ||
| setIsProcessing(false); | ||
| } | ||
| }, | ||
| [chatService] | ||
| ); | ||
|
|
||
| const clearMessages = useCallback(() => { | ||
| setMessages([]); | ||
| }, []); | ||
|
|
||
| return ( | ||
| <AIChatContext.Provider | ||
| value={{ | ||
| messages, | ||
| isProcessing, | ||
| sendMessage, | ||
| clearMessages | ||
| }} | ||
| > | ||
| {children} | ||
| </AIChatContext.Provider> | ||
| ); | ||
| }; | ||
|
|
||
| export const useAIChat = () => { | ||
| const context = useContext(AIChatContext); | ||
| if (!context) { | ||
| throw new Error('useAIChat must be used within an AIChatProvider'); | ||
| } | ||
| return context; | ||
| }; | ||
|
|
||
| export default AIChatContext; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| import { getGenerativeModel } from 'firebase/vertexai'; | ||
| import { vertexAI } from '../services/firebase.js'; | ||
|
|
||
| class AIChatService { | ||
| constructor(firebaseApp) { | ||
| this.model = null; | ||
| this.initPromise = this.initialize(firebaseApp); | ||
| } | ||
|
|
||
| async initialize(firebaseApp) { | ||
| try { | ||
| this.model = getGenerativeModel(vertexAI, { | ||
| model: 'gemini-1.5-flash' | ||
| }); | ||
| } catch (error) { | ||
| console.error('Error initializing AI Chat Service:', error); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| async generateResponse(prompt, sceneState) { | ||
| if (!this.model) { | ||
| throw new Error('AI model not initialized'); | ||
| } | ||
|
|
||
| const formattedPrompt = this.formatPrompt(prompt, sceneState); | ||
| const result = await this.model.generateContent(formattedPrompt); | ||
| return result.response.text(); | ||
| } | ||
|
|
||
| formatPrompt(userInput, sceneState) { | ||
| return ` | ||
| Context: You are a 3D street scene assistant for the 3DStreet application. | ||
| The current scene has the following state: | ||
| ${JSON.stringify(sceneState, null, 2)} | ||
| User request: ${userInput} | ||
| Please analyze the request and provide one of the following: | ||
| 1. If the user is asking to modify the scene, provide specific JSON-formatted changes | ||
| 2. If the user is asking about the scene, provide a natural language explanation | ||
| 3. If the user needs help, provide relevant guidance about the 3DStreet editor | ||
| For scene modifications, use this JSON format: | ||
| { | ||
| "action": "modify_scene", | ||
| "changes": [ | ||
| { | ||
| "type": "add"|"remove"|"update", | ||
| "element": "<element_type>", | ||
| "properties": {} | ||
| } | ||
| ] | ||
| } | ||
| `; | ||
| } | ||
|
|
||
| parseResponse(response) { | ||
| try { | ||
| // Check if the response is JSON | ||
| const parsed = JSON.parse(response); | ||
| if (parsed.action === 'modify_scene') { | ||
| return { | ||
| type: 'scene_modification', | ||
| data: parsed.changes | ||
| }; | ||
| } | ||
| return { | ||
| type: 'text', | ||
| data: response | ||
| }; | ||
| } catch (e) { | ||
| // If not JSON, treat as regular text response | ||
| return { | ||
| type: 'text', | ||
| data: response | ||
| }; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| export default AIChatService; |
Oops, something went wrong.