From d75b01836f845c52aaa45b05ad7ef63a4f0603e2 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:01:08 +0530 Subject: [PATCH 01/13] add support for image paste in tasks component --- extensions/spectacular/src/HelloWorldPanel.ts | 8 +- extensions/spectacular/src/backend/joules.ts | 118 +++++++++++------- extensions/spectacular/src/backend/tasks.ts | 14 +-- extensions/spectacular/src/types.ts | 14 ++- .../src/components/JouleComponent.tsx | 107 +++++++++------- .../src/components/PreviewImage.tsx | 38 ++++++ .../webview-ui/src/components/Tasks.tsx | 74 ++++++++--- .../webview-ui/src/lib/constants.ts | 5 + .../spectacular/webview-ui/src/lib/errors.ts | 14 +++ .../spectacular/webview-ui/src/lib/utils.ts | 95 ++++++++++++++ .../spectacular/webview-ui/src/types.ts | 7 ++ 11 files changed, 374 insertions(+), 120 deletions(-) create mode 100644 extensions/spectacular/webview-ui/src/components/PreviewImage.tsx create mode 100644 extensions/spectacular/webview-ui/src/lib/constants.ts create mode 100644 extensions/spectacular/webview-ui/src/lib/errors.ts diff --git a/extensions/spectacular/src/HelloWorldPanel.ts b/extensions/spectacular/src/HelloWorldPanel.ts index b9bd173a..6d40f81a 100644 --- a/extensions/spectacular/src/HelloWorldPanel.ts +++ b/extensions/spectacular/src/HelloWorldPanel.ts @@ -13,7 +13,7 @@ import { createNewDehydratedTask } from "./backend/tasks"; import * as config from "./util/config"; import { WebviewNotifier } from "./services/WebviewNotifier"; import { FileManager } from "./services/FileManager"; -import { DehydratedTask, RpcMethod } from "./types"; +import { DehydratedTask, RpcMethod, UserAttachedImage } from "./types"; import { Coder } from "./backend/assistants/coder"; import { Vanilla } from "./backend/assistants/vanilla"; import { GitManager } from "./services/GitManager"; @@ -229,7 +229,7 @@ export class HelloWorldPanel implements WebviewViewProvider { return this.rpcGetLatestCommit(); case "chatMessage": return await this.rpcStartResponse( - params.text, params.taskId + params.text, params.taskId, params.images ); case "createTask": return await this.rpcCreateTask( @@ -463,12 +463,12 @@ export class HelloWorldPanel implements WebviewViewProvider { return vscode.window.activeColorTheme.kind === vscode.ColorThemeKind.Dark ? 'dark' : 'light'; } - private async rpcStartResponse(text: string, taskId: string): Promise { + private async rpcStartResponse(text: string, taskId: string, images?: UserAttachedImage[]): Promise { const task = this._taskManager.getActiveTask(taskId)!; if (!task) { throw new Error(`Tried to chat with an inactive task ${taskId} (active task is ${this._taskManager.getActiveTaskId()})`); } - return task.startResponse(text); + return task.startResponse(text, images); } private async rpcStopResponse(taskId: string): Promise { diff --git a/extensions/spectacular/src/backend/joules.ts b/extensions/spectacular/src/backend/joules.ts index 49798ed2..0af4613d 100644 --- a/extensions/spectacular/src/backend/joules.ts +++ b/extensions/spectacular/src/backend/joules.ts @@ -1,75 +1,97 @@ import { v4 as uuidv4 } from "uuid"; -import { Joule, JouleHuman, JouleBot, BotExecInfo, DiffInfo } from "../types"; +import { Joule, JouleHuman, JouleBot, BotExecInfo, DiffInfo, ClaudeMessage, UserAttachedImage } from "../types"; export function createJouleError(errorMessage: string): JouleBot { - return createJouleBot(errorMessage, { - rawOutput: "[error encountered]", - contextPaths: { + return createJouleBot(errorMessage, { + rawOutput: "[error encountered]", + contextPaths: { meltyRoot: '', paths: [] }, - }); + }); } -export function createJouleHuman(message: string): JouleHuman { - return createJouleHumanWithChanges(message, null, null); +export function createJouleHuman(message: string, images?: UserAttachedImage[]): JouleHuman { + return createJouleHumanWithChanges(message, null, null, images); } export function createJouleHumanWithChanges( - message: string, - commit: string | null, - diffInfo: DiffInfo | null + message: string, + commit: string | null, + diffInfo: DiffInfo | null, + images?: UserAttachedImage[] ): JouleHuman { - const id = uuidv4(); - return { - id, - message, - author: "human", - state: "complete", - commit, - diffInfo, - }; + const id = uuidv4(); + return { + id, + message, + author: "human", + state: "complete", + commit, + diffInfo, + images + }; } export function createJouleBot( - message: string, - botExecInfo: BotExecInfo, - state: "complete" | "partial" = "complete" + message: string, + botExecInfo: BotExecInfo, + state: "complete" | "partial" = "complete" ): JouleBot { - return createJouleBotWithChanges(message, botExecInfo, null, null, state); + return createJouleBotWithChanges(message, botExecInfo, null, null, state); } export function createJouleBotWithChanges( - message: string, - botExecInfo: BotExecInfo, - commit: string | null, - diffInfo: DiffInfo | null, - state: "complete" | "partial" = "complete" + message: string, + botExecInfo: BotExecInfo, + commit: string | null, + diffInfo: DiffInfo | null, + state: "complete" | "partial" = "complete" ): JouleBot { - const id = uuidv4(); - return { - id, - message, - author: "bot", - state, - commit, - diffInfo: diffInfo, - botExecInfo: botExecInfo, - }; + const id = uuidv4(); + return { + id, + message, + author: "bot", + state, + commit, + diffInfo: diffInfo, + botExecInfo: botExecInfo, + }; } export function updateMessage(joule: Joule, message: string): Joule { - return { ...joule, message }; + return { ...joule, message }; } -export function formatMessageForClaude(joule: Joule): string { - // note that if we show a processed message, we'll need to use `message.length ? message : "..."` - // to ensure no Anthropic API errors - switch (joule.author) { - case "human": - return joule.message; - case "bot": - return (joule as JouleBot).botExecInfo.rawOutput ?? ""; - } +export function formatMessageForClaude(joule: Joule): ClaudeMessage['content'] { + // note that if we show a processed message, we'll need to use `message.length ? message : "..."` + // to ensure no Anthropic API errors + switch (joule.author) { + case "human": + if (joule.images && joule.images.length > 0) { + const content: ClaudeMessage['content'] = []; + for (const img of joule.images) { + content.push({ + type: 'image', + source: { + type: 'base64', + data: img.base64.replace(/^data:image\/\w+;base64,/, ''), + media_type: img.mimeType + } + }); + } + if (joule.message) { + content.push({ + type: 'text', + text: joule.message + }); + } + return content; + } + return joule.message; + case "bot": + return (joule as JouleBot).botExecInfo.rawOutput ?? ""; + } } diff --git a/extensions/spectacular/src/backend/tasks.ts b/extensions/spectacular/src/backend/tasks.ts index cf0b4856..98ae505c 100644 --- a/extensions/spectacular/src/backend/tasks.ts +++ b/extensions/spectacular/src/backend/tasks.ts @@ -1,5 +1,5 @@ import * as vscode from "vscode"; -import { Joule, Conversation, TaskMode, DehydratedTask } from "../types"; +import { Joule, Conversation, TaskMode, DehydratedTask, UserAttachedImage } from "../types"; import * as conversations from "./conversations"; import * as joules from "./joules"; import * as utils from "../util/utils"; @@ -168,7 +168,7 @@ export class Task { /** * Adds a human message (and changes) to the conversation. */ - private async respondHuman(message: string): Promise { + private async respondHuman(message: string, images?: UserAttachedImage[]): Promise { this.conversation = conversations.forceReadyForResponseFrom( this.conversation, "human" @@ -184,10 +184,10 @@ export class Task { webviewNotifier.resetStatusMessage(); newJoule = commitResult !== null - ? joules.createJouleHumanWithChanges(message, commitResult.commit, commitResult.diffInfo) - : joules.createJouleHuman(message); + ? joules.createJouleHumanWithChanges(message, commitResult.commit, commitResult.diffInfo, images) + : joules.createJouleHuman(message, images); } else { - newJoule = joules.createJouleHuman(message); + newJoule = joules.createJouleHuman(message, images); } this.conversation = conversations.addJoule(this.conversation, newJoule); @@ -203,7 +203,7 @@ export class Task { /** * @returns whether launched successfully or not */ - public startResponse(text: string): boolean { + public startResponse(text: string, images?: UserAttachedImage[]): boolean { this._webviewNotifier.updateStatusMessage("Starting up"); if (this.inFlightOperationCancellationTokenSource) { @@ -215,7 +215,7 @@ export class Task { (async () => { // human response - await this.respondHuman(text); + await this.respondHuman(text, images); webviewNotifier.sendNotification("updateTask", { task: this.dehydrateForWire(), }); diff --git a/extensions/spectacular/src/types.ts b/extensions/spectacular/src/types.ts index dd8e1af1..c6e3fbaf 100644 --- a/extensions/spectacular/src/types.ts +++ b/extensions/spectacular/src/types.ts @@ -1,3 +1,5 @@ +import Anthropic from '@anthropic-ai/sdk'; + // implemented by the Task class. this is the UI-facing one // note that datastores.ts has an independent list of properties // that will get loaded from disk @@ -24,11 +26,18 @@ export interface AssistantInfo { description: string; } +export type UserAttachedImage = { + blobUrl: string; + mimeType: "image/jpeg" | "image/png" | "image/gif" | "image/webp"; + base64: string; +}; + export type Joule = { readonly id: string; readonly author: "human" | "bot"; readonly state: "complete" | "partial" | "error"; readonly message: string; + readonly images?: UserAttachedImage[]; readonly commit: string | null; readonly diffInfo: DiffInfo | null; }; @@ -74,7 +83,7 @@ export interface Message { export type ClaudeMessage = { readonly role: "user" | "assistant"; - readonly content: string; + readonly content: Anthropic.Messages.MessageParam['content']; }; export type ClaudeConversation = { @@ -119,4 +128,5 @@ export type RpcMethod = | "createGitRepository" | "createAndOpenWorkspace" | "checkOnboardingComplete" - | "setOnboardingComplete"; + | "setOnboardingComplete" + | "showNotification"; diff --git a/extensions/spectacular/webview-ui/src/components/JouleComponent.tsx b/extensions/spectacular/webview-ui/src/components/JouleComponent.tsx index 4b022ada..d081940e 100644 --- a/extensions/spectacular/webview-ui/src/components/JouleComponent.tsx +++ b/extensions/spectacular/webview-ui/src/components/JouleComponent.tsx @@ -39,51 +39,72 @@ export function JouleComponent({ ); } - const MessageContent = () => ( -
- { + let md = ''; + if (joule.images && joule.images.length > 0) { + for (const img of joule.images) { + md += `![image](${img.base64})\n`; + } + } + md += joule.message; + return ( +
+ v} + components={{ + img({ node, ...props }) { return ( -
- Writing code... - 
,
-											HTMLPreElement
-										>)}
-									>
-										{children}
-
-
- ); - } - return match ? ( -
- {!isPartial && ( - - )} - , + HTMLImageElement + >)} + className="max-w-full" /> -
- ) : ( - - {children} - - ); - }, - }} - > - {joule.message} - -
- ); + ); + }, + code({ node, className, children, ...props }) { + const match = /language-(\w+)/.exec(className || ""); + if (match && match[1] === "codechange") { + return ( +
+ Writing code... + 
,
+												HTMLPreElement
+											>)}
+										>
+											{children}
+
+
+ ); + } + return match ? ( +
+ {!isPartial && ( + + )} + +
+ ) : ( + + {children} + + ); + }, + }} + > + {md} + +
+ ); + } return (
diff --git a/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx b/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx new file mode 100644 index 00000000..233ea703 --- /dev/null +++ b/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx @@ -0,0 +1,38 @@ +import { DialogTrigger } from '@radix-ui/react-dialog'; +import { XIcon } from 'lucide-react'; +import React from 'react'; + +import { Dialog, DialogContent } from './ui/dialog'; +import { cn } from '@/lib/utils'; + +interface PreviewImageProps extends React.ImgHTMLAttributes { + handleRemoveImage: () => void; + src: string; + // when set to true, it will stop the propagation of the escape key event and prevent the parent dialog from closing + stopEscapePropagation?: boolean; +} + +export function PreviewImage({ + handleRemoveImage, + src, + stopEscapePropagation, + ...props +}: PreviewImageProps) { + return ( + + + + preview-image + + + stopEscapePropagation && e.stopPropagation()}> + preview-image + + + ) +} diff --git a/extensions/spectacular/webview-ui/src/components/Tasks.tsx b/extensions/spectacular/webview-ui/src/components/Tasks.tsx index b9d847a0..5ed839e2 100644 --- a/extensions/spectacular/webview-ui/src/components/Tasks.tsx +++ b/extensions/spectacular/webview-ui/src/components/Tasks.tsx @@ -1,6 +1,4 @@ -import * as vscode from "vscode"; - -import { useState, useEffect, useCallback, useRef } from "react"; +import { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { Card, CardHeader, CardTitle, CardContent } from "./ui/card"; import { RpcClient } from "../RpcClient"; import { Button } from "./ui/button"; @@ -19,9 +17,9 @@ import { MouseEvent, KeyboardEvent } from "react"; import Ascii from "./Ascii"; import OnboardingSection from './OnboardingSection'; import "diff2html/bundles/css/diff2html.min.css"; -import { Link, useNavigate } from "react-router-dom"; +import { useNavigate } from "react-router-dom"; import AutoExpandingTextarea from "./AutoExpandingTextarea"; -import { DehydratedTask, TaskMode, AssistantInfo } from "../types"; +import { DehydratedTask, TaskMode, AssistantInfo, UserAttachedImage } from "../types"; import { Select, SelectContent, @@ -34,6 +32,10 @@ import { AddFileButton } from "./AddFileButton"; import * as strings from "@/utilities/strings"; import { FastFilePicker } from "./FastFilePicker"; import { EventManager } from "@/eventManager"; +import { imagePasteHandler, showNotification } from '@/lib/utils'; +import { PreviewImage } from './PreviewImage'; +import { CustomError } from '@/lib/errors'; +import { MAX_IMAGES } from '@/lib/constants'; // Utility function to format the date function formatDate(date: Date): string { @@ -69,6 +71,7 @@ export function Tasks({ const [gitConfigError, setGitConfigError] = useState(null); const navigate = useNavigate(); const [workspaceFilePaths, setWorkspaceFilePaths] = useState([]); + const [images, setImages] = useState([]); const [meltyMindFilePaths, setMeltyMindFilePaths] = useState( initialMeltyMindFiles || [] ); @@ -81,7 +84,6 @@ export function Tasks({ }); const [suggestions, setSuggestions] = useState([]); - useEffect(() => { if (shouldFocus) { textareaRef.current?.focus(); @@ -144,8 +146,8 @@ export function Tasks({ [fetchTasks] ); - const handleSendMessage = useCallback((text: string, taskId: string) => { - rpcClient.run("chatMessage", { text, taskId }); + const handleSendMessage = useCallback((text: string, taskId: string, images?: UserAttachedImage[]) => { + rpcClient.run("chatMessage", { text, taskId, images }); }, []); /* ===================================================== @@ -157,6 +159,7 @@ export function Tasks({ name: taskName.trim(), taskMode: taskMode, files: meltyMindFilePaths, + images, })) as string; console.log(`[Tasks.tsx] created new task ${newTaskId}`); return newTaskId; @@ -174,7 +177,7 @@ export function Tasks({ const didActivate = await rpcClient.run("activateTask", { taskId }); if (didActivate) { - handleSendMessage(message, taskId); + handleSendMessage(message, taskId, images); navigate(`/task/${taskId}`); } } @@ -228,6 +231,11 @@ export function Tasks({ setShouldFocus(true); }, []); + const handleDropImage = useCallback((idx: number) => { + const updatedImages = images.filter((_, i) => i !== idx); + setImages(updatedImages); + }, [images]); + async function activateAndNavigateToTask(taskId: string) { const didActivate = await rpcClient.run("activateTask", { taskId }); if (didActivate) { @@ -301,6 +309,25 @@ export function Tasks({ }, [fetchTasks, fetchFilePaths, checkGitConfig, addSuggestion]); // DO NOT add anything to the initialization dependency array that isn't a constant + const handlePaste = async (event: React.ClipboardEvent) => { + try { + await imagePasteHandler(event, (imgs) => { + const totalImages = images.length + imgs.length; + if (totalImages > MAX_IMAGES) { + throw new CustomError(`You can only attach up to ${MAX_IMAGES} images.`); + } + + setImages(i => [...i, ...imgs]); + }) + } catch (error) { + console.error(error) + let errorMessage = "Failed to paste image. Please try again."; + if (error instanceof CustomError) { + errorMessage = error.getDisplayMessage(); + } + showNotification(errorMessage, 'error'); + } + } return (
@@ -330,13 +357,16 @@ export function Tasks({
: -
-
-

Git config error

-

Oops! Try restarting Melty?

-

{gitConfigError}

+ ( + gitConfigError && +
+
+

Git config error

+

Oops! Try restarting Melty?

+

{gitConfigError}

+
-
+ ) ) : ( <> setMessageText(e.target.value)} + onPaste={handlePaste} onKeyDown={handleKeyDown} className="flex-grow p-3 pr-12 pb-12" ref={textareaRef} @@ -422,7 +453,7 @@ export function Tasks({

Melty can see your codebase structure but not the full content of your files. Too much context confuses language - models. + models. Melty can also see any images you attach/paste. {" "} Only add files that are helpful to the current task. @@ -460,6 +491,17 @@ export function Tasks({ ))}

+
+ {images.map((image, i) => ( +
+ handleDropImage(i)} + stopEscapePropagation + /> +
+ ))} +
{tasks.length === 0 && diff --git a/extensions/spectacular/webview-ui/src/lib/constants.ts b/extensions/spectacular/webview-ui/src/lib/constants.ts new file mode 100644 index 00000000..874705f5 --- /dev/null +++ b/extensions/spectacular/webview-ui/src/lib/constants.ts @@ -0,0 +1,5 @@ +export const MAX_IMAGES = 5; +export const ALLOWED_IMAGE_MIME_TYPES = [ + "image/jpeg", "image/png", "image/gif", "image/webp" +] +export const MAX_IMAGE_SIZE = 5 * 1024 * 1024; // 5MB diff --git a/extensions/spectacular/webview-ui/src/lib/errors.ts b/extensions/spectacular/webview-ui/src/lib/errors.ts new file mode 100644 index 00000000..952b2a92 --- /dev/null +++ b/extensions/spectacular/webview-ui/src/lib/errors.ts @@ -0,0 +1,14 @@ + +export class CustomError extends Error { + private displayMessage: string; + + constructor(message: string, error?: Error) { + super(error?.message || message); + this.displayMessage = message; + this.name = 'CustomError'; + } + + getDisplayMessage(): string { + return this.displayMessage; + } +} diff --git a/extensions/spectacular/webview-ui/src/lib/utils.ts b/extensions/spectacular/webview-ui/src/lib/utils.ts index ac680b30..6da92224 100644 --- a/extensions/spectacular/webview-ui/src/lib/utils.ts +++ b/extensions/spectacular/webview-ui/src/lib/utils.ts @@ -1,6 +1,101 @@ +import { UserAttachedImage } from '@/types'; import { type ClassValue, clsx } from "clsx"; import { twMerge } from "tailwind-merge"; +import { CustomError } from "./errors"; +import { vscode } from '@/utilities/vscode'; +import { ALLOWED_IMAGE_MIME_TYPES, MAX_IMAGE_SIZE } from '@/lib/constants'; export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)); } + +function isImageMimeTypeSupported(mimeType: string) { + const allowedMimeTypes = ALLOWED_IMAGE_MIME_TYPES + + return allowedMimeTypes.includes(mimeType); +} + +export async function imagePasteHandler(event: React.ClipboardEvent, callback: (imgs: UserAttachedImage[]) => void) { + const clipboardData = event.clipboardData; + if (clipboardData && clipboardData.items) { + const images = Array.from(clipboardData.items).filter((item) => item.kind === 'file' && item.type.indexOf("image") !== -1); + try { + const isImageSupported = images.every((image) => isImageMimeTypeSupported(image.type)); + if (!isImageSupported) { + throw new CustomError('Only JPEG, PNG, GIF, and WebP images are supported'); + } + + if (images.length > 0) { + // do not bubble up the event + event.preventDefault(); + const imgs: UserAttachedImage[] = []; + for (const image of images) { + const type = image.type; + const file = image.getAsFile()!; + if (!file) { + throw new CustomError('Cannot read the image file. Please try again'); + } + if (file.size >= MAX_IMAGE_SIZE) { + throw new CustomError('Image size exceeds 5MB. Please paste a smaller image'); + } + const blobUrl = URL.createObjectURL(file); + const base64 = await blobToBase64(blobUrl); + + // this check is required because claude api throws error even if file size is less than 5MB + // but the base64 string is greater than 5MB + if (base64.length >= MAX_IMAGE_SIZE) { + throw new CustomError('Image size exceeds 5MB. Please paste a smaller image'); + } + + imgs.push({ + blobUrl, + mimeType: type, + base64 + }); + } + callback(imgs); + } + } catch (error) { + if (error instanceof CustomError) { + throw error; + } + const wrappedError = new CustomError('Failed to paste image', error as Error); + throw wrappedError; + } + } +} + +export function blobToBase64(blobUrl: string): Promise { + return new Promise((resolve, reject) => { + // Fetch the Blob from the blob URL + fetch(blobUrl) + .then((response) => response.blob()) + .then((blob) => { + const reader = new FileReader(); + + // When FileReader finishes reading, resolve the promise with base64 string + reader.onloadend = () => { + if (typeof reader.result !== "string") { + reject(new Error("Failed to read Blob as base64")); + return; + } + resolve(reader.result); + }; + + // Handle errors in reading + reader.onerror = reject; + + // Read the Blob as a data URL (base64) + reader.readAsDataURL(blob); + }) + .catch(reject); + }); +} + +export function showNotification(message: string, type: 'success' | 'error' = 'success') { + vscode.postMessage({ + type: 'rpc', method: 'showNotification', params: { + message, notificationType: type + } + }); +} diff --git a/extensions/spectacular/webview-ui/src/types.ts b/extensions/spectacular/webview-ui/src/types.ts index dd8e1af1..cca08d64 100644 --- a/extensions/spectacular/webview-ui/src/types.ts +++ b/extensions/spectacular/webview-ui/src/types.ts @@ -31,6 +31,7 @@ export type Joule = { readonly message: string; readonly commit: string | null; readonly diffInfo: DiffInfo | null; + images?: UserAttachedImage[]; }; export type JouleHuman = Joule & { @@ -120,3 +121,9 @@ export type RpcMethod = | "createAndOpenWorkspace" | "checkOnboardingComplete" | "setOnboardingComplete"; + +export type UserAttachedImage = { + blobUrl: string; + mimeType: string; + base64: string; +} From 66dfd94ba87e8696a3e8e8ba0f544d0e73262357 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:02:04 +0530 Subject: [PATCH 02/13] update csp headers to render images in joule component --- extensions/spectacular/src/HelloWorldPanel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/spectacular/src/HelloWorldPanel.ts b/extensions/spectacular/src/HelloWorldPanel.ts index 6d40f81a..77affc79 100644 --- a/extensions/spectacular/src/HelloWorldPanel.ts +++ b/extensions/spectacular/src/HelloWorldPanel.ts @@ -135,7 +135,7 @@ export class HelloWorldPanel implements WebviewViewProvider { - + Melty From 1e444a63c6d0d8ee9e55be0f5591ae55afef2fc5 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:02:17 +0530 Subject: [PATCH 03/13] add showNotofication rpc --- extensions/spectacular/src/HelloWorldPanel.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/extensions/spectacular/src/HelloWorldPanel.ts b/extensions/spectacular/src/HelloWorldPanel.ts index 77affc79..06fb5851 100644 --- a/extensions/spectacular/src/HelloWorldPanel.ts +++ b/extensions/spectacular/src/HelloWorldPanel.ts @@ -7,7 +7,7 @@ import { } from "vscode"; import * as vscode from "vscode"; import { getUri, getNonce } from "./util/utils"; -import { Conversation, TaskMode } from "./types"; +import { TaskMode } from "./types"; import { MeltyExtension } from "./extension"; import { createNewDehydratedTask } from "./backend/tasks"; import * as config from "./util/config"; @@ -20,7 +20,6 @@ import { GitManager } from "./services/GitManager"; import { GitHubManager } from './services/GitHubManager'; import { TaskManager } from './services/TaskManager'; import posthog from "posthog-js"; -import { create } from 'domain'; /** * This class manages the state and behavior of HelloWorld webview panels. @@ -257,6 +256,13 @@ export class HelloWorldPanel implements WebviewViewProvider { return this.rpcCheckOnboardingComplete(); case "setOnboardingComplete": return this.rpcSetOnboardingComplete(); + case "showNotification": + if (params.notificationType === 'error') { + vscode.window.showErrorMessage(params.message); + } else { + vscode.window.showInformationMessage(params.message); + } + return true; default: throw new Error(`Unknown RPC method: ${method}`); } From 8876d49e93079d0def93b5c9e12892af2706a97c Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:02:30 +0530 Subject: [PATCH 04/13] image paste in conversation view component --- .../src/components/ConversationView.tsx | 61 ++++++++++++++++--- 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/extensions/spectacular/webview-ui/src/components/ConversationView.tsx b/extensions/spectacular/webview-ui/src/components/ConversationView.tsx index f9212d25..d1816958 100644 --- a/extensions/spectacular/webview-ui/src/components/ConversationView.tsx +++ b/extensions/spectacular/webview-ui/src/components/ConversationView.tsx @@ -1,5 +1,5 @@ import React, { useState, useEffect, useMemo, useCallback, useRef, memo, useLayoutEffect } from "react"; -import { useParams, Link } from "react-router-dom"; +import { useParams } from "react-router-dom"; import { XIcon, ArrowUp, @@ -15,10 +15,12 @@ import { RpcClient } from "@/RpcClient"; import { JouleComponent } from "./JouleComponent"; import * as strings from "@/utilities/strings"; import { EventManager } from '@/eventManager'; -import { DehydratedTask } from "types"; +import { DehydratedTask, UserAttachedImage } from "types"; import { useNavigate } from "react-router-dom"; - -import * as vscode from "vscode"; +import { imagePasteHandler, showNotification } from '@/lib/utils'; +import { MAX_IMAGES } from '@/lib/constants'; +import { CustomError } from '@/lib/errors'; +import { PreviewImage } from '@/components/PreviewImage'; const MemoizedJouleComponent = memo(JouleComponent); const rpcClient = RpcClient.getInstance(); @@ -27,6 +29,7 @@ export function ConversationView() { const { taskId } = useParams<{ taskId: string }>(); const inputRef = useRef(null); const [meltyFiles, setMeltyFiles] = useState([]); + const [images, setImages] = useState([]); const [workspaceFiles, setWorkspaceFiles] = useState([]); const [pickerOpen, setPickerOpen] = useState(false); const isInitialRender = useRef(true); @@ -79,6 +82,11 @@ export function ConversationView() { setPickerOpen(false); } + const handleDropImage = useCallback((idx: number) => { + const updatedImages = images.filter((_, i) => i !== idx); + setImages(updatedImages); + }, [images]); + useLayoutEffect(() => { if (isInitialRender.current) { inputRef.current?.focus(); @@ -100,14 +108,14 @@ export function ConversationView() { setWorkspaceFiles(workspaceFiles); }, [setMeltyFiles, setWorkspaceFiles]); - function handleSendMessage(text: string, taskId: string) { + function handleSendMessage(text: string, taskId: string, images?: UserAttachedImage[]) { setNonInitialHumanMessageInFlight(true); const result = posthog.capture("chatmessage_sent", { message: text, task_id: taskId, }); console.log("posthog event captured!", result); - rpcClient.run("chatMessage", { text, taskId }); + rpcClient.run("chatMessage", { text, taskId, images }); } async function handleCreatePR() { @@ -234,9 +242,10 @@ export function ConversationView() { event.preventDefault(); const form = event.target as HTMLFormElement; const message = form.message.value; - handleSendMessage(message, taskId!); + handleSendMessage(message, taskId!, images); setMessageText(""); form.reset(); + setImages([]); }; const handleKeyDown = (event: React.KeyboardEvent) => { @@ -245,14 +254,35 @@ export function ConversationView() { if (event.currentTarget && event.currentTarget.value !== undefined) { const form = event.currentTarget.form; if (form) { - handleSendMessage(event.currentTarget.value, taskId!); + handleSendMessage(event.currentTarget.value, taskId!, images); setMessageText(""); + setImages([]); event.currentTarget.value = ""; } } } }; + const handlePaste = async (event: React.ClipboardEvent) => { + try { + await imagePasteHandler(event, (imgs) => { + const totalImages = images.length + imgs.length; + if (totalImages > MAX_IMAGES) { + throw new CustomError(`You can only attach up to ${MAX_IMAGES} images.`); + } + + setImages(i => [...i, ...imgs]); + }) + } catch (error) { + console.error(error) + let errorMessage = "Failed to paste image. Please try again."; + if (error instanceof CustomError) { + errorMessage = error.getDisplayMessage(); + } + showNotification(errorMessage, 'error'); + } + } + const handleBack = useCallback(async () => { await rpcClient.run("deactivateTask", { taskId }); navigate("/"); @@ -364,6 +394,7 @@ export function ConversationView() { onChange={(e) => setMessageText(e.target.value)} onKeyDown={handleKeyDown} autoFocus={true} + onPaste={handlePaste} /> {messageText.trim() !== "" && ( @@ -402,7 +433,7 @@ export function ConversationView() { {meltyFiles.map((file, i) => ( ))} +
+ {images.map((image, i) => ( +
+ handleDropImage(i)} + stopEscapePropagation + /> +
+ ))} +
From 575c3790ebcb0d7efc21975adf0b5ff6c3ac317e Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:02:51 +0530 Subject: [PATCH 05/13] Update vite.config.js to add external dependency 'vscode' --- extensions/spectacular/webview-ui/vite.config.js | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/spectacular/webview-ui/vite.config.js b/extensions/spectacular/webview-ui/vite.config.js index a99ce945..73f48614 100644 --- a/extensions/spectacular/webview-ui/vite.config.js +++ b/extensions/spectacular/webview-ui/vite.config.js @@ -8,6 +8,7 @@ export default defineConfig(() => { build: { outDir: 'build', rollupOptions: { + external: ['vscode'], output: { // Ensure that the output files are named `main.js` and `main.css` entryFileNames: 'assets/main.js', From b36815acb741dff46764d6eb533cf8b238ff92c8 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 12:18:51 +0530 Subject: [PATCH 06/13] fix merging classnames in previewImage --- .../spectacular/webview-ui/src/components/PreviewImage.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx b/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx index 233ea703..c5056da9 100644 --- a/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx +++ b/extensions/spectacular/webview-ui/src/components/PreviewImage.tsx @@ -16,6 +16,7 @@ export function PreviewImage({ handleRemoveImage, src, stopEscapePropagation, + className, ...props }: PreviewImageProps) { return ( @@ -27,7 +28,7 @@ export function PreviewImage({ - preview-image + preview-image stopEscapePropagation && e.stopPropagation()}> From 7c4b2115242f4363efd5551b5187a00b12409973 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 15:07:49 +0530 Subject: [PATCH 07/13] Optimize listInactiveTasks method in TaskManager --- extensions/spectacular/src/services/TaskManager.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/extensions/spectacular/src/services/TaskManager.ts b/extensions/spectacular/src/services/TaskManager.ts index 50a49f08..2ee9740d 100644 --- a/extensions/spectacular/src/services/TaskManager.ts +++ b/extensions/spectacular/src/services/TaskManager.ts @@ -69,10 +69,15 @@ export class TaskManager { * Optimization to try to make Tasks page faster */ public listInactiveTasks(): DehydratedTask[] { - const tasks = Array.from(this.inactiveTasks.values()); - // for (const task of tasks) { - // task.conversation = { joules: [] }; - // } + const inactiveTasks = Array.from(this.inactiveTasks.values()); + const tasks: DehydratedTask[] = []; + // remove conversation data as it is not needed in the list view + for (const task of inactiveTasks) { + tasks.push({ + ...task, + conversation: { joules: [] } + }); + } return tasks; } From 1769654cbfdd70027cf1e67fcbec08c408fe6fc4 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:23:01 +0530 Subject: [PATCH 08/13] move constants file in webview-ui outside the lib/ directory --- .../spectacular/webview-ui/src/components/ConversationView.tsx | 2 +- extensions/spectacular/webview-ui/src/components/Tasks.tsx | 3 +-- extensions/spectacular/webview-ui/src/{lib => }/constants.ts | 0 extensions/spectacular/webview-ui/src/lib/utils.ts | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) rename extensions/spectacular/webview-ui/src/{lib => }/constants.ts (100%) diff --git a/extensions/spectacular/webview-ui/src/components/ConversationView.tsx b/extensions/spectacular/webview-ui/src/components/ConversationView.tsx index d1816958..e7870400 100644 --- a/extensions/spectacular/webview-ui/src/components/ConversationView.tsx +++ b/extensions/spectacular/webview-ui/src/components/ConversationView.tsx @@ -18,7 +18,7 @@ import { EventManager } from '@/eventManager'; import { DehydratedTask, UserAttachedImage } from "types"; import { useNavigate } from "react-router-dom"; import { imagePasteHandler, showNotification } from '@/lib/utils'; -import { MAX_IMAGES } from '@/lib/constants'; +import { MAX_IMAGES } from '@/constants'; import { CustomError } from '@/lib/errors'; import { PreviewImage } from '@/components/PreviewImage'; diff --git a/extensions/spectacular/webview-ui/src/components/Tasks.tsx b/extensions/spectacular/webview-ui/src/components/Tasks.tsx index 5ed839e2..e0500df8 100644 --- a/extensions/spectacular/webview-ui/src/components/Tasks.tsx +++ b/extensions/spectacular/webview-ui/src/components/Tasks.tsx @@ -35,7 +35,7 @@ import { EventManager } from "@/eventManager"; import { imagePasteHandler, showNotification } from '@/lib/utils'; import { PreviewImage } from './PreviewImage'; import { CustomError } from '@/lib/errors'; -import { MAX_IMAGES } from '@/lib/constants'; +import { MAX_IMAGES } from '@/constants'; // Utility function to format the date function formatDate(date: Date): string { @@ -159,7 +159,6 @@ export function Tasks({ name: taskName.trim(), taskMode: taskMode, files: meltyMindFilePaths, - images, })) as string; console.log(`[Tasks.tsx] created new task ${newTaskId}`); return newTaskId; diff --git a/extensions/spectacular/webview-ui/src/lib/constants.ts b/extensions/spectacular/webview-ui/src/constants.ts similarity index 100% rename from extensions/spectacular/webview-ui/src/lib/constants.ts rename to extensions/spectacular/webview-ui/src/constants.ts diff --git a/extensions/spectacular/webview-ui/src/lib/utils.ts b/extensions/spectacular/webview-ui/src/lib/utils.ts index 6da92224..99afd995 100644 --- a/extensions/spectacular/webview-ui/src/lib/utils.ts +++ b/extensions/spectacular/webview-ui/src/lib/utils.ts @@ -3,7 +3,7 @@ import { type ClassValue, clsx } from "clsx"; import { twMerge } from "tailwind-merge"; import { CustomError } from "./errors"; import { vscode } from '@/utilities/vscode'; -import { ALLOWED_IMAGE_MIME_TYPES, MAX_IMAGE_SIZE } from '@/lib/constants'; +import { ALLOWED_IMAGE_MIME_TYPES, MAX_IMAGE_SIZE } from '@/constants'; export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)); From 9e834cd9cfea59d6e271292a3f2c33eb36913bf1 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:23:50 +0530 Subject: [PATCH 09/13] create fs utils based on promises --- extensions/spectacular/src/util/utils.ts | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/extensions/spectacular/src/util/utils.ts b/extensions/spectacular/src/util/utils.ts index a9b9e4dc..545910dd 100644 --- a/extensions/spectacular/src/util/utils.ts +++ b/extensions/spectacular/src/util/utils.ts @@ -2,6 +2,8 @@ import { Uri, Webview } from "vscode"; import { ChangeSet } from "../types"; import * as os from "os"; import * as diff from "diff"; +import path from 'path'; +import fs from 'fs/promises'; /** * Log an arbitrary error object verbosely @@ -112,3 +114,40 @@ export function highlightNonAsciiChars(input: string): string { }) .join(''); } + +export async function pathExists(path: string): Promise { + try { + await fs.access(path); + return true; + } catch { + return false; + } +} + +export async function isDirectory(path: string): Promise { + try { + const stats = await fs.stat(path); + return stats.isDirectory(); + } catch { + return false; + } +} + +export async function isFile(path: string): Promise { + try { + const stats = await fs.stat(path); + return stats.isFile(); + } catch { + return false; + } +} + +// return true if path1 is inside path2 +export function isPathInside(path1: string, path2: string) { + const absolutePath1 = path.resolve(path1); + const absolutePath2 = path.resolve(path2); + + const relative = path.relative(absolutePath2, absolutePath1); + + return !relative.startsWith('..') && !path.isAbsolute(relative); +} From bfa921d4976a1ce4e7e478b95ac2fc25d913c9ca Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:28:47 +0530 Subject: [PATCH 10/13] datastore methods use fs/promises --- .../spectacular/src/backend/datastores.ts | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/extensions/spectacular/src/backend/datastores.ts b/extensions/spectacular/src/backend/datastores.ts index 2f69f94c..8d1f3a89 100644 --- a/extensions/spectacular/src/backend/datastores.ts +++ b/extensions/spectacular/src/backend/datastores.ts @@ -1,8 +1,11 @@ -import * as fs from "fs"; +import fs from "fs/promises"; import * as path from "path"; import * as vscode from "vscode"; -import { resolveTildePath } from "../util/utils"; -import { DehydratedTask } from "types"; +import { isPathInside, resolveTildePath, isDirectory, isFile, pathExists } from "../util/utils"; +import { DehydratedTask, JouleImage, UserAttachedImage } from "types"; +import { v4 as uuidv4 } from "uuid"; +import { LRUCache } from 'lru-cache'; +import { IMAGE_CACHE_TTL_MS, MAX_IMAGES_TO_CACHE } from '../constants'; function getMeltyDir(): string { const config = vscode.workspace.getConfiguration('melty'); @@ -11,17 +14,22 @@ function getMeltyDir(): string { return resolvedPath; } -export function loadTasksFromDisk(): Map { +export async function loadTasksFromDisk(): Promise> { const meltyDir = getMeltyDir(); - if (!fs.existsSync(meltyDir)) { + if (!(await pathExists(meltyDir))) { return new Map(); } - const taskFiles = fs.readdirSync(meltyDir); + const taskFiles = await fs.readdir(meltyDir); const taskMap = new Map(); - for (const file of taskFiles) { + await Promise.all(taskFiles.map(async (file) => { + const filePath = path.join(meltyDir, file); + if ((await isDirectory(filePath))) { + return; + } + const rawTask = JSON.parse( - fs.readFileSync(path.join(meltyDir, file), "utf8") + await fs.readFile(filePath, "utf8") ); const task = Object.fromEntries( @@ -37,18 +45,19 @@ export function loadTasksFromDisk(): Map { ].includes(key)) ) as DehydratedTask; taskMap.set(task.id, task); - } + })); + return taskMap; } export async function dumpTaskToDisk(task: DehydratedTask): Promise { const meltyDir = getMeltyDir(); - if (!fs.existsSync(meltyDir)) { - fs.mkdirSync(meltyDir, { recursive: true }); + if (!(await pathExists(meltyDir))) { + await fs.mkdir(meltyDir, { recursive: true }); } const taskPath = path.join(meltyDir, `${task.id}.json`); - fs.writeFileSync(taskPath, JSON.stringify(task, null, 2)); + await fs.writeFile(taskPath, JSON.stringify(task, null, 2)); } export async function deleteTaskFromDisk(task: DehydratedTask): Promise { From 827da10d987ce73adf40040eac414ef234d3b620 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:30:42 +0530 Subject: [PATCH 11/13] removeEmptyJoules method simplified --- .../spectacular/src/backend/conversations.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/extensions/spectacular/src/backend/conversations.ts b/extensions/spectacular/src/backend/conversations.ts index 3229cfbc..444ad752 100644 --- a/extensions/spectacular/src/backend/conversations.ts +++ b/extensions/spectacular/src/backend/conversations.ts @@ -74,11 +74,16 @@ function removeLeadingBotJoules(conversation: Conversation): Conversation { } function removeEmptyJoules(conversation: Conversation): Conversation { - return { - joules: conversation.joules.filter((joule) => - joules.formatMessageForClaude(joule).length > 0 - ), - }; + const newJoules: Joule[] = []; + for (const joule of conversation.joules) { + if (joule.author === 'human' && (joule.message.trim().length > 0 || (joule.images && joule.images.length > 0))) { + newJoules.push(joule); + } + if (joule.author === 'bot' && (joule as JouleBot)?.botExecInfo?.rawOutput?.trim()?.length > 0) { + newJoules.push(joule); + } + } + return { joules: newJoules }; } /** From 08efcc8b4e5f683e54e14c5d90257a0ca4a6593a Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:31:15 +0530 Subject: [PATCH 12/13] loadTasks function marked async --- extensions/spectacular/src/extension.ts | 2 +- extensions/spectacular/src/services/TaskManager.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/spectacular/src/extension.ts b/extensions/spectacular/src/extension.ts index ebecc1a5..b2d5f2a7 100644 --- a/extensions/spectacular/src/extension.ts +++ b/extensions/spectacular/src/extension.ts @@ -34,7 +34,7 @@ export class MeltyExtension { async activate() { outputChannel.appendLine("Melty activation started"); - this._taskManager.loadTasks(); + await this._taskManager.loadTasks(); // Start the branch check interval // this.branchCheckInterval = setInterval( diff --git a/extensions/spectacular/src/services/TaskManager.ts b/extensions/spectacular/src/services/TaskManager.ts index 2ee9740d..f5695eea 100644 --- a/extensions/spectacular/src/services/TaskManager.ts +++ b/extensions/spectacular/src/services/TaskManager.ts @@ -56,12 +56,12 @@ export class TaskManager { /** * Load tasks from disk. This should only be called once, when the extension is first loaded. */ - public loadTasks(): boolean { + public async loadTasks(): Promise { if (this.inactiveTasks.size > 0) { console.error("Can't load tasks when tasks already exist"); return false; } - this.inactiveTasks = datastores.loadTasksFromDisk(); + this.inactiveTasks = await datastores.loadTasksFromDisk(); return true; } From 69b33c3a3205e288194b1536968b7b85e7e08935 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Sun, 8 Sep 2024 19:35:34 +0530 Subject: [PATCH 13/13] add logic for dehydrating conversation to ui and implement basic lru image cache --- extensions/spectacular/package-lock.json | 17 +++- extensions/spectacular/package.json | 1 + extensions/spectacular/src/HelloWorldPanel.ts | 8 +- .../src/backend/assistants/baseAssistant.ts | 14 ++-- .../src/backend/assistants/coder.ts | 6 +- .../spectacular/src/backend/conversations.ts | 9 ++- .../spectacular/src/backend/datastores.ts | 77 ++++++++++++++++++- extensions/spectacular/src/backend/joules.ts | 44 +++++++++-- extensions/spectacular/src/backend/tasks.ts | 27 +++---- extensions/spectacular/src/constants.ts | 7 ++ extensions/spectacular/src/types.ts | 24 +++++- 11 files changed, 189 insertions(+), 45 deletions(-) create mode 100644 extensions/spectacular/src/constants.ts diff --git a/extensions/spectacular/package-lock.json b/extensions/spectacular/package-lock.json index 7a5c03f3..e241c3ab 100644 --- a/extensions/spectacular/package-lock.json +++ b/extensions/spectacular/package-lock.json @@ -15,6 +15,7 @@ "axios": "^1.7.3", "diff": "^6.0.0-beta", "dotenv": "^16.4.5", + "lru-cache": "^11.0.1", "node-tree-sitter": "^0.0.1", "posthog-js": "^1.157.2", "python-ast": "^0.1.0", @@ -2649,10 +2650,12 @@ } }, "node_modules/lru-cache": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", - "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", - "dev": true + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.1.tgz", + "integrity": "sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==", + "engines": { + "node": "20 || >=22" + } }, "node_modules/make-dir": { "version": "4.0.0", @@ -3267,6 +3270,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true + }, "node_modules/path-to-regexp": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", diff --git a/extensions/spectacular/package.json b/extensions/spectacular/package.json index 23065b5f..725037d9 100644 --- a/extensions/spectacular/package.json +++ b/extensions/spectacular/package.json @@ -149,6 +149,7 @@ "axios": "^1.7.3", "diff": "^6.0.0-beta", "dotenv": "^16.4.5", + "lru-cache": "^11.0.1", "node-tree-sitter": "^0.0.1", "posthog-js": "^1.157.2", "python-ast": "^0.1.0", diff --git a/extensions/spectacular/src/HelloWorldPanel.ts b/extensions/spectacular/src/HelloWorldPanel.ts index 06fb5851..36d627bb 100644 --- a/extensions/spectacular/src/HelloWorldPanel.ts +++ b/extensions/spectacular/src/HelloWorldPanel.ts @@ -199,7 +199,7 @@ export class HelloWorldPanel implements WebviewViewProvider { } task.addErrorJoule(message); await WebviewNotifier.getInstance().sendNotification("updateTask", { - task: task.dehydrateForWire(), + task: await task.dehydrateForWire(), }); } @@ -342,7 +342,7 @@ export class HelloWorldPanel implements WebviewViewProvider { // { uri: newFolderUri } // ); - const openResult: any = await vscode.commands.executeCommand('vscode.openFolder', newFolderUri, false) + const openResult: any = await vscode.commands.executeCommand('vscode.openFolder', newFolderUri, false); return openResult === undefined; } else { return false; @@ -374,12 +374,12 @@ export class HelloWorldPanel implements WebviewViewProvider { } } - private async rpcGetActiveTask(taskId: string): Promise { + private async rpcGetActiveTask(taskId: string) { const task = this._taskManager.getActiveTask(taskId); if (!task) { vscode.window.showErrorMessage(`Failed to get active task ${taskId}`); } - return task!.dehydrate(); + return task!.dehydrateForWire(); } private async rpcListMeltyFiles(): Promise { diff --git a/extensions/spectacular/src/backend/assistants/baseAssistant.ts b/extensions/spectacular/src/backend/assistants/baseAssistant.ts index 683dc778..7b227605 100644 --- a/extensions/spectacular/src/backend/assistants/baseAssistant.ts +++ b/extensions/spectacular/src/backend/assistants/baseAssistant.ts @@ -12,10 +12,10 @@ export abstract class BaseAssistant { abstract respond( conversation: Conversation, contextPaths: ContextPaths, - processPartial: (partialConversation: Conversation) => void + processPartial: (partialConversation: Conversation) => Promise, ): Promise; - protected encodeMessages(conversation: Conversation): ClaudeMessage[] { + protected async encodeMessages(conversation: Conversation): Promise { const userPrompt = getUserPrompt(); const messages: ClaudeMessage[] = []; @@ -35,12 +35,12 @@ export abstract class BaseAssistant { return author === "human" ? "user" : "assistant"; } - messages.push( - ...conversation.joules.map((joule: Joule) => ({ + for (const joule of conversation.joules) { + messages.push({ role: authorToRole(joule.author), - content: joules.formatMessageForClaude(joule), - })) - ); + content: await joules.formatMessageForClaude(joule), + }); + } return messages; } diff --git a/extensions/spectacular/src/backend/assistants/coder.ts b/extensions/spectacular/src/backend/assistants/coder.ts index 8e97de5c..325e1bc5 100644 --- a/extensions/spectacular/src/backend/assistants/coder.ts +++ b/extensions/spectacular/src/backend/assistants/coder.ts @@ -41,7 +41,7 @@ export class Coder extends BaseAssistant { async respond( conversation: Conversation, contextPaths: ContextPaths, - processPartial: (partialConversation: Conversation) => void, + processPartial: (partialConversation: Conversation) => Promise, cancellationToken?: vscode.CancellationToken ) { webviewNotifier.updateStatusMessage("Preparing context"); @@ -83,7 +83,7 @@ export class Coder extends BaseAssistant { messages: [ // TODOV2 user system info ...this.codebaseView(contextPaths, repoMapString), - ...this.encodeMessages(conversation), + ...(await this.encodeMessages(conversation)), ], }; @@ -110,7 +110,7 @@ export class Coder extends BaseAssistant { contextPaths, true // ignore changes ); - processPartial(newConversation); + await processPartial(newConversation); } } ); diff --git a/extensions/spectacular/src/backend/conversations.ts b/extensions/spectacular/src/backend/conversations.ts index 444ad752..eb985662 100644 --- a/extensions/spectacular/src/backend/conversations.ts +++ b/extensions/spectacular/src/backend/conversations.ts @@ -1,4 +1,4 @@ -import { Joule, Conversation } from "../types"; +import { Joule, Conversation, JouleBot, DehydratedConversation } from "../types"; import * as vscode from "vscode"; import * as joules from "./joules"; export function create(): Conversation { @@ -131,3 +131,10 @@ function removeFinalJoulesFrom( }; } } + +export async function dehydrate(conversation: Conversation): Promise { + return { + ...conversation, + joules: await Promise.all(conversation.joules.map(j => joules.dehydrate(j))), + }; +} diff --git a/extensions/spectacular/src/backend/datastores.ts b/extensions/spectacular/src/backend/datastores.ts index 8d1f3a89..fd1003ec 100644 --- a/extensions/spectacular/src/backend/datastores.ts +++ b/extensions/spectacular/src/backend/datastores.ts @@ -64,10 +64,83 @@ export async function deleteTaskFromDisk(task: DehydratedTask): Promise { const meltyDir = getMeltyDir(); const taskPath = path.join(meltyDir, `${task.id}.json`); - if (fs.existsSync(taskPath)) { - fs.unlinkSync(taskPath); + // remove the images in task conversation as well + const images = task.conversation.joules.flatMap(j => j.images || []); + await Promise.allSettled(images.map(async (image) => { + deleteFromImageCache(image.path); + if (image.path && (await isFile(image.path)) && isPathInside(image.path, meltyDir)) { + await fs.unlink(image.path); + } else { + console.log(`Skipping deletion of image ${image.path}`); + } + })); + + if (await pathExists(taskPath)) { + await fs.unlink(taskPath); console.log(`Deleted task file for task ${task.id}`); } else { console.log(`Task file for task ${task.id} not found, skipping deletion`); } } + +export async function saveJouleImagesToDisk(images: UserAttachedImage[]) { + const meltyDir = getMeltyDir(); + const imagesDir = path.join(meltyDir, "assets", "images"); + if (!(await pathExists(imagesDir))) { + await fs.mkdir(imagesDir, { recursive: true }); + } + + const imageData: JouleImage[] = await Promise.all(images.map(async (image) => { + const imageId = uuidv4(); + const extension = image.mimeType.split("/")[1]; + const base64 = image.base64.replace(/^data:image\/\w+;base64,/, ""); + const buff = Buffer.from(base64, "base64"); + const imagePath = path.join(imagesDir, `${imageId}.${extension}`); + await fs.writeFile(imagePath, buff); + return { + path: imagePath, mimeType: image.mimeType + }; + })); + + return imageData; +} + +const imageCache = new LRUCache({ max: MAX_IMAGES_TO_CACHE, ttl: IMAGE_CACHE_TTL_MS }); + +export async function readImageFromDisk(imagePath: string): Promise<{ buffer: Buffer, exists: boolean; }> { + try { + if (imageCache.has(imagePath)) { + return { + buffer: imageCache.get(imagePath)!, + exists: true + }; + } + + if (!await pathExists(imagePath)) { + const buffer = Buffer.from(''); + imageCache.set(imagePath, buffer); + return { + buffer, + exists: false, + }; + } + + const buffer = await fs.readFile(imagePath); + imageCache.set(imagePath, buffer); + return { + buffer, + exists: true + }; + } catch (error) { + console.error(`Failed to read image from disk: ${error}`); + return { + buffer: Buffer.from(''), + exists: false + }; + } +} + +// if imagePath is not provided, clear the entire cache +export function deleteFromImageCache(imagePath: string) { + return imageCache.delete(imagePath); +} diff --git a/extensions/spectacular/src/backend/joules.ts b/extensions/spectacular/src/backend/joules.ts index 0af4613d..9cf866f5 100644 --- a/extensions/spectacular/src/backend/joules.ts +++ b/extensions/spectacular/src/backend/joules.ts @@ -1,6 +1,7 @@ import { v4 as uuidv4 } from "uuid"; -import { Joule, JouleHuman, JouleBot, BotExecInfo, DiffInfo, ClaudeMessage, UserAttachedImage } from "../types"; +import { Joule, JouleHuman, JouleBot, BotExecInfo, DiffInfo, ClaudeMessage, UserAttachedImage, JouleImage, DehydratedJoule } from "../types"; +import * as datastores from './datastores'; export function createJouleError(errorMessage: string): JouleBot { return createJouleBot(errorMessage, { @@ -12,17 +13,19 @@ export function createJouleError(errorMessage: string): JouleBot { }); } -export function createJouleHuman(message: string, images?: UserAttachedImage[]): JouleHuman { +export async function createJouleHuman(message: string, images?: UserAttachedImage[]): Promise { return createJouleHumanWithChanges(message, null, null, images); } -export function createJouleHumanWithChanges( +export async function createJouleHumanWithChanges( message: string, commit: string | null, diffInfo: DiffInfo | null, images?: UserAttachedImage[] -): JouleHuman { +): Promise { const id = uuidv4(); + const imagesData: JouleImage[] | undefined = images ? await datastores.saveJouleImagesToDisk(images) : []; + return { id, message, @@ -30,7 +33,7 @@ export function createJouleHumanWithChanges( state: "complete", commit, diffInfo, - images + images: imagesData, }; } @@ -65,7 +68,7 @@ export function updateMessage(joule: Joule, message: string): Joule { return { ...joule, message }; } -export function formatMessageForClaude(joule: Joule): ClaudeMessage['content'] { +export async function formatMessageForClaude(joule: Joule): Promise { // note that if we show a processed message, we'll need to use `message.length ? message : "..."` // to ensure no Anthropic API errors switch (joule.author) { @@ -73,11 +76,18 @@ export function formatMessageForClaude(joule: Joule): ClaudeMessage['content'] { if (joule.images && joule.images.length > 0) { const content: ClaudeMessage['content'] = []; for (const img of joule.images) { + const { buffer, exists } = await datastores.readImageFromDisk(img.path); + if (!exists) { + console.warn(`Skipping image ${img.path} as it doesn't exist`); + continue; + } + + const base64 = buffer.toString('base64'); content.push({ type: 'image', source: { type: 'base64', - data: img.base64.replace(/^data:image\/\w+;base64,/, ''), + data: base64, media_type: img.mimeType } }); @@ -92,6 +102,24 @@ export function formatMessageForClaude(joule: Joule): ClaudeMessage['content'] { } return joule.message; case "bot": - return (joule as JouleBot).botExecInfo.rawOutput ?? ""; + return (joule as JouleBot)?.botExecInfo?.rawOutput ?? ""; } } + +// reads images in the joule and converts them to base64 to be rendered on the ui +export async function dehydrate(joule: Joule): Promise { + return { + ...joule, + images: joule.images ? await Promise.all(joule.images.map(async (img) => { + const { buffer } = await datastores.readImageFromDisk(img.path); + let base64 = buffer.toString('base64'); + // add the data:image\/\w+;base64, prefix + base64 = `data:${img.mimeType};base64,${base64}`; + + return { + mimeType: img.mimeType, + base64: base64 + }; + })) : undefined + }; +} diff --git a/extensions/spectacular/src/backend/tasks.ts b/extensions/spectacular/src/backend/tasks.ts index 98ae505c..59cb1195 100644 --- a/extensions/spectacular/src/backend/tasks.ts +++ b/extensions/spectacular/src/backend/tasks.ts @@ -1,5 +1,5 @@ import * as vscode from "vscode"; -import { Joule, Conversation, TaskMode, DehydratedTask, UserAttachedImage } from "../types"; +import { Joule, Conversation, TaskMode, DehydratedTask, UserAttachedImage, DehydratedTaskWithDehydratedConversation } from "../types"; import * as conversations from "./conversations"; import * as joules from "./joules"; import * as utils from "../util/utils"; @@ -92,7 +92,7 @@ export class Task { * @param processPartial - a function to process the partial joule */ private async respondBot( - processPartial: (partialConversation: Conversation) => void, + processPartial: (partialConversation: Conversation) => Promise, cancellationToken?: vscode.CancellationToken, ): Promise { try { @@ -184,10 +184,10 @@ export class Task { webviewNotifier.resetStatusMessage(); newJoule = commitResult !== null - ? joules.createJouleHumanWithChanges(message, commitResult.commit, commitResult.diffInfo, images) - : joules.createJouleHuman(message, images); + ? await joules.createJouleHumanWithChanges(message, commitResult.commit, commitResult.diffInfo, images) + : await joules.createJouleHuman(message, images); } else { - newJoule = joules.createJouleHuman(message, images); + newJoule = await joules.createJouleHuman(message, images); } this.conversation = conversations.addJoule(this.conversation, newJoule); @@ -217,13 +217,13 @@ export class Task { // human response await this.respondHuman(text, images); webviewNotifier.sendNotification("updateTask", { - task: this.dehydrateForWire(), + task: await this.dehydrateForWire(), }); // bot response - const processPartial = (partialConversation: Conversation) => { - const dehydratedTask = this.dehydrateForWire(); - dehydratedTask.conversation = partialConversation; + const processPartial = async (partialConversation: Conversation) => { + const dehydratedTask = await this.dehydrateForWire(); + dehydratedTask.conversation = await conversations.dehydrate(partialConversation); webviewNotifier.sendNotification("updateTask", { task: dehydratedTask, }); @@ -234,7 +234,7 @@ export class Task { ); webviewNotifier.sendNotification("updateTask", { - task: this.dehydrateForWire(), + task: await this.dehydrateForWire(), }); webviewNotifier.resetStatusMessage(); @@ -257,14 +257,15 @@ export class Task { } /** - * Leaves out the melty mind files + * Leaves out the melty mind files and dehydrates the conversation to include joule images as base64 strings */ - public dehydrateForWire(): DehydratedTask { + public async dehydrateForWire(): Promise { return { ...this, _fileManager: undefined, _gitManager: undefined, - files: null + files: null, + conversation: await conversations.dehydrate(this.conversation), }; } diff --git a/extensions/spectacular/src/constants.ts b/extensions/spectacular/src/constants.ts new file mode 100644 index 00000000..4fc726c0 --- /dev/null +++ b/extensions/spectacular/src/constants.ts @@ -0,0 +1,7 @@ +export const MAX_IMAGES = 5; +export const ALLOWED_IMAGE_MIME_TYPES = [ + "image/jpeg", "image/png", "image/gif", "image/webp" +]; +export const MAX_IMAGE_SIZE = 5 * 1024 * 1024; // 5MB +export const MAX_IMAGES_TO_CACHE = 50; +export const IMAGE_CACHE_TTL_MS = 10 * 60 * 1000; // 10 minutes diff --git a/extensions/spectacular/src/types.ts b/extensions/spectacular/src/types.ts index c6e3fbaf..a4406ea2 100644 --- a/extensions/spectacular/src/types.ts +++ b/extensions/spectacular/src/types.ts @@ -14,6 +14,10 @@ export type DehydratedTask = { meltyMindFiles: string[]; }; +export type DehydratedTaskWithDehydratedConversation = Omit & { + conversation: DehydratedConversation; +}; + export type ContextPaths = { readonly paths: string[]; meltyRoot: string; @@ -26,22 +30,32 @@ export interface AssistantInfo { description: string; } +export type AllowedMimeTypes = "image/jpeg" | "image/png" | "image/gif" | "image/webp"; + export type UserAttachedImage = { - blobUrl: string; - mimeType: "image/jpeg" | "image/png" | "image/gif" | "image/webp"; + mimeType: AllowedMimeTypes; base64: string; }; +export type JouleImage = { + readonly path: string; + mimeType: AllowedMimeTypes; +}; + export type Joule = { readonly id: string; readonly author: "human" | "bot"; readonly state: "complete" | "partial" | "error"; readonly message: string; - readonly images?: UserAttachedImage[]; + readonly images?: JouleImage[]; readonly commit: string | null; readonly diffInfo: DiffInfo | null; }; +export type DehydratedJoule = Omit & { + readonly images?: UserAttachedImage[]; +}; + export type JouleHuman = Joule & { readonly author: "human"; }; @@ -95,6 +109,10 @@ export type Conversation = { readonly joules: ReadonlyArray; }; +export type DehydratedConversation = { + readonly joules: ReadonlyArray; +}; + export type MeltyFile = { readonly relPath: string; readonly contents: string;