(jdecampos) Feature: copy and paste images

extensions/spectacular/package.json

@@ -149,6 +149,7 @@
     "axios": "^1.7.3",
     "diff": "^6.0.0-beta",
     "dotenv": "^16.4.5",
+    "lru-cache": "^11.0.1",
     "node-tree-sitter": "^0.0.1",
     "posthog-js": "^1.157.2",
     "python-ast": "^0.1.0",

extensions/spectacular/src/HelloWorldPanel.ts

@@ -7,20 +7,19 @@ import {
 } from "vscode";
 import * as vscode from "vscode";
 import { getUri, getNonce } from "./util/utils";
-import { Conversation, TaskMode } from "./types";
+import { TaskMode } from "./types";
 import { MeltyExtension } from "./extension";
 import { createNewDehydratedTask } from "./backend/tasks";
 import * as config from "./util/config";
 import { WebviewNotifier } from "./services/WebviewNotifier";
 import { FileManager } from "./services/FileManager";
-import { DehydratedTask, RpcMethod } from "./types";
+import { DehydratedTask, RpcMethod, UserAttachedImage } from "./types";
 import { Coder } from "./backend/assistants/coder";
 import { Vanilla } from "./backend/assistants/vanilla";
 import { GitManager } from "./services/GitManager";
 import { GitHubManager } from './services/GitHubManager';
 import { TaskManager } from './services/TaskManager';
 import posthog from "posthog-js";
-import { create } from 'domain';
 
 /**
  * This class manages the state and behavior of HelloWorld webview panels.
@@ -135,7 +134,7 @@ export class HelloWorldPanel implements WebviewViewProvider {
           <meta charset="utf-8">
           <meta name="viewport" content="width=device-width,initial-scale=1,shrink-to-fit=no">
           <meta name="theme-color" content="#000000">
-          <meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src ${webview.cspSource} https://*.posthog.com; style-src ${webview.cspSource}; script-src 'nonce-${nonce}' 'unsafe-inline' https://*.posthog.com; connect-src https://*.posthog.com;">
+          <meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src ${webview.cspSource} https://*.posthog.com blob: data:; style-src ${webview.cspSource} 'unsafe-inline'; script-src 'nonce-${nonce}' 'unsafe-inline' https://*.posthog.com; connect-src https://*.posthog.com blob:;">
           <link rel="stylesheet" type="text/css" href="${stylesUri}">
           <title>Melty</title>
         </head>
@@ -200,7 +199,7 @@ export class HelloWorldPanel implements WebviewViewProvider {
 		}
 		task.addErrorJoule(message);
 		await WebviewNotifier.getInstance().sendNotification("updateTask", {
-			task: task.dehydrateForWire(),
+			task: await task.dehydrateForWire(),
 		});
 	}
 
@@ -229,7 +228,7 @@ export class HelloWorldPanel implements WebviewViewProvider {
 					return this.rpcGetLatestCommit();
 				case "chatMessage":
 					return await this.rpcStartResponse(
-						params.text, params.taskId
+						params.text, params.taskId, params.images
 					);
 				case "createTask":
 					return await this.rpcCreateTask(
@@ -257,6 +256,13 @@ export class HelloWorldPanel implements WebviewViewProvider {
 					return this.rpcCheckOnboardingComplete();
 				case "setOnboardingComplete":
 					return this.rpcSetOnboardingComplete();
+				case "showNotification":
+					if (params.notificationType === 'error') {
+						vscode.window.showErrorMessage(params.message);
+					} else {
+						vscode.window.showInformationMessage(params.message);
+					}
+					return true;
 				default:
 					throw new Error(`Unknown RPC method: ${method}`);
 			}
@@ -336,7 +342,7 @@ export class HelloWorldPanel implements WebviewViewProvider {
 			// 	{ uri: newFolderUri }
 			// );
 
-			const openResult: any = await vscode.commands.executeCommand('vscode.openFolder', newFolderUri, false)
+			const openResult: any = await vscode.commands.executeCommand('vscode.openFolder', newFolderUri, false);
 			return openResult === undefined;
 		} else {
 			return false;
@@ -368,12 +374,12 @@ export class HelloWorldPanel implements WebviewViewProvider {
 		}
 	}
 
-	private async rpcGetActiveTask(taskId: string): Promise<DehydratedTask | undefined> {
+	private async rpcGetActiveTask(taskId: string) {
 		const task = this._taskManager.getActiveTask(taskId);
 		if (!task) {
 			vscode.window.showErrorMessage(`Failed to get active task ${taskId}`);
 		}
-		return task!.dehydrate();
+		return task!.dehydrateForWire();
 	}
 
 	private async rpcListMeltyFiles(): Promise<string[]> {
@@ -463,12 +469,12 @@ export class HelloWorldPanel implements WebviewViewProvider {
 		return vscode.window.activeColorTheme.kind === vscode.ColorThemeKind.Dark ? 'dark' : 'light';
 	}
 
-	private async rpcStartResponse(text: string, taskId: string): Promise<boolean> {
+	private async rpcStartResponse(text: string, taskId: string, images?: UserAttachedImage[]): Promise<boolean> {
 		const task = this._taskManager.getActiveTask(taskId)!;
 		if (!task) {
 			throw new Error(`Tried to chat with an inactive task ${taskId} (active task is ${this._taskManager.getActiveTaskId()})`);
 		}
-		return task.startResponse(text);
+		return task.startResponse(text, images);
 	}
 
 	private async rpcStopResponse(taskId: string): Promise<void> {

extensions/spectacular/src/backend/assistants/baseAssistant.ts

@@ -12,10 +12,10 @@ export abstract class BaseAssistant {
 	abstract respond(
 		conversation: Conversation,
 		contextPaths: ContextPaths,
-		processPartial: (partialConversation: Conversation) => void
+		processPartial: (partialConversation: Conversation) => Promise<void>,
 	): Promise<Conversation>;
 
-	protected encodeMessages(conversation: Conversation): ClaudeMessage[] {
+	protected async encodeMessages(conversation: Conversation): Promise<ClaudeMessage[]> {
 		const userPrompt = getUserPrompt();
 		const messages: ClaudeMessage[] = [];
 
@@ -35,12 +35,12 @@ export abstract class BaseAssistant {
 			return author === "human" ? "user" : "assistant";
 		}
 
-		messages.push(
-			...conversation.joules.map((joule: Joule) => ({
+		for (const joule of conversation.joules) {
+			messages.push({
 				role: authorToRole(joule.author),
-				content: joules.formatMessageForClaude(joule),
-			}))
-		);
+				content: await joules.formatMessageForClaude(joule),
+			});
+		}
 
 		return messages;
 	}

extensions/spectacular/src/backend/assistants/coder.ts

@@ -41,7 +41,7 @@ export class Coder extends BaseAssistant {
 	async respond(
 		conversation: Conversation,
 		contextPaths: ContextPaths,
-		processPartial: (partialConversation: Conversation) => void,
+		processPartial: (partialConversation: Conversation) => Promise<void>,
 		cancellationToken?: vscode.CancellationToken
 	) {
 		webviewNotifier.updateStatusMessage("Preparing context");
@@ -83,7 +83,7 @@ export class Coder extends BaseAssistant {
 			messages: [
 				// TODOV2 user system info
 				...this.codebaseView(contextPaths, repoMapString),
-				...this.encodeMessages(conversation),
+				...(await this.encodeMessages(conversation)),
 			],
 		};
 
@@ -110,7 +110,7 @@ export class Coder extends BaseAssistant {
 						contextPaths,
 						true // ignore changes
 					);
-					processPartial(newConversation);
+					await processPartial(newConversation);
 				}
 			}
 		);

extensions/spectacular/src/backend/conversations.ts

@@ -1,4 +1,4 @@
-import { Joule, Conversation } from "../types";
+import { Joule, Conversation, JouleBot, DehydratedConversation } from "../types";
 import * as vscode from "vscode";
 import * as joules from "./joules";
 export function create(): Conversation {
@@ -74,11 +74,16 @@ function removeLeadingBotJoules(conversation: Conversation): Conversation {
 }
 
 function removeEmptyJoules(conversation: Conversation): Conversation {
-	return {
-		joules: conversation.joules.filter((joule) =>
-			joules.formatMessageForClaude(joule).length > 0
-		),
-	};
+	const newJoules: Joule[] = [];
+	for (const joule of conversation.joules) {
+		if (joule.author === 'human' && (joule.message.trim().length > 0 || (joule.images && joule.images.length > 0))) {
+			newJoules.push(joule);
+		}
+		if (joule.author === 'bot' && (joule as JouleBot)?.botExecInfo?.rawOutput?.trim()?.length > 0) {
+			newJoules.push(joule);
+		}
+	}
+	return { joules: newJoules };
 }
 
 /**
@@ -126,3 +131,10 @@ function removeFinalJoulesFrom(
 		};
 	}
 }
+
+export async function dehydrate(conversation: Conversation): Promise<DehydratedConversation> {
+	return {
+		...conversation,
+		joules: await Promise.all(conversation.joules.map(j => joules.dehydrate(j))),
+	};
+}

extensions/spectacular/src/backend/datastores.ts

@@ -1,8 +1,11 @@
-import * as fs from "fs";
+import fs from "fs/promises";
 import * as path from "path";
 import * as vscode from "vscode";
-import { resolveTildePath } from "../util/utils";
-import { DehydratedTask } from "types";
+import { isPathInside, resolveTildePath, isDirectory, isFile, pathExists } from "../util/utils";
+import { DehydratedTask, JouleImage, UserAttachedImage } from "types";
+import { v4 as uuidv4 } from "uuid";
+import { LRUCache } from 'lru-cache';
+import { IMAGE_CACHE_TTL_MS, MAX_IMAGES_TO_CACHE } from '../constants';
 
 function getMeltyDir(): string {
 	const config = vscode.workspace.getConfiguration('melty');
@@ -11,17 +14,22 @@ function getMeltyDir(): string {
 	return resolvedPath;
 }
 
-export function loadTasksFromDisk(): Map<string, DehydratedTask> {
+export async function loadTasksFromDisk(): Promise<Map<string, DehydratedTask>> {
 	const meltyDir = getMeltyDir();
-	if (!fs.existsSync(meltyDir)) {
+	if (!(await pathExists(meltyDir))) {
 		return new Map();
 	}
 
-	const taskFiles = fs.readdirSync(meltyDir);
+	const taskFiles = await fs.readdir(meltyDir);
 	const taskMap = new Map<string, DehydratedTask>();
-	for (const file of taskFiles) {
+	await Promise.all(taskFiles.map(async (file) => {
+		const filePath = path.join(meltyDir, file);
+		if ((await isDirectory(filePath))) {
+			return;
+		}
+
 		const rawTask = JSON.parse(
-			fs.readFileSync(path.join(meltyDir, file), "utf8")
+			await fs.readFile(filePath, "utf8")
 		);
 
 		const task = Object.fromEntries(
@@ -37,28 +45,102 @@ export function loadTasksFromDisk(): Map<string, DehydratedTask> {
 			].includes(key))
 		) as DehydratedTask;
 		taskMap.set(task.id, task);
-	}
+	}));
+
 	return taskMap;
 }
 
 export async function dumpTaskToDisk(task: DehydratedTask): Promise<void> {
 	const meltyDir = getMeltyDir();
-	if (!fs.existsSync(meltyDir)) {
-		fs.mkdirSync(meltyDir, { recursive: true });
+	if (!(await pathExists(meltyDir))) {
+		await fs.mkdir(meltyDir, { recursive: true });
 	}
 
 	const taskPath = path.join(meltyDir, `${task.id}.json`);
-	fs.writeFileSync(taskPath, JSON.stringify(task, null, 2));
+	await fs.writeFile(taskPath, JSON.stringify(task, null, 2));
 }
 
 export async function deleteTaskFromDisk(task: DehydratedTask): Promise<void> {
 	const meltyDir = getMeltyDir();
 	const taskPath = path.join(meltyDir, `${task.id}.json`);
 
-	if (fs.existsSync(taskPath)) {
-		fs.unlinkSync(taskPath);
+	// remove the images in task conversation as well
+	const images = task.conversation.joules.flatMap(j => j.images || []);
+	await Promise.allSettled(images.map(async (image) => {
+		deleteFromImageCache(image.path);
+		if (image.path && (await isFile(image.path)) && isPathInside(image.path, meltyDir)) {
+			await fs.unlink(image.path);
+		} else {
+			console.log(`Skipping deletion of image ${image.path}`);
+		}
+	}));
+
+	if (await pathExists(taskPath)) {
+		await fs.unlink(taskPath);
 		console.log(`Deleted task file for task ${task.id}`);
 	} else {
 		console.log(`Task file for task ${task.id} not found, skipping deletion`);
 	}
 }
+
+export async function saveJouleImagesToDisk(images: UserAttachedImage[]) {
+	const meltyDir = getMeltyDir();
+	const imagesDir = path.join(meltyDir, "assets", "images");
+	if (!(await pathExists(imagesDir))) {
+		await fs.mkdir(imagesDir, { recursive: true });
+	}
+
+	const imageData: JouleImage[] = await Promise.all(images.map(async (image) => {
+		const imageId = uuidv4();
+		const extension = image.mimeType.split("/")[1];
+		const base64 = image.base64.replace(/^data:image\/\w+;base64,/, "");
+		const buff = Buffer.from(base64, "base64");
+		const imagePath = path.join(imagesDir, `${imageId}.${extension}`);
+		await fs.writeFile(imagePath, buff);
+		return {
+			path: imagePath, mimeType: image.mimeType
+		};
+	}));
+
+	return imageData;
+}
+
+const imageCache = new LRUCache<string, Buffer>({ max: MAX_IMAGES_TO_CACHE, ttl: IMAGE_CACHE_TTL_MS });
+
+export async function readImageFromDisk(imagePath: string): Promise<{ buffer: Buffer, exists: boolean; }> {
+	try {
+		if (imageCache.has(imagePath)) {
+			return {
+				buffer: imageCache.get(imagePath)!,
+				exists: true
+			};
+		}
+
+		if (!await pathExists(imagePath)) {
+			const buffer = Buffer.from('');
+			imageCache.set(imagePath, buffer);
+			return {
+				buffer,
+				exists: false,
+			};
+		}
+
+		const buffer = await fs.readFile(imagePath);
+		imageCache.set(imagePath, buffer);
+		return {
+			buffer,
+			exists: true
+		};
+	} catch (error) {
+		console.error(`Failed to read image from disk: ${error}`);
+		return {
+			buffer: Buffer.from(''),
+			exists: false
+		};
+	}
+}
+
+// if imagePath is not provided, clear the entire cache
+export function deleteFromImageCache(imagePath: string) {
+	return imageCache.delete(imagePath);
+}