Skip to content

Commit

Permalink
[editor] Image Output Rendering (#744)
Browse files Browse the repository at this point in the history
# [editor] Image Output Rendering

Get image output rendering with the updated output types, handling both
file_uri and base64 encoding.

## Testing:
- Tested both dall-e-2 and 3 models to ensure outputs render, toggle
between uri/base64 and raw JSON:


https://github.com/lastmile-ai/aiconfig/assets/5060851/0f73b765-d509-4305-a3bd-98ddb3005ff9


- Made sure the gpt4-v input rendering still works with updated
MimeTypeRenderer component:
<img width="1329" alt="Screenshot 2024-01-03 at 4 31 52 PM"
src="https://github.com/lastmile-ai/aiconfig/assets/5060851/3ee6c82c-d393-41d3-9698-7e17a4a456dc">
  • Loading branch information
rholinshead authored Jan 3, 2024
2 parents 2567e43 + 2080486 commit 98dced1
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 20 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import { memo } from "react";
import { Image } from "@mantine/core";

type Props = {
mimeType?: string;
content: string;
};

export default memo(function MimeTypeRenderer({ mimeType, content }: Props) {
const mimetype = (mimeType ?? "text/plain").split("/", 1)[0]; // MIME type without subtype
switch (mimetype) {
case "image":
// TODO: Figure out base64 encoding
return <Image alt="Attachment image" src={content} maw={300} />;
default: // "text"
return <span>{content}</span>;
}
});

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { memo } from "react";
import type { Attachment as InputAttachment, JSONObject } from "aiconfig";
import { PromptInputObjectAttachmentsSchema } from "../../../../utils/promptUtils";
import Attachment from "./Attachment";
import { ActionIcon, Container, Flex, Tooltip } from "@mantine/core";
import { IconEdit, IconTrash } from "@tabler/icons-react";
import AttachmentMetadata from "./AttachmentMetadata";
import MimeTypeRenderer from "../../../MimeTypeRenderer";

type Props = {
schema: PromptInputObjectAttachmentsSchema;
Expand Down Expand Up @@ -39,7 +39,10 @@ export default memo(function AttachmentContainer({
</ActionIcon>
)}
</Flex>
<Attachment attachment={attachment} />
<MimeTypeRenderer
mimeType={attachment.mime_type}
content={attachment.data as string}
/>
{schema.items.properties?.metadata && (
<AttachmentMetadata
schema={schema.items.properties.metadata}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { memo } from "react";
import { TextRenderer } from "../TextRenderer";
import JSONOutput from "./JSONOutput";
import PromptOutputWrapper from "./PromptOutputWrapper";
import MimeTypeRenderer from "../../MimeTypeRenderer";

type Props = {
outputs: Output[];
Expand Down Expand Up @@ -43,6 +44,34 @@ const ExecuteResultOutput = memo(function ExecuteResultOutput({
Object.prototype.hasOwnProperty.call(output.data, "kind")
) {
switch ((output.data as OutputDataWithValue).kind) {
case "file_uri":
return (
<PromptOutputWrapper
copyContent={(output.data as OutputDataWithValue).value as string}
output={output}
withRawJSONToggle
>
<MimeTypeRenderer
mimeType={output.mime_type}
content={(output.data as OutputDataWithValue).value as string}
/>
</PromptOutputWrapper>
);
case "base64":
return (
<PromptOutputWrapper
copyContent={(output.data as OutputDataWithValue).value as string}
output={output}
withRawJSONToggle
>
<MimeTypeRenderer
mimeType={output.mime_type}
content={`data:${output.mime_type};base64, ${
(output.data as OutputDataWithValue).value as string
}`}
/>
</PromptOutputWrapper>
);
// TODO: Tool calls rendering
default:
return (
Expand Down

0 comments on commit 98dced1

Please sign in to comment.