img txt 2 txt fully working

src/lib/components/inference-playground/conversation.svelte

@@ -58,8 +58,8 @@
 	{#if !viewCode}
 		{#each conversation.messages as _msg, idx}
 			<Message
-				bind:content={conversation.messages[idx]!.content}
-				role={conversation.messages[idx]!.role}
+				bind:message={conversation.messages[idx]!}
+				{conversation}
 				autofocus={idx === conversation.messages.length - 1}
 				{loading}
 				onDelete={() => deleteMessage(idx)}

src/lib/components/inference-playground/message.svelte

@@ -1,83 +1,140 @@
 <script lang="ts">
-	import { TextareaAutosize } from "$lib/spells/textarea-autosize.svelte.js";
-	import type { ConversationMessage } from "$lib/types.js";
 	import Tooltip from "$lib/components/tooltip.svelte";
+	import { TextareaAutosize } from "$lib/spells/textarea-autosize.svelte.js";
+	import { PipelineTag, type Conversation, type ConversationMessage } from "$lib/types.js";
+	import { fileToDataURL } from "$lib/utils/file.js";
+	import { FileUpload } from "melt/builders";
+	import { fade } from "svelte/transition";
 	import IconImage from "~icons/carbon/image-reference";
 
 	type Props = {
-		content: ConversationMessage["content"];
-		role: ConversationMessage["role"];
+		conversation: Conversation;
+		message: ConversationMessage;
 		loading?: boolean;
 		autofocus?: boolean;
 		onDelete?: () => void;
 	};
 
-	let { content = $bindable(""), role, loading, autofocus, onDelete }: Props = $props();
+	let { message = $bindable(), conversation, loading, autofocus, onDelete }: Props = $props();
 
 	let element = $state<HTMLTextAreaElement>();
 	new TextareaAutosize({
 		element: () => element,
-		input: () => content,
+		input: () => message.content ?? "",
+	});
+
+	const canUploadImgs = $derived(
+		message.role === "user" && conversation.model.pipeline_tag === PipelineTag.ImageTextToText
+	);
+	const fileUpload = new FileUpload({
+		accept: "image/*",
+		async onAccept(file) {
+			if (!message.images) message.images = [];
+
+			const dataUrl = await fileToDataURL(file);
+			if (message.images.includes(dataUrl)) return;
+
+			message.images.push(await fileToDataURL(file));
+			// We're dealing with files ourselves, so we don't want fileUpload to have any internal state,
+			// to avoid conflicts
+			fileUpload.clear();
+		},
+		disabled: () => !canUploadImgs,
 	});
 </script>
 
 <div
-	class="group/message group flex flex-col items-start gap-x-4 gap-y-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70
-	@2xl:flex-row @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
+	class="group/message group relative flex flex-col items-start gap-x-4 gap-y-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70
+	 @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
 	class:pointer-events-none={loading}
+	{...fileUpload.dropzone}
+	onclick={undefined}
 >
-	<div class="pt-3 text-sm font-semibold uppercase @2xl:basis-[130px]">
-		{role}
-	</div>
-	<div class="flex w-full items-center gap-4">
-		<!-- svelte-ignore a11y_autofocus -->
-		<!-- svelte-ignore a11y_positive_tabindex -->
-		<textarea
-			bind:this={element}
-			{autofocus}
-			bind:value={content}
-			placeholder="Enter {role} message"
-			class="grow resize-none overflow-hidden rounded-lg bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:bg-white focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
-			rows="1"
-			tabindex="2"
-		></textarea>
+	<div class=" flex w-full flex-col items-start gap-x-4 gap-y-2 @2xl:flex-row">
+		{#if fileUpload.isDragging}
+			<div
+				class="absolute inset-2 z-10 flex flex-col items-center justify-center rounded-xl bg-gray-800/50 backdrop-blur-md"
+				transition:fade={{ duration: 100 }}
+			>
+				<IconImage />
+				<p>Drop the image here to upload</p>
+			</div>
+		{/if}
+
+		<div class="pt-3 text-sm font-semibold uppercase @2xl:basis-[130px]">
+			{message.role}
+		</div>
+		<div class="flex w-full gap-4">
+			<!-- svelte-ignore a11y_autofocus -->
+			<!-- svelte-ignore a11y_positive_tabindex -->
+			<textarea
+				bind:this={element}
+				{autofocus}
+				bind:value={message.content}
+				placeholder="Enter {message.role} message"
+				class="grow resize-none overflow-hidden rounded-lg bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:bg-white focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
+				rows="1"
+				tabindex="2"
+			></textarea>
 
-		<Tooltip openDelay={250}>
-			{#snippet trigger(tooltip)}
-				<button
-					tabindex="0"
-					onclick={onDelete}
-					type="button"
-					class="grid size-8 place-items-center rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
+			{#if canUploadImgs}
+				<Tooltip openDelay={250}>
+					{#snippet trigger(tooltip)}
+						<button
+							tabindex="0"
+							type="button"
+							class="mt-1.5 grid size-8 place-items-center rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
 			group-focus-within/message:visible group-hover/message:visible hover:bg-gray-100
 			hover:text-blue-700 focus:z-10 focus:ring-4
 			focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600 dark:bg-gray-800
 			dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
-					{...tooltip.trigger}
-				>
-					<IconImage />
-				</button>
-			{/snippet}
-			Add image
-		</Tooltip>
+							{...tooltip.trigger}
+							{...fileUpload.trigger}
+						>
+							<IconImage />
+						</button>
+						<input {...fileUpload.input} />
+					{/snippet}
+					Add image
+				</Tooltip>
+			{/if}
 
-		<Tooltip>
-			{#snippet trigger(tooltip)}
-				<button
-					tabindex="0"
-					onclick={onDelete}
-					type="button"
-					class="size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
+			<Tooltip>
+				{#snippet trigger(tooltip)}
+					<button
+						tabindex="0"
+						onclick={onDelete}
+						type="button"
+						class="mt-1.5 size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
 			group-focus-within/message:visible group-hover/message:visible hover:bg-gray-100
 			hover:text-blue-700 focus:z-10 focus:ring-4
 			focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600 dark:bg-gray-800
 			dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
-					{...tooltip.trigger}
-				>
-					✕
-				</button>
-			{/snippet}
-			Delete
-		</Tooltip>
+						{...tooltip.trigger}
+					>
+						✕
+					</button>
+				{/snippet}
+				Delete
+			</Tooltip>
+		</div>
 	</div>
+	{#if message.images?.length}
+		<div class="mt-2">
+			<div class="flex items-center gap-2">
+				{#each message.images as img (img)}
+					<div class="group/img relative">
+						<img src={img} alt="uploaded" class="size-12 rounded-lg object-cover" />
+						<button
+							type="button"
+							onclick={() => (message.images = message.images?.filter(i => i !== img))}
+							class="invisible absolute -top-1 -right-1 grid size-5 place-items-center rounded-full bg-gray-800 text-xs text-white group-hover/img:visible hover:bg-gray-700"
+						>
+							✕
+						</button>
+					</div>
+				{/each}
+			</div>
+		</div>
+	{/if}
 </div>

src/lib/components/inference-playground/utils.ts

@@ -1,8 +1,29 @@
-import type { Conversation, ModelWithTokenizer } from "$lib/types.js";
-import type { InferenceSnippet } from "@huggingface/tasks";
+import type { Conversation, ConversationMessage, ModelWithTokenizer } from "$lib/types.js";
+import type { ChatCompletionInputMessage, InferenceSnippet } from "@huggingface/tasks";
 import { type ChatCompletionOutputMessage } from "@huggingface/tasks";
 
 import { HfInference, snippets, type InferenceProvider } from "@huggingface/inference";
+type ChatCompletionInputMessageChunk =
+	NonNullable<ChatCompletionInputMessage["content"]> extends string | (infer U)[] ? U : never;
+
+function parseMessage(message: ConversationMessage): ChatCompletionInputMessage {
+	if (!message.images) return message;
+	return {
+		...message,
+		content: [
+			{
+				type: "text",
+				text: message.content ?? "",
+			},
+			...message.images.map(img => {
+				return {
+					type: "image_url",
+					image_url: { url: img },
+				} satisfies ChatCompletionInputMessageChunk;
+			}),
+		],
+	};
+}
 
 export async function handleStreamingResponse(
 	hf: HfInference,
@@ -19,7 +40,7 @@ export async function handleStreamingResponse(
 	for await (const chunk of hf.chatCompletionStream(
 		{
 			model: model.id,
-			messages,
+			messages: messages.map(parseMessage),
 			provider: conversation.provider,
 			...conversation.config,
 		},
@@ -44,7 +65,7 @@ export async function handleNonStreamingResponse(
 
 	const response = await hf.chatCompletion({
 		model: model.id,
-		messages,
+		messages: messages.map(parseMessage),
 		provider: conversation.provider,
 		...conversation.config,
 	});

src/lib/types.ts

@@ -3,6 +3,7 @@ import type { ChatCompletionInputMessage } from "@huggingface/tasks";
 
 export type ConversationMessage = Pick<ChatCompletionInputMessage, "name" | "role" | "tool_calls"> & {
 	content?: string;
+	images?: string[];
 };
 
 export type Conversation = {

src/lib/utils/file.ts

@@ -0,0 +1,15 @@
+export function fileToDataURL(file: File): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const reader = new FileReader();
+
+		reader.onload = function (event) {
+			resolve(event.target?.result as string);
+		};
+
+		reader.onerror = function (error) {
+			reject(error);
+		};
+
+		reader.readAsDataURL(file);
+	});
+}