track pending requests, improve ui, add qwen-2-0.5B

src/routes/+page.svelte

@@ -4,6 +4,8 @@
     import * as webllm from "@mlc-ai/web-llm";
     import { onMount } from 'svelte';
 
+    let selectedModel = "SmolLM-360M-Instruct-q4f16_1-MLC";
+
     let engine: webllm.MLCEngineInterface;
     let isLoading = false;
     let loadingStatus = '';
@@ -12,8 +14,8 @@
     let error = '';
     let completionSpeed: number | null = null;
     let tokensPerSecond: number | null = null;
-    let selectedModel = "SmolLM-360M-Instruct-q4f16_1-MLC";
     let isGenerating = false;
+    let pendingRequest: string | null = null;
 
     async function loadWebLLM() {
         isLoading = true;
@@ -24,11 +26,18 @@
 
         const appConfig: webllm.AppConfig = {
             model_list: [{
-                model: `https://huggingface.co/mlc-ai/${selectedModel}`,
-                model_id: selectedModel,
+                model: `https://huggingface.co/mlc-ai/SmolLM-360M-Instruct-q4f16_1-MLC`,
+                model_id: 'SmolLM-360M-Instruct-q4f16_1-MLC',
                 model_lib: `${webllm.modelLibURLPrefix}${webllm.modelVersion}/SmolLM-360M-Instruct-q4f16_1-ctx2k_cs1k-webgpu.wasm`,
                 overrides: { context_window_size: 2048 },
-            }],
+            },
+            {
+                model: `https://huggingface.co/mlc-ai/Qwen2-0.5B-Instruct-q4f16_1-MLC`,
+                model_id: 'Qwen2-0.5B-Instruct-q4f16_1-MLC',
+                model_lib: `${webllm.modelLibURLPrefix}${webllm.modelVersion}/Qwen2-0.5B-Instruct-q4f16_1-ctx4k_cs1k-webgpu.wasm`,
+                overrides: { context_window_size: 2048 },
+            }
+        ],
         };
 
         try {
@@ -44,18 +53,37 @@
         }
     }
 
-    async function generateCompletion() {
-        if (!engine || !inputText.trim() || isGenerating) return;
+    async function generateCompletion(content: string) {
+        if (!engine || isGenerating) {
+            /**
+             * This is used to store the most recent request from user
+             * while the current request is being processed.
+             */
+            pendingRequest = content.trim();
+            return;
+        }
+
+        if (!content.trim()) return;
 
         isGenerating = true;
         const startTime = performance.now();
         try {
+            console.log("Generating completion:", content);
             const response = await engine.chat.completions.create({
-                messages: [{role:"system",content: "You are a helpful AI agent helping users. Try your best to answer the users request.",},{ role: "user", content: inputText }],
+                messages: [
+                    {role:"system", content: "You are a helpful AI agent helping users. Try your best to answer the users request."},
+                    {role: "user", content: content}
+                ],
                 max_tokens: 10,
             });
 
             outputText = response.choices[0].message.content || "";
+
+            // indicate that the response was cut short
+            if (response.choices[0].finish_reason === "length") {
+                outputText += "...";
+            }
+
             const endTime = performance.now();
             const elapsedTimeInSeconds = (endTime - startTime) / 1000;
             completionSpeed = Math.round(endTime - startTime);
@@ -68,6 +96,13 @@
             error = `Error: ${(err as Error).message}`;
         } finally {
             isGenerating = false;
+            
+            // process pending request if exists
+            if (pendingRequest && pendingRequest !== content) {
+                const nextRequest = pendingRequest;
+                pendingRequest = null;
+                await generateCompletion(nextRequest);
+            }
         }
     }
 
@@ -79,16 +114,11 @@
     <p class="text-center font-mono text-sm mb-4">Powered by {selectedModel}</p>
     <Textarea 
         bind:value={inputText} 
-        on:input={() => {
-            if (!isGenerating) {
-                generateCompletion();
-            }
-        }} 
+        on:input={() => generateCompletion(inputText)} 
         disabled={isLoading}
         class="w-full" 
         placeholder="Say something..."
     />
-    <pre class="text-lg whitespace-pre-wrap">{outputText}</pre>
     {#if isLoading}
         <p class="text-sm text-slate-600 text-center">{loadingStatus}</p>
     {:else if error}
@@ -101,7 +131,9 @@
             {#if tokensPerSecond !== null}
                 <Badge>{tokensPerSecond} tok/s</Badge>
             {/if}
-            <Badge class="bg-green-700">{selectedModel}</Badge>
+            <Badge>{selectedModel}</Badge>
         </div>
     {/if}
+    <pre class="text-lg font-bold whitespace-pre-wrap">{outputText}</pre>
+
 </div>