Spaces:

cfahlgren1
/

inference-proxy

Sleeping

App Files Files Community

cfahlgren1 HF Staff commited on Apr 7

Commit

572e0e2

1 Parent(s): 552cdd6

update the home page

Browse files

Files changed (2) hide show

index.html +97 -0
index.ts +64 -22

index.html ADDED Viewed

	@@ -0,0 +1,97 @@

+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Inference Proxy</title>
+  <!-- Include Tailwind CSS -->
+  <script src="https://cdn.tailwindcss.com"></script>
+  <!-- Include Prism.js for syntax highlighting -->
+  <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.25.0/themes/prism-tomorrow.min.css" rel="stylesheet" />
+  <style>
+    code {
+      font-size: 0.75rem !important;
+    }
+  </style>
+</head>
+<body class="bg-gray-50 text-gray-800 font-sans antialiased">
+  <div class="max-w-4xl mx-auto px-4 py-8 md:py-12">
+    <header class="mb-10">
+      <h1 class="text-3xl md:text-4xl font-bold text-gray-800 mb-2">Inference Proxy</h1>
+      <div class="h-1 w-20 bg-gray-300 rounded"></div>
+    </header>
+    <main>
+      <section class="mb-8">
+        <h2 class="text-xl md:text-2xl font-semibold text-gray-800 mb-4">Setup</h2>
+        <p class="mb-6 text-gray-700">This proxy captures and stores traces from LLM API requests to your personal Hugging Face dataset.</p>
+        <div class="space-y-6">
+          <div class="bg-white rounded-lg shadow-md p-6">
+            <h3 class="text-lg font-medium text-gray-800 mb-3">1. Duplicate Space</h3>
+            <p class="text-gray-600 mb-4">First, duplicate this space to your account to set up your own instance.</p>
+          </div>
+          <div class="bg-white rounded-lg shadow-md p-6">
+            <h3 class="text-lg font-medium text-gray-800 mb-3">2. Set Environment Variables</h3>
+            <p class="text-gray-600 mb-4">Configure these required environment variables in your space settings:</p>
+            <div class="bg-gray-100 p-3 rounded-md mb-4">
+              <code class="text-sm text-gray-700">HF_ACCESS_TOKEN=your_huggingface_token</code>
+            </div>
+            <div class="bg-gray-100 p-3 rounded-md">
+              <code class="text-sm text-gray-700">USER_NAME=your_huggingface_username</code>
+            </div>
+          </div>
+        </div>
+      </section>
+      <section class="mb-8">
+        <h2 class="text-xl md:text-2xl font-semibold text-gray-800 mb-4">Example Usage</h2>
+        <div class="bg-gray-800 rounded-lg shadow-lg overflow-hidden">
+          <div class="flex items-center px-4 py-2 bg-gray-900">
+            <div class="flex space-x-2 mr-2">
+              <div class="w-3 h-3 rounded-full bg-red-500"></div>
+              <div class="w-3 h-3 rounded-full bg-yellow-500"></div>
+              <div class="w-3 h-3 rounded-full bg-green-500"></div>
+            </div>
+            <p class="text-xs text-gray-400">JavaScript</p>
+          </div>
+          <pre class="p-4 overflow-x-auto text-xs font-mono"><code class="language-javascript">import { OpenAI } from "openai";
+const client = new OpenAI({
+  <span class="bg-yellow-700 px-1 rounded">baseURL: "{{HOST_URL}}/fireworks-ai/inference/v1",</span>
+  apiKey: process.env.HF_API_KEY,
+});
+let out = "";
+const stream = await client.chat.completions.create({
+  model: "accounts/fireworks/models/deepseek-v3",
+  messages: [
+    {
+      role: "user",
+      content: "What is the capital of France?",
+    },
+  ],
+  stream: true,
+  max_tokens: 500,
+});
+for await (const chunk of stream) {
+  if (chunk.choices && chunk.choices.length > 0) {
+    const newContent = chunk.choices[0].delta.content;
+    out += newContent;
+    console.log(newContent);
+  }
+}</code></pre>
+        </div>
+      </section>
+    </main>
+  </div>
+  <!-- Include Prism.js JavaScript for syntax highlighting -->
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.25.0/components/prism-core.min.js"></script>
+  <script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.25.0/plugins/autoloader/prism-autoloader.min.js"></script>
+</body>
+</html>

index.ts CHANGED Viewed

@@ -230,7 +230,20 @@ function checkAndFlushTraces() {
 }
 app.get('/', (c) => {
-  return c.text('Hono forwarding proxy running!');
 });
 async function storeStreamedResponse(streamToLog: ReadableStream<Uint8Array>, contentType: string | null, targetUrl: string, traceIndex: number) {
@@ -342,11 +355,18 @@ app.all('*', async (c) => {
     const targetPath = url.pathname;
     const targetUrl = `${TARGET_BASE_URL}${targetPath}${url.search}`;
     // Extract provider from the URL path
     const pathParts = targetPath.split('/');
     const provider = pathParts.length > 1 ? pathParts[1] : 'unknown';
-    console.log(`Forwarding request for ${url.pathname} to ${targetUrl}`);
     const headers = new Headers(c.req.header());
     headers.delete('host');
@@ -379,8 +399,8 @@ app.all('*', async (c) => {
       }
     }
-    const traceIndex = requestTraces.length;
-    const traceEntry: typeof requestTraces[0] = {
       timestamp_start: new Date().toISOString(),
       provider
     };
@@ -396,6 +416,11 @@ app.all('*', async (c) => {
         }
       }
       if (parsedRequestBody.messages) {
         traceEntry.messages = parsedRequestBody.messages;
@@ -413,11 +438,18 @@ app.all('*', async (c) => {
       } else if (parsedRequestBody.parameters) {
         traceEntry.arguments = parsedRequestBody.parameters;
       }
     }
-    requestTraces.push(traceEntry);
-    // Check if we need to flush based on batch size
-    checkAndFlushTraces();
     const response = await fetch(targetUrl, {
       method: c.req.method,
@@ -425,7 +457,10 @@ app.all('*', async (c) => {
       body: requestBody,
     });
-    console.log(`Received response status ${response.status} from ${targetUrl}`);
     c.status(response.status as StatusCode);
     response.headers.forEach((value, key) => {
@@ -441,26 +476,33 @@ app.all('*', async (c) => {
         const [streamForClient, streamForStorage] = response.body.tee();
         const contentType = response.headers.get('content-type');
-        storeStreamedResponse(streamForStorage, contentType, targetUrl, traceIndex).catch(err => {
-             console.error("Error in background stream storage:", err);
-        });
         return stream(c, async (streamInstance) => {
             await streamInstance.pipe(streamForClient);
         });
     } else {
-        console.log(`Received response with no body from ${targetUrl}.`);
-        requestTraces[traceIndex].timestamp_end = new Date().toISOString();
-        // Calculate duration if we have both timestamps
-        if (requestTraces[traceIndex].timestamp_start && requestTraces[traceIndex].timestamp_end) {
-          const startTime = new Date(requestTraces[traceIndex].timestamp_start).getTime();
-          const endTime = new Date(requestTraces[traceIndex].timestamp_end).getTime();
-          requestTraces[traceIndex].duration_ms = endTime - startTime;
         }
-        // Check if we need to flush based on batch size
-        checkAndFlushTraces();
         return c.body(null);
     }
   } catch (error) {

 }
 app.get('/', (c) => {
+  const hostUrl = new URL(c.req.url).origin;
+  try {
+    let html = readFileSync('index.html', 'utf8');
+    // Replace template variables
+    html = html.replace(/{{TARGET_BASE_URL}}/g, TARGET_BASE_URL)
+               .replace(/{{HOST_URL}}/g, hostUrl);
+    return c.html(html);
+  } catch (error) {
+    console.error('Error reading index.html:', error);
+    return c.text('Hono forwarding proxy running!', 500);
+  }
 });
 async function storeStreamedResponse(streamToLog: ReadableStream<Uint8Array>, contentType: string | null, targetUrl: string, traceIndex: number) {
     const targetPath = url.pathname;
     const targetUrl = `${TARGET_BASE_URL}${targetPath}${url.search}`;
+    // Skip trace creation for favicon and other common browser requests
+    const skipTracePatterns = ['/favicon.ico', '/robots.txt'];
+    const shouldSkipTrace = skipTracePatterns.some(pattern => targetPath.includes(pattern));
     // Extract provider from the URL path
     const pathParts = targetPath.split('/');
     const provider = pathParts.length > 1 ? pathParts[1] : 'unknown';
+    // Only log if we're not skipping the trace
+    if (!shouldSkipTrace) {
+      console.log(`Forwarding request for ${url.pathname} to ${targetUrl}`);
+    }
     const headers = new Headers(c.req.header());
     headers.delete('host');
       }
     }
+    let shouldCreateTrace = !shouldSkipTrace;
+    let traceEntry: typeof requestTraces[0] = {
       timestamp_start: new Date().toISOString(),
       provider
     };
         }
       }
+      // Skip traces without a valid model
+      if (!traceEntry.model || traceEntry.model === 'unknown') {
+        shouldCreateTrace = false;
+      }
       if (parsedRequestBody.messages) {
         traceEntry.messages = parsedRequestBody.messages;
       } else if (parsedRequestBody.parameters) {
         traceEntry.arguments = parsedRequestBody.parameters;
       }
+    } else {
+      // Skip traces without a request body
+      shouldCreateTrace = false;
     }
+    const traceIndex = shouldCreateTrace ? requestTraces.length : -1;
+    if (shouldCreateTrace) {
+      requestTraces.push(traceEntry);
+      // Check if we need to flush based on batch size
+      checkAndFlushTraces();
+    }
     const response = await fetch(targetUrl, {
       method: c.req.method,
       body: requestBody,
     });
+    // Only log if we're not skipping the trace
+    if (!shouldSkipTrace) {
+      console.log(`Received response status ${response.status} from ${targetUrl}`);
+    }
     c.status(response.status as StatusCode);
     response.headers.forEach((value, key) => {
         const [streamForClient, streamForStorage] = response.body.tee();
         const contentType = response.headers.get('content-type');
+        if (shouldCreateTrace && traceIndex >= 0) {
+          storeStreamedResponse(streamForStorage, contentType, targetUrl, traceIndex).catch(err => {
+               console.error("Error in background stream storage:", err);
+          });
+        }
         return stream(c, async (streamInstance) => {
             await streamInstance.pipe(streamForClient);
         });
     } else {
+        // Only log if we're not skipping the trace
+        if (!shouldSkipTrace) {
+          console.log(`Received response with no body from ${targetUrl}.`);
+        }
+        if (shouldCreateTrace && traceIndex >= 0) {
+          requestTraces[traceIndex].timestamp_end = new Date().toISOString();
+          // Calculate duration if we have both timestamps
+          if (requestTraces[traceIndex].timestamp_start && requestTraces[traceIndex].timestamp_end) {
+            const startTime = new Date(requestTraces[traceIndex].timestamp_start).getTime();
+            const endTime = new Date(requestTraces[traceIndex].timestamp_end).getTime();
+            requestTraces[traceIndex].duration_ms = endTime - startTime;
+          }
+          // Check if we need to flush based on batch size
+          checkAndFlushTraces();
         }
         return c.body(null);
     }
   } catch (error) {