fix tokenizer

src/routes/api/models/+server.ts

@@ -1,4 +1,3 @@
-import { env } from "$env/dynamic/private";
 import type { Model, ModelWithTokenizer } from "$lib/types";
 import { json } from "@sveltejs/kit";
 import type { RequestHandler } from "./$types";
@@ -7,7 +6,7 @@ import { dev } from "$app/environment";
 let cache: ModelWithTokenizer[] | undefined;
 
 export const GET: RequestHandler = async ({ fetch }) => {
-	if (cache && dev) {
+	if (cache?.length && dev) {
 		console.log("Skipping load, using in memory cache");
 		return json(cache);
 	}
@@ -15,14 +14,9 @@ export const GET: RequestHandler = async ({ fetch }) => {
 	const apiUrl =
 		"https://huggingface.co/api/models?pipeline_tag=text-generation&filter=conversational&inference_provider=all&limit=100&expand[]=inferenceProviderMapping&expand[]=config&expand[]=library_name&expand[]=pipeline_tag&expand[]=tags&expand[]=mask_token&expand[]=trendingScore";
 
-	const HF_TOKEN = env.HF_TOKEN;
-
 	const res = await fetch(apiUrl, {
 		credentials: "include",
 		headers: {
-			"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:135.0) Gecko/20100101 Firefox/135.0",
-			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language": "en-US,en;q=0.5",
 			"Upgrade-Insecure-Requests": "1",
 			"Sec-Fetch-Dest": "document",
 			"Sec-Fetch-Mode": "navigate",
@@ -47,13 +41,23 @@ export const GET: RequestHandler = async ({ fetch }) => {
 	const promises = compatibleModels.map(async model => {
 		const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
 		const res = await fetch(configUrl, {
+			credentials: "include",
 			headers: {
-				Authorization: `Bearer ${HF_TOKEN}`,
+				"Upgrade-Insecure-Requests": "1",
+				"Sec-Fetch-Dest": "document",
+				"Sec-Fetch-Mode": "navigate",
+				"Sec-Fetch-Site": "none",
+				"Sec-Fetch-User": "?1",
+				"Priority": "u=0, i",
+				"Pragma": "no-cache",
+				"Cache-Control": "no-cache",
 			},
+			method: "GET",
+			mode: "cors",
 		});
 
 		if (!res.ok) {
-			console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
+			// console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
 			return null; // Ignore failed requests by returning null
 		}