pages/api/functions/vector-store.ts

import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
import { MemoryVectorStore } from 'langchain/vectorstores/memory';
import { HuggingFaceTransformersEmbeddings } from "langchain/embeddings/hf_transformers";

const DEFAULT_CHUNK_SIZE = 1000;
const VECTOR_STORE_SIZE = 5;
const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: DEFAULT_CHUNK_SIZE });

const model = new HuggingFaceTransformersEmbeddings({
    modelName: "Xenova/all-MiniLM-L6-v2",
});

export const similaritySearch = async (input: string, content: string) => {
    const documents = await textSplitter.createDocuments([content]);
  
    const vectorStore = await MemoryVectorStore.fromTexts(    
      [...documents.map(doc => doc.pageContent)],
      [...documents.map((v, k) => k)],
      model
    )
    const queryResult = await vectorStore.similaritySearch(input, VECTOR_STORE_SIZE);
    return queryResult;
}