updated for qdrant cloud data source

app/retriever.py

@@ -4,7 +4,7 @@ from langchain.schema import Document
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
 from langchain.retrievers.document_compressors import CrossEncoderReranker
 from .utils import getconfig
-from .vectorstore_interface import create_vectorstore, VectorStoreInterface
+from .vectorstore_interface import create_vectorstore, VectorStoreInterface, QdrantVectorStore
 import logging
 
 # Load configuration
@@ -198,6 +198,12 @@ def get_context(
             "model_name": config.get("embeddings", "MODEL_NAME")
         }
         
+        # filter support for QdrantVectorStore
+        if isinstance(vectorstore, QdrantVectorStore):
+            filter_obj = create_filter(reports, sources, subtype, year)
+            if filter_obj:
+                search_kwargs["filter"] = filter_obj
+        
         # Perform initial retrieval
         retrieved_docs = vectorstore.search(query, top_k, **search_kwargs)
         

app/utils.py

@@ -1,5 +1,12 @@
 import configparser
 import logging
+import os
+import ast
+import re
+from dotenv import load_dotenv
+
+# Local .env file
+load_dotenv()
 
 def getconfig(configfile_path: str):
     """
@@ -13,4 +20,88 @@ def getconfig(configfile_path: str):
         config.read_file(open(configfile_path))
         return config
     except:
-        logging.warning("config file not found")
+        logging.warning("config file not found")
+
+
+def get_auth(provider: str) -> dict:
+    """Get authentication configuration for different providers"""
+    auth_configs = {
+        "huggingface": {"api_key": os.getenv("HF_TOKEN")},
+        "qdrant": {"api_key": os.getenv("QDRANT_API_KEY")},
+    }
+    
+    provider = provider.lower()  # Normalize to lowercase
+    
+    if provider not in auth_configs:
+        raise ValueError(f"Unsupported provider: {provider}")
+    
+    auth_config = auth_configs[provider]
+    api_key = auth_config.get("api_key")
+    
+    if not api_key:
+        logging.warning(f"No API key found for provider '{provider}'. Please set the appropriate environment variable.")
+        auth_config["api_key"] = None
+    
+    return auth_config
+
+
+def process_content(content: str) -> str:
+    """
+    Process and clean malformed content that may contain stringified nested lists.
+    The test DB on qdrant somehow got a bit malformed in the processing - but probably good to have this anyway
+    
+    Args:
+        content: Raw content from vector store
+        
+    Returns:
+        Cleaned, readable text content
+    """
+    if not content:
+        return content
+    
+    # Check if content looks like a stringified list/nested structure
+    content_stripped = content.strip()
+    if content_stripped.startswith('[') and content_stripped.endswith(']'):
+        try:
+            # Parse as literal list structure
+            parsed_content = ast.literal_eval(content_stripped)
+            
+            if isinstance(parsed_content, list):
+                # Flatten nested lists and extract meaningful text
+                def extract_text_from_nested(obj):
+                    if isinstance(obj, list):
+                        text_items = []
+                        for item in obj:
+                            extracted = extract_text_from_nested(item)
+                            if extracted and extracted.strip():
+                                text_items.append(extracted)
+                        return ' '.join(text_items)
+                    elif isinstance(obj, str) and obj.strip():
+                        return obj.strip()
+                    elif isinstance(obj, dict):
+                        # Handle dict structures if present
+                        text_items = []
+                        for key, value in obj.items():
+                            if isinstance(value, str) and value.strip():
+                                text_items.append(f"{key}: {value}")
+                        return ' '.join(text_items)
+                    else:
+                        return ''
+                
+                extracted_text = extract_text_from_nested(parsed_content)
+                
+                if extracted_text and len(extracted_text.strip()) > 0:
+                    # Clean up extra whitespace and format nicely
+                    cleaned_text = re.sub(r'\s+', ' ', extracted_text).strip()
+                    logging.debug(f"Successfully processed nested list content: {len(cleaned_text)} chars")
+                    return cleaned_text
+                else:
+                    logging.warning("Parsed list content but no meaningful text found")
+                    return content  # Return original if no meaningful text extracted
+                    
+        except (ValueError, SyntaxError) as e:
+            logging.debug(f"Content looks like list but failed to parse: {e}")
+            # Fall through to return original content
+    
+    # For regular text content, just clean up whitespace
+    return re.sub(r'\s+', ' ', content).strip()

app/vectorstore_interface.py

@@ -2,11 +2,11 @@ from abc import ABC, abstractmethod
 from typing import List, Dict, Any, Optional
 from gradio_client import Client
 import logging
-import os
-import time
 from dotenv import load_dotenv
+from .utils import get_auth, process_content
 load_dotenv()
 
+
 class VectorStoreInterface(ABC):
     """Abstract interface for different vector store implementations."""
     
@@ -15,17 +15,17 @@ class VectorStoreInterface(ABC):
         """Search for similar documents."""
         pass
 
+
 class HuggingFaceSpacesVectorStore(VectorStoreInterface):
     """Vector store implementation for Hugging Face Spaces with MCP endpoints."""
     
-    def __init__(self, space_url: str, collection_name: str, hf_token: Optional[str] = None):
-        token = os.getenv("HF_TOKEN")
-        repo_id = space_url
+    def __init__(self, url: str, collection_name: str, api_key: Optional[str] = None):
+        repo_id = url
             
         logging.info(f"Connecting to Hugging Face Space: {repo_id}")
         
-        if token:
-            self.client = Client(repo_id, hf_token=token)
+        if api_key:
+            self.client = Client(repo_id, hf_token=api_key)
         else:
             self.client = Client(repo_id)
             
@@ -50,42 +50,102 @@ class HuggingFaceSpacesVectorStore(VectorStoreInterface):
             logging.error(f"Error searching Hugging Face Spaces: {str(e)}")
             raise e
 
-# class QdrantVectorStore(VectorStoreInterface):
-#     """Vector store implementation for direct Qdrant connection."""
-#     # needs to be generalized for other vector stores (or add a new class for each vector store)
-#     def __init__(self, host: str, port: int, collection_name: str, api_key: Optional[str] = None):
-#         from qdrant_client import QdrantClient
-#         from langchain_community.vectorstores import Qdrant
+class QdrantVectorStore(VectorStoreInterface):
+    """Vector store implementation for direct Qdrant connection."""
+    
+    def __init__(self, url: str, collection_name: str, api_key: Optional[str] = None):
+        from qdrant_client import QdrantClient
+        from sentence_transformers import SentenceTransformer
+        
+        self.client = QdrantClient(
+            url=url,  # Use url parameter which handles full URLs with protocol
+            api_key=api_key
+        )
+        self.collection_name = collection_name
+        # Initialize embedding model as None - will be loaded on first use
+        self._embedding_model = None
+        self._current_model_name = None
         
-#         self.client = QdrantClient(
-#             host=host,
-#             port=port,
-#             api_key=api_key
-#         )
-#         self.collection_name = collection_name
-#          # Embedding model not implemented 
+    def _get_embedding_model(self, model_name: str = None):
+        """Lazy load embedding model to avoid loading if not needed."""
+        if model_name is None:
+            model_name = "BAAI/bge-m3"  # Default from config
+            
+        # Only reload if model name changed
+        if self._embedding_model is None or self._current_model_name != model_name:
+            logging.info(f"Loading embedding model: {model_name}")
+            from sentence_transformers import SentenceTransformer
+            self._embedding_model = SentenceTransformer(model_name)
+            self._current_model_name = model_name
+            logging.info(f"Successfully loaded embedding model: {model_name}")
+            
+        return self._embedding_model
         
-#     def search(self, query: str, top_k: int, **kwargs) -> List[Dict[str, Any]]:
-#         """Search using direct Qdrant connection."""
-#         # Embedding model not implemented 
-#         raise NotImplementedError("Direct Qdrant search needs embedding model configuration")
+    def search(self, query: str, top_k: int, **kwargs) -> List[Dict[str, Any]]:
+        """Search using direct Qdrant connection."""
+        try:
+            # Get embedding model
+            model_name = kwargs.get('model_name')
+            embedding_model = self._get_embedding_model(model_name)
+            
+            # Convert query to embedding
+            logging.info(f"Converting query to embedding using model: {self._current_model_name}")
+            query_embedding = embedding_model.encode(query).tolist()
+            
+            # Get filter from kwargs if provided
+            filter_obj = kwargs.get('filter', None)
+            
+            # Perform vector search
+            logging.info(f"Searching Qdrant collection '{self.collection_name}' for top {top_k} results")
+            search_result = self.client.search(
+                collection_name=self.collection_name,
+                query_vector=query_embedding,
+                query_filter=filter_obj,  # Add filter support
+                limit=top_k,
+                with_payload=True,
+                with_vectors=False
+            )
+            
+            # Format results to match expected output format
+            results = []
+            for hit in search_result:
+                raw_content = hit.payload.get('page_content', '')
+                # Process content to handle malformed nested list structures
+                processed_content = process_content(raw_content)
+                
+                result_dict = {
+                    'answer': processed_content,
+                    'answer_metadata': hit.payload.get('metadata', {}),
+                    'score': hit.score
+                }
+                results.append(result_dict)
+            
+            logging.info(f"Successfully retrieved {len(results)} documents from Qdrant")
+            return results
+            
+        except Exception as e:
+            logging.error(f"Error searching Qdrant: {str(e)}")
+            raise e
 
 def create_vectorstore(config: Any) -> VectorStoreInterface:
     """Factory function to create appropriate vector store based on configuration."""
-    vectorstore_type = config.get("vectorstore", "TYPE")
+    vectorstore_type = config.get("vectorstore", "PROVIDER")
+    
+    # Get authentication config based on provider
+    auth_config = get_auth(vectorstore_type.lower())
     
-    if vectorstore_type.lower() == "huggingface_spaces":
-        space_url = config.get("vectorstore", "SPACE_URL")
+    if vectorstore_type.lower() == "huggingface":
+        url = config.get("vectorstore", "URL")
         collection_name = config.get("vectorstore", "COLLECTION_NAME")
-        hf_token = config.get("vectorstore", "HF_TOKEN", fallback=None)
-        return HuggingFaceSpacesVectorStore(space_url, collection_name, hf_token)
+        api_key = auth_config["api_key"]
+        return HuggingFaceSpacesVectorStore(url, collection_name, api_key)
     
     elif vectorstore_type.lower() == "qdrant":
-        host = config.get("vectorstore", "HOST")
-        port = int(config.get("vectorstore", "PORT"))
+        url = config.get("vectorstore", "URL")  # Use the full URL
         collection_name = config.get("vectorstore", "COLLECTION_NAME")
-        api_key = config.get("vectorstore", "API_KEY", fallback=None)
-        return QdrantVectorStore(host, port, collection_name, api_key)
+        api_key = auth_config["api_key"]
+        # Remove port parameter since it's included in the URL
+        return QdrantVectorStore(url, collection_name, api_key)
     
     else:
         raise ValueError(f"Unsupported vector store type: {vectorstore_type}") 

params.cfg

@@ -1,16 +1,16 @@
 [vectorstore]
-TYPE = huggingface_spaces
-SPACE_URL = GIZ/audit_data
-COLLECTION_NAME = docling
-# For future direct Qdrant usage:
-# TYPE = qdrant
-# HOST = ip address  
-# PORT = 6333
-# COLLECTION_NAME = "collection name"
-# API_KEY = api key for source
+# huggingface_spaces usage:
+# PROVIDER = huggingface
+# URL = GIZ/audit_data
+# COLLECTION_NAME = docling
+
+# direct Qdrant usage:
+PROVIDER = qdrant
+URL = https://de438521-e2dd-43d9-b41b-b2e18299a2c0.europe-west3-0.gcp.cloud.qdrant.io:6333  
+COLLECTION_NAME = allreports
 
 [embeddings]
-MODEL_NAME = BAAI/bge-m3
+MODEL_NAME = BAAI/bge-large-en-v1.5
 # DEVICE = cpu
 
 [retriever]