test_agent.py

import logging
import pytest
import requests
from agent import AgentRunner

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Suppress specific warnings
pytestmark = pytest.mark.filterwarnings(
    "ignore::DeprecationWarning:httpx._models"
)

# Constants
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
QUESTIONS_URL = f"{DEFAULT_API_URL}/questions"

@pytest.fixture(scope="session")
def agent():
    """Fixture to create and return an AgentRunner instance."""
    logger.info("Creating AgentRunner instance")
    return AgentRunner()

# @pytest.fixture(scope="session")
# def questions_data():
#     """Fixture to fetch questions from the API."""
#     logger.info(f"Fetching questions from: {QUESTIONS_URL}")
#     try:
#         response = requests.get(QUESTIONS_URL, timeout=15)
#         response.raise_for_status()
#         data = response.json()
#         if not data:
#             logger.error("Fetched questions list is empty.")
#             return []
#         logger.info(f"Fetched {len(data)} questions.")
#         return data
#     except requests.exceptions.RequestException as e:
#         logger.error(f"Error fetching questions: {e}")
#         return []
#     except requests.exceptions.JSONDecodeError as e:
#         logger.error(f"Error decoding JSON response from questions endpoint: {e}")
#         return []
#     except Exception as e:
#         logger.error(f"An unexpected error occurred fetching questions: {e}")
#         return []
#
# class TestAppQuestions:
#     """Test cases for questions from the app."""
#     
#     def test_first_app_question(self, agent, questions_data):
#         """Test the agent's response to the first app question."""
#         if not questions_data:
#             pytest.skip("No questions available from API")
#             
#         first_question = questions_data[0]
#         question_text = first_question.get("question")
#         task_id = first_question.get("task_id")
#         
#         if not question_text or not task_id:
#             pytest.skip("First question is missing required fields")
#             
#         logger.info(f"Testing with app question: {question_text}")
#         
#         response = agent(question_text)
#         logger.info(f"Agent response: {response}")
#         
#         # Check that the response contains the expected information
#         assert "Mercedes Sosa" in response, "Response should mention Mercedes Sosa"
#         assert "studio albums" in response.lower(), "Response should mention studio albums"
#         assert "2000" in response and "2009" in response, "Response should mention the year range"
#         
#         # Verify that a number is mentioned (either as word or digit)
#         import re
#         number_pattern = r'\b(one|two|three|four|five|six|seven|eight|nine|ten|\d+)\b'
#         has_number = bool(re.search(number_pattern, response.lower()))
#         assert has_number, "Response should include the number of albums"
#         
#         # Check for album names in the response
#         known_albums = [
#             "Corazón Libre",
#             "Cantora",
#             "Hermano",
#             "Acústico",
#             "Argentina quiere cantar"
#         ]
#         found_albums = [album for album in known_albums if album in response]
#         assert len(found_albums) > 0, "Response should mention at least some of the known albums"
#         
#         # Check for a structured response
#         assert re.search(r'\d+\.\s+[^(]+\(\d{4}\)', response), \
#             "Response should list albums with years"

class TestBasicCodeAgentCapabilities:
    """Test cases for basic CodeAgent capabilities using examples from the YAML file."""
    
    def test_simple_math_calculation(self, agent):
        """Test the agent's ability to perform basic mathematical operations."""
        # Test the second example from code_agent.yaml
        question = "What is the result of the following operation: 5 + 3 + 1294.678?"
        
        logger.info("Testing simple math calculation capabilities")
        logger.info(f"Question: {question}")
        
        response = agent(question)
        logger.info(f"Agent response: {response}")
        
        # Verify the response contains the correct result
        expected_result = str(5 + 3 + 1294.678)
        assert expected_result in response, f"Response should contain the result {expected_result}"
        
        # Check that the response is a clear answer
        assert "answer" in response.lower(), "Response should indicate it's providing an answer"
    
    def test_document_qa_and_image_generation(self, agent):
        """Test the agent's ability to process a document QA task and generate an image."""
        # Test the first example from code_agent.yaml
        question = "Generate an image of the oldest person in this document."
        
        logger.info("Testing document QA and image generation capabilities")
        logger.info(f"Question: {question}")
        
        response = agent(question)
        logger.info(f"Agent response: {response}")
        
        # Verify the response contains key elements
        assert "Bob Wilson" in response, "Response should identify Bob Wilson as the oldest person"
        assert "60" in response, "Response should mention the age 60"
        assert "engineer" in response, "Response should mention the profession"
        assert "Vancouver" in response, "Response should mention the location"
        
        # Check for image generation related content
        assert "image" in response.lower() or "portrait" in response.lower(), \
            "Response should indicate image generation"
        assert "description" in response.lower(), \
            "Response should include a description of the image"

if __name__ == "__main__":
    pytest.main([__file__, "-v", "-x"])