improved backend

backend/content_generator.py

@@ -163,7 +163,7 @@ class ContentGenerator:
         except Exception as e:
             logger.error(f"Failed to generate simulation for lesson {lesson_index}: {e}")
         
-        return content
+        return content_ids
     
     async def generate_all_content_for_curriculum(
         self,
@@ -171,56 +171,85 @@ class ContentGenerator:
         max_concurrent_lessons: int = 3
     ):
         """Generate all learning content for a curriculum"""
-        # Get curriculum details
-        curriculum_data = await db.get_curriculum(curriculum_id)
-        if not curriculum_data:
-            logger.error(f"Curriculum not found: {curriculum_id}")
-            return
-        
-        # Parse curriculum JSON
         try:
-            curriculum = json.loads(curriculum_data['curriculum_json'])
-            lessons = curriculum.get('sub_topics', [])
-        except json.JSONDecodeError:
-            logger.error(f"Failed to parse curriculum JSON for {curriculum_id}")
-            return
-        
-        # Prepare metadata
-        metadata = {
-            'native_language': curriculum_data['native_language'],
-            'target_language': curriculum_data['target_language'],
-            'proficiency': curriculum_data['proficiency']
-        }
-        
-        logger.info(f"Starting content generation for {len(lessons)} lessons")
-        
-        # Process lessons in batches to avoid overwhelming the API
-        for i in range(0, len(lessons), max_concurrent_lessons):
-            batch = lessons[i:i + max_concurrent_lessons]
-            batch_indices = list(range(i, min(i + max_concurrent_lessons, len(lessons))))
+            # Update status to generating
+            await db.update_content_generation_status(
+                curriculum_id=curriculum_id,
+                status='generating'
+            )
             
-            # Generate content for batch concurrently
-            tasks = [
-                self.generate_content_for_lesson(
+            # Get curriculum details
+            curriculum_data = await db.get_curriculum(curriculum_id)
+            if not curriculum_data:
+                logger.error(f"Curriculum not found: {curriculum_id}")
+                await db.update_content_generation_status(
+                    curriculum_id=curriculum_id,
+                    status='failed',
+                    error_message="Curriculum not found"
+                )
+                return
+        
+            # Parse curriculum JSON
+            try:
+                curriculum = json.loads(curriculum_data['curriculum_json'])
+                lessons = curriculum.get('sub_topics', [])
+            except json.JSONDecodeError:
+                logger.error(f"Failed to parse curriculum JSON for {curriculum_id}")
+                await db.update_content_generation_status(
                     curriculum_id=curriculum_id,
-                    lesson_index=idx,
-                    lesson=lesson,
-                    metadata=metadata
+                    status='failed',
+                    error_message="Failed to parse curriculum JSON"
                 )
-                for idx, lesson in zip(batch_indices, batch)
-            ]
+                return
+        
+            # Prepare metadata
+            metadata = {
+                'native_language': curriculum_data['native_language'],
+                'target_language': curriculum_data['target_language'],
+                'proficiency': curriculum_data['proficiency']
+            }
             
-            results = await asyncio.gather(*tasks, return_exceptions=True)
+            logger.info(f"Starting content generation for {len(lessons)} lessons")
             
-            for idx, result in zip(batch_indices, results):
-                if isinstance(result, Exception):
-                    logger.error(f"Failed to generate content for lesson {idx}: {result}")
-                else:
-                    logger.info(f"Generated content for lesson {idx}: {result}")
-        
-        # Mark curriculum as content generated
-        await db.mark_curriculum_content_generated(curriculum_id)
-        logger.info(f"Completed content generation for curriculum {curriculum_id}")
+            # Process lessons in batches to avoid overwhelming the API
+            for i in range(0, len(lessons), max_concurrent_lessons):
+                batch = lessons[i:i + max_concurrent_lessons]
+                batch_indices = list(range(i, min(i + max_concurrent_lessons, len(lessons))))
+                
+                # Generate content for batch concurrently
+                tasks = [
+                    self.generate_content_for_lesson(
+                        curriculum_id=curriculum_id,
+                        lesson_index=idx,
+                        lesson=lesson,
+                        metadata=metadata
+                    )
+                    for idx, lesson in zip(batch_indices, batch)
+                ]
+                
+                results = await asyncio.gather(*tasks, return_exceptions=True)
+                
+                for idx, result in zip(batch_indices, results):
+                    if isinstance(result, Exception):
+                        logger.error(f"Failed to generate content for lesson {idx}: {result}")
+                    else:
+                        logger.info(f"Generated content for lesson {idx}: {result}")
+            
+            # Mark curriculum as content generated
+            await db.mark_curriculum_content_generated(curriculum_id)
+            await db.update_content_generation_status(
+                curriculum_id=curriculum_id,
+                status='completed'
+            )
+            logger.info(f"Completed content generation for curriculum {curriculum_id}")
+            
+        except Exception as e:
+            logger.error(f"Failed to generate content for curriculum {curriculum_id}: {e}")
+            await db.update_content_generation_status(
+                curriculum_id=curriculum_id,
+                status='failed',
+                error_message=str(e)
+            )
     
     async def process_metadata_extraction(
         self,
@@ -228,44 +257,46 @@ class ContentGenerator:
         query: str,
         metadata: Dict[str, Any],
         user_id: Optional[int] = None,
-        generate_content: bool = True
+        generate_content: bool = True,
+        skip_curriculum_lookup: bool = False
     ) -> Dict[str, Any]:
         """Process a metadata extraction by checking for existing curriculum or generating new one"""
         
-        # Check for existing curriculum first
-        existing_curriculum = await db.find_existing_curriculum(
-            query=query,
-            native_language=metadata['native_language'],
-            target_language=metadata['target_language'],
-            proficiency=metadata['proficiency'],
-            user_id=user_id
-        )
-        
-        if existing_curriculum:
-            # If we found an exact match for this user, return it
-            if existing_curriculum.get('user_id') == user_id:
-                logger.info(f"Found existing curriculum for user {user_id}: {existing_curriculum['id']}")
-                return {
-                    'curriculum_id': existing_curriculum['id'],
-                    'content_generation_started': False,
-                    'cached': True,
-                    'cache_type': 'user_exact_match'
-                }
+        if not skip_curriculum_lookup:
+            # Check for existing curriculum first
+            existing_curriculum = await db.find_existing_curriculum(
+                query=query,
+                native_language=metadata['native_language'],
+                target_language=metadata['target_language'],
+                proficiency=metadata['proficiency'],
+                user_id=user_id
+            )
             
-            # If we found a similar curriculum from another user, copy it
-            elif existing_curriculum.get('is_content_generated') == 1:
-                logger.info(f"Copying existing curriculum {existing_curriculum['id']} for user {user_id}")
-                curriculum_id = await db.copy_curriculum_for_user(
-                    source_curriculum_id=existing_curriculum['id'],
-                    metadata_extraction_id=extraction_id,
-                    user_id=user_id
-                )
-                return {
-                    'curriculum_id': curriculum_id,
-                    'content_generation_started': False,
-                    'cached': True,
-                    'cache_type': 'copied_from_similar'
-                }
+            if existing_curriculum:
+                # If we found an exact match for this user, return it
+                if existing_curriculum.get('user_id') == user_id:
+                    logger.info(f"Found existing curriculum for user {user_id}: {existing_curriculum['id']}")
+                    return {
+                        'curriculum_id': existing_curriculum['id'],
+                        'content_generation_started': False,
+                        'cached': True,
+                        'cache_type': 'user_exact_match'
+                    }
+                
+                # If we found a similar curriculum from another user, copy it
+                elif existing_curriculum.get('is_content_generated') == 1:
+                    logger.info(f"Copying existing curriculum {existing_curriculum['id']} for user {user_id}")
+                    curriculum_id = await db.copy_curriculum_for_user(
+                        source_curriculum_id=existing_curriculum['id'],
+                        metadata_extraction_id=extraction_id,
+                        user_id=user_id
+                    )
+                    return {
+                        'curriculum_id': curriculum_id,
+                        'content_generation_started': False,
+                        'cached': True,
+                        'cache_type': 'copied_from_similar'
+                    }
         
         # No suitable existing curriculum found, generate new one
         logger.info(f"No existing curriculum found, generating new one for user {user_id}")

backend/db.py

@@ -37,12 +37,16 @@ class Database:
         proficiency: str,
         user_id: Optional[int] = None
     ) -> Optional[Dict[str, Any]]:
-        """Find existing curriculum for similar query and metadata"""
+        """Find existing curriculum for exact query and metadata match"""
+        logger.info(f"Looking for curriculum: query='{query[:50]}...', native={native_language}, target={target_language}, proficiency={proficiency}, user_id={user_id}")
+        
         async with aiosqlite.connect(self.db_path) as db:
             db.row_factory = aiosqlite.Row
             
+            # Always look for exact query matches first, prioritizing user-specific matches
             if user_id is not None:
-                # User-specific search: First try to find exact query match for the user
+                # User-specific search: Find exact query match for the user
+                logger.info(f"Searching for exact match for user {user_id}")
                 async with db.execute("""
                     SELECT c.*, m.native_language, m.target_language, m.proficiency, m.title, m.query
                     FROM curricula c
@@ -54,35 +58,27 @@ class Database:
                 """, (user_id, query, native_language, target_language, proficiency)) as cursor:
                     row = await cursor.fetchone()
                     if row:
+                        logger.info(f"Found exact user match: {dict(row)['id']}")
                         return dict(row)
-                
-                # Then try to find similar curriculum with same metadata (any user)
-                async with db.execute("""
-                    SELECT c.*, m.native_language, m.target_language, m.proficiency, m.title, m.query
-                    FROM curricula c
-                    JOIN metadata_extractions m ON c.metadata_extraction_id = m.id
-                    WHERE m.native_language = ? AND m.target_language = ? AND m.proficiency = ?
-                    AND c.is_content_generated = 1
-                    ORDER BY c.created_at DESC
-                    LIMIT 1
-                """, (native_language, target_language, proficiency)) as cursor:
-                    row = await cursor.fetchone()
-                    if row:
-                        return dict(row)
-            else:
-                # User-independent search: Find exact query match regardless of user
-                async with db.execute("""
-                    SELECT c.*, m.native_language, m.target_language, m.proficiency, m.title, m.query
-                    FROM curricula c
-                    JOIN metadata_extractions m ON c.metadata_extraction_id = m.id
-                    WHERE m.query = ? AND m.native_language = ? AND m.target_language = ? AND m.proficiency = ?
-                    ORDER BY c.created_at DESC
-                    LIMIT 1
-                """, (query, native_language, target_language, proficiency)) as cursor:
-                    row = await cursor.fetchone()
-                    if row:
-                        return dict(row)
+            
+            # Look for exact query match from any user (only if the query is exactly the same)
+            logger.info("Searching for exact query match (any user)")
+            async with db.execute("""
+                SELECT c.*, m.native_language, m.target_language, m.proficiency, m.title, m.query
+                FROM curricula c
+                JOIN metadata_extractions m ON c.metadata_extraction_id = m.id
+                WHERE m.query = ? AND m.native_language = ? AND m.target_language = ? AND m.proficiency = ?
+                ORDER BY c.created_at DESC
+                LIMIT 1
+            """, (query, native_language, target_language, proficiency)) as cursor:
+                row = await cursor.fetchone()
+                if row:
+                    logger.info(f"Found exact query match: {dict(row)['id']}")
+                    return dict(row)
+                else:
+                    logger.info("No exact query match found")
         
+        logger.info("No existing curriculum found")
         return None
 
     async def save_metadata_extraction(
@@ -93,7 +89,17 @@ class Database:
     ) -> str:
         """Save extracted metadata and return extraction ID"""
         extraction_id = str(uuid.uuid4())
-        
+
+        # Validate proficiency before inserting into the database
+        allowed_proficiencies = {"beginner", "intermediate", "advanced"}
+        proficiency = metadata.get('proficiency')
+        if proficiency not in allowed_proficiencies:
+            logger.warning(
+                f"Unknown proficiency '{proficiency}' received; defaulting to 'beginner'."
+            )
+            proficiency = "beginner"
+            metadata["proficiency"] = "beginner"
+
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute("""
                 INSERT INTO metadata_extractions 
@@ -127,8 +133,8 @@ class Database:
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute("""
                 INSERT INTO curricula 
-                (id, metadata_extraction_id, user_id, lesson_topic, curriculum_json)
-                VALUES (?, ?, ?, ?, ?)
+                (id, metadata_extraction_id, user_id, lesson_topic, curriculum_json, content_generation_status)
+                VALUES (?, ?, ?, ?, ?, 'pending')
             """, (
                 curriculum_id,
                 metadata_extraction_id,
@@ -164,8 +170,8 @@ class Database:
             # Create new curriculum
             await db.execute("""
                 INSERT INTO curricula 
-                (id, metadata_extraction_id, user_id, lesson_topic, curriculum_json, is_content_generated)
-                VALUES (?, ?, ?, ?, ?, 0)
+                (id, metadata_extraction_id, user_id, lesson_topic, curriculum_json, is_content_generated, content_generation_status)
+                VALUES (?, ?, ?, ?, ?, 0, 'pending')
             """, (
                 new_curriculum_id,
                 metadata_extraction_id,
@@ -192,7 +198,9 @@ class Database:
             # Mark as content generated
             await db.execute("""
                 UPDATE curricula 
-                SET is_content_generated = 1 
+                SET is_content_generated = 1,
+                    content_generation_status = 'completed',
+                    content_generation_completed_at = CURRENT_TIMESTAMP
                 WHERE id = ?
             """, (new_curriculum_id,))
             
@@ -235,11 +243,74 @@ class Database:
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute("""
                 UPDATE curricula 
-                SET is_content_generated = 1 
+                SET is_content_generated = 1,
+                    content_generation_status = 'completed',
+                    content_generation_completed_at = CURRENT_TIMESTAMP
                 WHERE id = ?
             """, (curriculum_id,))
             await db.commit()
     
+    async def update_content_generation_status(
+        self, 
+        curriculum_id: str, 
+        status: str, 
+        error_message: Optional[str] = None
+    ):
+        """Update content generation status for a curriculum"""
+        async with aiosqlite.connect(self.db_path) as db:
+            if status == 'generating':
+                await db.execute("""
+                    UPDATE curricula 
+                    SET content_generation_status = ?,
+                        content_generation_started_at = CURRENT_TIMESTAMP,
+                        content_generation_error = NULL
+                    WHERE id = ?
+                """, (status, curriculum_id))
+            elif status == 'completed':
+                await db.execute("""
+                    UPDATE curricula 
+                    SET content_generation_status = ?,
+                        content_generation_completed_at = CURRENT_TIMESTAMP,
+                        content_generation_error = NULL,
+                        is_content_generated = 1
+                    WHERE id = ?
+                """, (status, curriculum_id))
+            elif status == 'failed':
+                await db.execute("""
+                    UPDATE curricula 
+                    SET content_generation_status = ?,
+                        content_generation_error = ?
+                    WHERE id = ?
+                """, (status, error_message, curriculum_id))
+            else:
+                await db.execute("""
+                    UPDATE curricula 
+                    SET content_generation_status = ?,
+                        content_generation_error = ?
+                    WHERE id = ?
+                """, (status, error_message, curriculum_id))
+            await db.commit()
+    
+    async def get_content_generation_status(self, curriculum_id: str) -> Optional[Dict[str, Any]]:
+        """Get content generation status for a curriculum"""
+        async with aiosqlite.connect(self.db_path) as db:
+            db.row_factory = aiosqlite.Row
+            async with db.execute("""
+                SELECT 
+                    id,
+                    content_generation_status,
+                    content_generation_error,
+                    content_generation_started_at,
+                    content_generation_completed_at,
+                    is_content_generated
+                FROM curricula 
+                WHERE id = ?
+            """, (curriculum_id,)) as cursor:
+                row = await cursor.fetchone()
+                if row:
+                    return dict(row)
+        return None
+    
     async def get_metadata_extraction(self, extraction_id: str) -> Optional[Dict[str, Any]]:
         """Get metadata extraction by ID"""
         async with aiosqlite.connect(self.db_path) as db:

backend/db_cache.py

@@ -86,10 +86,10 @@ class ApiCache:
         else:
             raise TypeError("Cached content must be a JSON string, dict, or list.")
 
-        # 3. Store in cache
+        # 3. Store in cache (use INSERT OR REPLACE to handle duplicates)
         async with aiosqlite.connect(self.db_path) as db:
             await db.execute(
-                "INSERT INTO api_cache (cache_key, category, content_json) VALUES (?, ?, ?)",
+                "INSERT OR REPLACE INTO api_cache (cache_key, category, content_json) VALUES (?, ?, ?)",
                 (cache_key, category, content_to_cache)
             )
             await db.commit()

backend/main.py

@@ -90,6 +90,9 @@ async def health_check():
             },
             status_code=200 if is_healthy else 503
         )
+    except ValueError as ve:
+        logging.error(f"Invalid input: {ve}")
+        raise HTTPException(status_code=400, detail=str(ve))
     except Exception as e:
         return JSONResponse(
             content={
@@ -104,7 +107,7 @@ async def health_check():
 async def repair_database():
     """Repair database issues (admin endpoint)"""
     try:
-        # repair_result = await db.repair_database() # This method doesn't exist on the Database class
+        repair_result = await db_initializer.repair_database()
         
         return JSONResponse(
             content={
@@ -158,11 +161,14 @@ async def extract_metadata(data: MetadataRequest):
     """Extract language learning metadata from user query"""
     logging.info(f"Extracting metadata for query: {data.query[:50]}...")
     try:
-        # Generate metadata using AI, with caching
+        # Generate metadata using AI, with caching (include user context)
         metadata_dict = await api_cache.get_or_set(
             category="metadata",
             key_text=data.query,
             coro=generate_completions.get_completions,
+            context={
+                'user_id': data.user_id
+            },
             prompt=data.query,
             instructions=config.language_metadata_extraction_prompt
         )
@@ -173,7 +179,7 @@ async def extract_metadata(data: MetadataRequest):
             native_language=metadata_dict['native_language'],
             target_language=metadata_dict['target_language'],
             proficiency=metadata_dict['proficiency'],
-            user_id=None  # Make it user-independent
+            user_id=data.user_id  # Use the actual user_id for consistent lookup
         )
 
         if existing_curriculum:
@@ -200,25 +206,37 @@ async def extract_metadata(data: MetadataRequest):
         )
 
         # Process extraction (generate curriculum and start content generation)
-        processing_result = await content_generator.process_metadata_extraction(
-            extraction_id=extraction_id,
-            query=data.query,
-            metadata=metadata_dict,
-            user_id=data.user_id,
-            generate_content=True  # Automatically generate all content
-        )
+        try:
+            processing_result = await content_generator.process_metadata_extraction(
+                extraction_id=extraction_id,
+                query=data.query,
+                metadata=metadata_dict,
+                user_id=data.user_id,
+                generate_content=True,  # Automatically generate all content
+                skip_curriculum_lookup=True  # Skip lookup since we already did it above
+            )
 
-        curriculum_id = processing_result['curriculum_id']
+            curriculum_id = processing_result['curriculum_id']
+            
+            # Update status to generating
+            await db.update_content_generation_status(curriculum_id, 'generating')
 
-        return JSONResponse(
-            content={
-                "message": "Content generation has been initiated.",
-                "curriculum_id": curriculum_id,
-                "status_endpoint": f"/content/status/{curriculum_id}",
-                "cached": False
-            },
-            status_code=202
-        )
+            return JSONResponse(
+                content={
+                    "message": "Content generation has been initiated.",
+                    "curriculum_id": curriculum_id,
+                    "status_endpoint": f"/content/status/{curriculum_id}",
+                    "cached": False
+                },
+                status_code=202
+            )
+        except Exception as content_error:
+            # If content generation fails, update status to failed
+            if 'curriculum_id' in locals():
+                await db.update_content_generation_status(
+                    curriculum_id, 'failed', str(content_error)
+                )
+            raise content_error
     except Exception as e:
         logging.error(f"Error extracting metadata: {e}")
         raise HTTPException(status_code=500, detail=str(e))
@@ -257,6 +275,22 @@ async def get_curriculum(curriculum_id: str = Path(..., description="Curriculum
     
     return JSONResponse(content=curriculum, status_code=200)
 
+@app.get("/content/status/{curriculum_id}")
+async def get_content_generation_status(curriculum_id: str = Path(..., description="Curriculum ID")):
+    """Get content generation status for a curriculum"""
+    status = await db.get_content_generation_status(curriculum_id)
+    if not status:
+        raise HTTPException(status_code=404, detail="Curriculum not found")
+    
+    return JSONResponse(content={
+        "curriculum_id": status['id'],
+        "status": status['content_generation_status'],
+        "error": status['content_generation_error'],
+        "started_at": status['content_generation_started_at'],
+        "completed_at": status['content_generation_completed_at'],
+        "is_content_generated": bool(status['is_content_generated'])
+    }, status_code=200)
+
 
 async def _get_lesson_content_by_type(
     curriculum_id: str,

backend/schema.sql

@@ -26,6 +26,10 @@ CREATE TABLE IF NOT EXISTS curricula (
     lesson_topic TEXT,
     curriculum_json TEXT NOT NULL, -- Full curriculum JSON with 25 lessons
     is_content_generated INTEGER DEFAULT 0, -- Boolean: has all content been generated?
+    content_generation_status TEXT DEFAULT 'pending' CHECK(content_generation_status IN ('pending', 'generating', 'completed', 'failed')),
+    content_generation_error TEXT, -- Store error message if generation fails
+    content_generation_started_at TIMESTAMP,
+    content_generation_completed_at TIMESTAMP,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     FOREIGN KEY (metadata_extraction_id) REFERENCES metadata_extractions(id) ON DELETE CASCADE
 );
@@ -65,6 +69,10 @@ SELECT
     c.id as curriculum_id,
     c.lesson_topic,
     c.is_content_generated,
+    c.content_generation_status,
+    c.content_generation_error,
+    c.content_generation_started_at,
+    c.content_generation_completed_at,
     m.created_at
 FROM metadata_extractions m
 LEFT JOIN curricula c ON m.id = c.metadata_extraction_id
@@ -76,6 +84,10 @@ SELECT
     c.id as curriculum_id,
     c.user_id,
     c.lesson_topic,
+    c.content_generation_status,
+    c.content_generation_error,
+    c.content_generation_started_at,
+    c.content_generation_completed_at,
     COUNT(DISTINCT lc.lesson_index) as lessons_with_content,
     COUNT(DISTINCT CASE WHEN lc.content_type = 'flashcards' THEN lc.lesson_index END) as lessons_with_flashcards,
     COUNT(DISTINCT CASE WHEN lc.content_type = 'exercises' THEN lc.lesson_index END) as lessons_with_exercises,
@@ -92,7 +104,7 @@ CREATE TABLE IF NOT EXISTS api_cache (
     content_json TEXT NOT NULL,
     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY (cache_key, category)
-);
+) WITHOUT ROWID;
 
 -- Index for faster cache lookups
 CREATE INDEX IF NOT EXISTS idx_api_cache_key_category ON api_cache(cache_key, category);