update

Dockerfile

@@ -2,7 +2,7 @@ FROM nvidia/cuda:12.6.0-runtime-ubuntu22.04
 
 ARG PORT=7860
 ENV PORT=${PORT}
-ENV CONFIG_PATH=config_dev.yaml
+ENV CONFIG_PATH=config_hf.yaml
 ENV SQLALCHEMY_DATABASE_URL=sqlite:////data/logs.db
 
 ENV PYTHONUNBUFFERED=1

alembic.ini

@@ -0,0 +1,119 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+# Use forward slashes (/) also on windows to provide an os agnostic path
+script_location = components/dbo/alembic
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
+# Any required deps can installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to ZoneInfo()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+# version_path_separator = newline
+#
+# Use os.pathsep. Default configuration used for new projects.
+version_path_separator = os
+
+# set to 'true' to search source files recursively
+# in each "version_locations" directory
+# new in Alembic version 1.10
+# recursive_version_locations = false
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = sqlite:///../data/logs.db
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
+# hooks = ruff
+# ruff.type = exec
+# ruff.executable = %(here)s/.venv/bin/ruff
+# ruff.options = check --fix REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

common/db.py

@@ -8,15 +8,7 @@ from sqlalchemy.orm import sessionmaker, scoped_session, Session
 
 from common.configuration import Configuration
 from components.dbo.models.base import Base
-import components.dbo.models.feedback
-import components.dbo.models.acronym
-import components.dbo.models.dataset
-import components.dbo.models.dataset_document
-import components.dbo.models.document
-import components.dbo.models.log
-import components.dbo.models.llm_prompt
-import components.dbo.models.llm_config
-import components.dbo.models.entity
+import common.db_schemas
 
 CONFIG_PATH = os.environ.get('CONFIG_PATH', './config_dev.yaml')
 config = Configuration(CONFIG_PATH)

common/db_schemas.py

@@ -0,0 +1,10 @@
+from components.dbo.models.base import Base
+import components.dbo.models.feedback
+import components.dbo.models.acronym
+import components.dbo.models.dataset
+import components.dbo.models.dataset_document
+import components.dbo.models.document
+import components.dbo.models.log
+import components.dbo.models.llm_prompt
+import components.dbo.models.llm_config
+import components.dbo.models.entity

common/dependencies.py

@@ -4,6 +4,7 @@ from logging import Logger
 from typing import Annotated
 
 from fastapi import Depends
+from components.services.log import LogService
 from ntr_text_fragmentation import InjectionBuilder
 from sqlalchemy.orm import Session, sessionmaker
 
@@ -34,6 +35,11 @@ def get_logger() -> Logger:
     return logging.getLogger(__name__)
 
 
+def get_log_service(
+    db: Annotated[sessionmaker, Depends(get_db)],
+) -> LogService:
+    return LogService(db)
+
 def get_embedding_extractor(
     config: Annotated[Configuration, Depends(get_config)],
 ) -> EmbeddingExtractor:

components/dbo/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

components/dbo/alembic/autoupdate_db.py

@@ -0,0 +1,58 @@
+import os
+import re
+import logging
+
+from sqlalchemy import inspect
+from sqlalchemy.sql import text
+from alembic.config import Config
+from alembic import command
+
+import common.dependencies as DI
+
+
+logger = logging.getLogger(__name__)
+
+def get_old_versions():
+    old_versions = list()
+    migration_dir = 'components/dbo/alembic/versions'
+    for file in os.listdir(migration_dir):
+        if not file.endswith('.py'):
+            continue
+        file_path = os.path.join(migration_dir, file)
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+            match = re.search(
+                r"^(down_revision: Union\[str, None\] = )(None|'[^']*')",
+                content,
+                re.MULTILINE)
+            if match:
+                old_versions.append(match.group(2).replace("'", ""))
+    return old_versions
+
+
+def get_cur_version():
+    session_factory = DI.get_db() 
+    session: Session = session_factory() 
+    
+    try:
+        inspector = inspect(session.bind)
+        if 'alembic_version' not in inspector.get_table_names():
+            return None
+            
+        result = session.execute(text("SELECT version_num FROM alembic_version")).scalar()
+        return result
+        
+    finally:
+        session.close()
+
+
+def update():
+    old_versions = get_old_versions()
+    cur_version = get_cur_version()
+    if cur_version not in old_versions and cur_version is not None:
+        return
+
+
+    logger.info(f"Updating the database from migration {cur_version}")
+    config = Config("alembic.ini")
+    command.upgrade(config, "head")

components/dbo/alembic/env.py

@@ -0,0 +1,81 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+from components.dbo.models.base import Base
+import common.db_schemas
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

components/dbo/alembic/script.py.mako

@@ -0,0 +1,28 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    ${downgrades if downgrades else "pass"}

components/dbo/alembic/versions/12bb1ebae3ff_logs_refactoring.py

@@ -0,0 +1,56 @@
+"""Logs refactoring
+
+Revision ID: 12bb1ebae3ff
+Revises: 6635b061c086
+Create Date: 2025-04-16 12:00:40.247356
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '12bb1ebae3ff'
+down_revision: Union[str, None] = '6635b061c086'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('log', sa.Column('user_request', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('qe_result', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('search_result', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('llm_result', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('llm_settings', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('user_name', sa.String(), nullable=True))
+    op.add_column('log', sa.Column('error', sa.String(), nullable=True))
+    op.drop_column('log', 'query_type')
+    op.drop_column('log', 'llm_classifier')
+    op.drop_column('log', 'llmResponse')
+    op.drop_column('log', 'userRequest')
+    op.drop_column('log', 'userName')
+    op.drop_column('log', 'llmPrompt')
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('log', sa.Column('llmPrompt', sa.VARCHAR(), nullable=True))
+    op.add_column('log', sa.Column('userName', sa.VARCHAR(), nullable=True))
+    op.add_column('log', sa.Column('userRequest', sa.VARCHAR(), nullable=True))
+    op.add_column('log', sa.Column('llmResponse', sa.VARCHAR(), nullable=True))
+    op.add_column('log', sa.Column('llm_classifier', sa.VARCHAR(), nullable=True))
+    op.add_column('log', sa.Column('query_type', sa.VARCHAR(), nullable=True))
+    op.drop_column('log', 'error')
+    op.drop_column('log', 'user_name')
+    op.drop_column('log', 'llm_settings')
+    op.drop_column('log', 'llm_result')
+    op.drop_column('log', 'search_result')
+    op.drop_column('log', 'qe_result')
+    op.drop_column('log', 'user_request')
+    # ### end Alembic commands ###

components/dbo/alembic/versions/6635b061c086_init.py

@@ -0,0 +1,32 @@
+"""Init
+
+Revision ID: 6635b061c086
+Revises: 
+Create Date: 2025-04-09 09:21:08.157225
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6635b061c086'
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

components/dbo/models/feedback.py

@@ -23,5 +23,3 @@ class Feedback(Base):
     llmEstimate = mapped_column(Integer)
 
     log_id = mapped_column(Integer, ForeignKey('log.id'), index=True)
-
-    log = relationship("Log", back_populates="feedback")

components/dbo/models/log.py

@@ -9,11 +9,10 @@ from components.dbo.models.base import Base
 class Log(Base):
     __tablename__ = 'log'
 
-    llmPrompt = mapped_column(String)
-    llmResponse = mapped_column(String)
-    llm_classifier = mapped_column(String)
-    userRequest = mapped_column(String)
-    query_type = mapped_column(String)
-    userName = mapped_column(String)
-
-    feedback = relationship("Feedback", back_populates="log")
+    user_request = mapped_column(String)
+    qe_result = mapped_column(String)
+    search_result = mapped_column(String)
+    llm_result = mapped_column(String)
+    llm_settings = mapped_column(String)
+    user_name = mapped_column(String)
+    error = mapped_column(String)

components/services/document.py

@@ -93,6 +93,7 @@ class DocumentService:
         file_location.parent.mkdir(parents=True, exist_ok=True)
         with open(file_location, 'wb') as buffer:
             buffer.write(file.file.read())
+            file.file.close()
 
         source_format = get_source_format(file.filename)
 

components/services/log.py

@@ -0,0 +1,106 @@
+import logging
+
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from components.dbo.models.log import Log as LogSQL
+from schemas.log import LogCreateSchema, LogFilterSchema, LogSchema, PaginatedLogResponse
+
+
+logger = logging.getLogger(__name__)
+
+
+class LogService:
+    """
+    Сервис для работы с параметрами LLM.
+    """
+
+    def __init__(self, db: Session):
+        logger.info("LogService initializing")
+        self.db = db
+
+    
+    def create(self, log_schema: LogCreateSchema):
+        logger.info("Creating a new log")
+        with self.db() as session:
+            new_log: LogSQL = LogSQL(**log_schema.model_dump())
+            session.add(new_log)
+            session.commit()
+            session.refresh(new_log)
+                
+            return LogSchema(**new_log.to_dict())
+
+
+    def get_list(self, filters: LogFilterSchema) -> PaginatedLogResponse:
+        logger.info(f"Fetching logs with filters: {filters.model_dump(exclude_none=True)}")
+        with self.db() as session:
+            query = session.query(LogSQL)
+            
+            # Применение фильтра по user_name
+            if filters.user_name:
+                query = query.filter(LogSQL.user_name == filters.user_name)
+            
+            # Применение фильтра по диапазону date_created
+            if filters.date_from:
+                query = query.filter(LogSQL.date_created >= filters.date_from)
+            if filters.date_to:
+                query = query.filter(LogSQL.date_created <= filters.date_to)
+            
+            total = query.count()
+            
+            # Применение пагинации
+            offset = (filters.page - 1) * filters.page_size
+            logs = query.offset(offset).limit(filters.page_size).all()
+            
+            # Вычисление общего количества страниц
+            total_pages = (total + filters.page_size - 1) // filters.page_size
+            
+            # Формирование ответа
+            return PaginatedLogResponse(
+                data=[LogSchema(**log.to_dict()) for log in logs],
+                total=total,
+                page=filters.page,
+                page_size=filters.page_size,
+                total_pages=total_pages
+            )
+    
+    def get_by_id(self, id: int) -> LogSchema:
+        with self.db() as session:
+            log: LogSQL = session.query(LogSQL).filter(LogSQL.id == id).first()
+
+            if not log:
+                raise HTTPException(
+                    status_code=400, detail=f"Item with id {id} not found"
+                    )
+
+            return LogSchema(**log.to_dict())
+
+    
+    def update(self, id: int, new_log: LogSchema): 
+        logger.info("Updating log")
+        with self.db() as session:
+            log: LogSQL = session.query(LogSQL).filter(LogSQL.id == id).first()  
+
+            if not log:
+                raise HTTPException(
+                    status_code=400, detail=f"Item with id {id} not found"
+                    )
+        
+            update_data = new_log.model_dump(exclude_unset=True)
+            
+            for key, value in update_data.items():
+                if hasattr(log, key):
+                    setattr(log, key, value)
+            
+            
+            session.commit()
+            session.refresh(log)
+            return log
+
+
+    def delete(self, id: int): 
+        logger.info("Deleting log: {id}")
+        with self.db() as session:
+            log_to_del: LogSQL = session.query(LogSQL).get(id)
+            session.delete(log_to_del)
+            session.commit()

config_hf.yaml

@@ -0,0 +1,39 @@
+common:
+  log_file_path: !ENV ${LOG_FILE_PATH:/data/logs/common.log}
+  log_sql_path: !ENV ${SQLALCHEMY_DATABASE_URL:sqlite:////data/logs.db}
+  log_level: !ENV ${LOG_LEVEL:INFO}
+
+bd:
+  entities:
+    # Варианты: fixed_size, sentence, paragraph, blm_sentence, blm_paragraph
+    strategy_name: !ENV ${ENTITIES_STRATEGY_NAME:paragraph}
+    strategy_params:
+    #   words_per_chunk: 50
+    #   overlap_words: 25
+    #   respect_sentence_boundaries: true
+    process_tables: true
+    neighbors_max_distance: 1
+
+  search:
+    use_qe: true
+    use_vector_search: true
+    vectorizer_path: !ENV ${EMBEDDING_MODEL_PATH:BAAI/bge-m3}
+    device: !ENV ${DEVICE:cuda}
+    max_entities_per_message: 150
+    max_entities_per_dialogue: 300
+
+  files:
+    empty_start: true
+    documents_path: /data/documents
+
+llm:
+  base_url: !ENV ${LLM_BASE_URL:https://api.deepinfra.com}
+  api_key_env: !ENV ${API_KEY_ENV:DEEPINFRA_API_KEY}
+  model: !ENV ${MODEL_NAME:meta-llama/Llama-3.3-70B-Instruct}
+  tokenizer_name: !ENV ${TOKENIZER_NAME:unsloth/Llama-3.3-70B-Instruct}
+  temperature: 0.14
+  top_p: 0.95
+  min_p: 0.05
+  frequency_penalty: -0.001
+  presence_penalty: 1.3
+  seed: 42

docker-compose-example.yaml

@@ -7,7 +7,7 @@ services:
       args:
         PORT: ${PORT:-8885}
     environment:
-      - CONFIG_PATH=/app/config_dev.yaml  # Конфиг
+      - CONFIG_PATH=/app/config_hf.yaml  # Конфиг
       - SQLALCHEMY_DATABASE_URL=sqlite:////data/logs.db  # Путь к БД
       - PORT=${PORT:-8885}
       - HF_HOME=/data/hf_cache 

main.py

@@ -1,8 +1,8 @@
 import logging
 import os
-from contextlib import asynccontextmanager
+from contextlib import asynccontextmanager  # noqa: F401
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated  # noqa: F401
 
 import dotenv
 import uvicorn
@@ -10,29 +10,34 @@ from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from transformers import AutoModel, AutoTokenizer
 
-# from routes.acronym import router as acronym_router
-from common import dependencies as DI
+from common import dependencies as DI  # noqa: F401
 from common.common import configure_logging
 from common.configuration import Configuration
+from routes.auth import router as auth_router
 from routes.dataset import router as dataset_router
 from routes.document import router as document_router
 from routes.entity import router as entity_router
+from routes.evaluation import router as evaluation_router
 from routes.llm import router as llm_router
 from routes.llm_config import router as llm_config_router
 from routes.llm_prompt import router as llm_prompt_router
+from routes.log import router as log_router
 from routes.auth import router as auth_router
+from components.dbo.alembic import autoupdate_db
 
-# from main_before import config
-
+# Защита от автоудаления линтером
+_ = DI
+_ = Annotated
+_ = asynccontextmanager
 
 # Загружаем переменные из .env
 dotenv.load_dotenv()
 
-# from routes.feedback import router as feedback_router
-# from routes.llm import router as llm_router
-# from routes.log import router as log_router
+autoupdate_db.update()
+
 
 CONFIG_PATH = os.environ.get('CONFIG_PATH', 'config_dev.yaml')
+
 print("config path: ")
 print(CONFIG_PATH)
 config = Configuration(CONFIG_PATH)
@@ -48,10 +53,14 @@ configure_logging(
 tmp_path = Path(os.environ.get("APP_TMP_PATH", '.')) / 'tmp.json'
 tmp_path.unlink(missing_ok=True)
 
-print("Downloading model to cache...")
-AutoTokenizer.from_pretrained(config.db_config.search.vectorizer_path)
-AutoModel.from_pretrained(config.db_config.search.vectorizer_path)
-print("Model cached successfully.")
+try:
+    print("Downloading model to cache...")
+    AutoTokenizer.from_pretrained(config.db_config.search.vectorizer_path)
+    AutoModel.from_pretrained(config.db_config.search.vectorizer_path)
+    print("Model cached successfully.")
+except Exception as e:
+    logger.error(f"Error downloading model from huggingface {config.db_config.search.vectorizer_path}: {str(e)}")
+    
             
 app = FastAPI(title="Assistant control panel")
     
@@ -66,20 +75,20 @@ app.add_middleware(
 )
 
 app.include_router(llm_router)
-# app.include_router(log_router)
-# app.include_router(feedback_router)
 app.include_router(dataset_router)
 app.include_router(document_router)
 app.include_router(llm_config_router)
 app.include_router(llm_prompt_router)
 app.include_router(entity_router)
+app.include_router(evaluation_router)
 app.include_router(auth_router)
+app.include_router(log_router)
 
 if __name__ == "__main__":
     uvicorn.run(
         "main:app",
         host="localhost",
-        port=8885,
+        port=7860,
         reload=False,
         workers=2
     )

requirements.txt

@@ -24,4 +24,5 @@ uvicorn==0.34.0
 python-multipart==0.0.20
 python-dotenv==1.1.0
 pyjwt==2.10.1
-fuzzywuzzy[speedup]
+fuzzywuzzy[speedup]
+alembic==1.15.2

routes/dataset.py

@@ -42,31 +42,31 @@ async def get_processing(dataset_service: Annotated[DatasetService, Depends(DI.g
 
 
 
-def try_create_default_dataset(dataset_service: DatasetService):
-    """
-    Создаёт датасет по умолчанию, если такого нет.
-    """
-
-    if not dataset_service.get_default_dataset():
-        print('creating default dataset')
-        if dataset_service.config.db_config.files.empty_start:
-            dataset_service.create_empty_dataset(is_default=True)
-        else:
-            dataset_service.create_dataset_from_directory(
-                is_default=True,
-                directory_with_documents=dataset_service.config.db_config.files.documents_path,
-            )
+# def try_create_default_dataset(dataset_service: DatasetService):
+#     """
+#     Создаёт датасет по умолчанию, если такого нет.
+#     """
+
+#     if not dataset_service.get_default_dataset():
+#         print('creating default dataset')
+#         if dataset_service.config.db_config.files.empty_start:
+#             dataset_service.create_empty_dataset(is_default=True)
+#         else:
+#             dataset_service.create_dataset_from_directory(
+#                 is_default=True,
+#                 directory_with_documents=dataset_service.config.db_config.files.documents_path,
+#             )
             
-@router.get('/try_init_default_dataset')
-async def try_init_default_dataset(dataset_service: Annotated[DatasetService, Depends(DI.get_dataset_service)],
-                                 current_user: Annotated[any, Depends(auth.get_current_user)]):
-    logger.info(f"Handling GET request try_init_default_dataset")
-    try_create_default_dataset(dataset_service)
-    try:
-        return {"ok": True}
-    except Exception as e:
-        logger.error(f"Error creating default dataset: {str(e)}")
-        raise
+# @router.get('/try_init_default_dataset')
+# async def try_init_default_dataset(dataset_service: Annotated[DatasetService, Depends(DI.get_dataset_service)],
+#                                  current_user: Annotated[any, Depends(auth.get_current_user)]):
+#     logger.info(f"Handling GET request try_init_default_dataset")
+#     try_create_default_dataset(dataset_service)
+#     try:
+#         return {"ok": True}
+#     except Exception as e:
+#         logger.error(f"Error creating default dataset: {str(e)}")
+#         raise
     
     
 @router.get('/{dataset_id}')

routes/llm.py

@@ -18,6 +18,8 @@ from components.services.dialogue import DialogueService
 from components.services.entity import EntityService
 from components.services.llm_config import LLMConfigService
 from components.services.llm_prompt import LlmPromptService
+from components.services.log import LogService
+from schemas.log import LogCreateSchema
 
 router = APIRouter(prefix='/llm', tags=['LLM chat'])
 logger = logging.getLogger(__name__)
@@ -53,7 +55,6 @@ def get_last_user_message(chat_request: ChatRequest) -> Optional[Message]:
             msg
             for msg in reversed(chat_request.history)
             if msg.role == "user"
-            and (msg.searchResults is None or not msg.searchResults)
         ),
         None,
     )
@@ -165,12 +166,22 @@ async def sse_generator(request: ChatRequest, llm_api: DeepInfraApi, system_prom
                         predict_params: LlmPredictParams,
                         dataset_service: DatasetService, 
                         entity_service: EntityService,
-                        dialogue_service: DialogueService) -> AsyncGenerator[str, None]:
+                        dialogue_service: DialogueService,
+                        log_service: LogService,
+                        current_user: auth.User) -> AsyncGenerator[str, None]:
     """
     Генератор для стриминга ответа LLM через SSE.
     """
+    # Создаем экземпляр "сквозного" лога через весь процесс
+    log = LogCreateSchema(user_name=current_user.username)
+    
     try:
         old_history = request.history
+        
+        # Сохраняем последнее сообщение в лог как исходный пользовательский запрос
+        last_message = get_last_user_message(request)
+        log.user_request = last_message.content if last_message is not None else None
+        
         new_history = [Message(
             role=msg.role,
             content=msg.content,
@@ -182,6 +193,10 @@ async def sse_generator(request: ChatRequest, llm_api: DeepInfraApi, system_prom
         
         
         qe_result = await dialogue_service.get_qe_result(request.history)
+        
+        # Запись результата qe в лог
+        log.qe_result = qe_result.model_dump_json()
+        
         try_insert_reasoning(request, qe_result.debug_message)
         
         # qe_debug_event = {
@@ -200,6 +215,9 @@ async def sse_generator(request: ChatRequest, llm_api: DeepInfraApi, system_prom
         }
         yield f"data: {json.dumps(qe_event, ensure_ascii=False)}\n\n"
     except Exception as e:
+        log.error = "Error in QE block: " + str(e)
+        log_service.create(log)
+        
         logger.error(f"Error in SSE chat stream while dialogue_service.get_qe_result: {str(e)}", stack_info=True)
         yield "data: {\"event\": \"error\", \"data\":\""+str(e)+"\" }\n\n"
         qe_result = dialogue_service.get_qe_result_from_chat(request.history)
@@ -216,6 +234,9 @@ async def sse_generator(request: ChatRequest, llm_api: DeepInfraApi, system_prom
             )
             text_chunks = await entity_service.build_text_async(chunk_ids, dataset.id, scores)
             
+            # Запись результатов поиска в лог
+            log.search_result = text_chunks
+            
             search_results_event = {
                 "event": "search_results",
                 "data": {
@@ -229,23 +250,35 @@ async def sse_generator(request: ChatRequest, llm_api: DeepInfraApi, system_prom
             
             try_insert_search_results(request, text_chunks)
     except Exception as e:
+        log.error = "Error in vector search block: " + str(e)
+        log_service.create(log)
+        
         logger.error(f"Error in SSE chat stream while searching: {str(e)}", stack_info=True)
         yield "data: {\"event\": \"error\", \"data\":\""+str(e)+"\" }\n\n"
+        
+    log_error = None
     try:          
         # Сворачиваем историю в первое сообщение
         collapsed_request = collapse_history_to_first_message(request)
-                    
+        
+        log.llm_result = ''
+        
         # Стриминг токенов ответа
         async for token in llm_api.get_predict_chat_generator(collapsed_request, system_prompt, predict_params):
             token_event = {"event": "token", "data": token}
-            # logger.info(f"Streaming token: {token}")
+            
+            log.llm_result += token
+            
             yield f"data: {json.dumps(token_event, ensure_ascii=False)}\n\n"
 
         # Финальное событие
         yield "data: {\"event\": \"done\"}\n\n"
     except Exception as e:
+        log.error = "Error in llm inference block: " + str(e)
         logger.error(f"Error in SSE chat stream while generating response: {str(e)}", stack_info=True)
         yield "data: {\"event\": \"error\", \"data\":\""+str(e)+"\" }\n\n"
+    finally:
+        log_service.create(log)
     
 
 @router.post("/chat/stream")
@@ -258,6 +291,7 @@ async def chat_stream(
     entity_service: Annotated[EntityService, Depends(DI.get_entity_service)],
     dataset_service: Annotated[DatasetService, Depends(DI.get_dataset_service)],
     dialogue_service: Annotated[DialogueService, Depends(DI.get_dialogue_service)],
+    log_service: Annotated[LogService, Depends(DI.get_log_service)],
     current_user: Annotated[any, Depends(auth.get_current_user)]
 ):
     try:
@@ -282,7 +316,7 @@ async def chat_stream(
             "Access-Control-Allow-Origin": "*",
         }
         return StreamingResponse(
-            sse_generator(request, llm_api, system_prompt.text, predict_params, dataset_service, entity_service, dialogue_service),
+            sse_generator(request, llm_api, system_prompt.text, predict_params, dataset_service, entity_service, dialogue_service, log_service, current_user),
             media_type="text/event-stream",
             headers=headers
         )

routes/log.py

@@ -1,119 +1,32 @@
 import logging
 from datetime import datetime
-from typing import Annotated, Optional
+from typing import Annotated, List, Optional
 
-from fastapi import APIRouter, Depends, Query
-from sqlalchemy.orm import aliased
-from starlette import status
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
 
 from common import auth
 from common.common import configure_logging
-from common.exceptions import LogNotFoundException
-from components.dbo.models.feedback import Feedback
-from components.dbo.models.log import Log
-from schemas.log import LogCreate
+from components.services.log import LogService
+from schemas.log import LogCreateSchema, LogFilterSchema, LogSchema, PaginatedLogResponse
 import common.dependencies as DI
-from sqlalchemy.orm import sessionmaker
 
 router = APIRouter(tags=['Logs'])
 
 logger = logging.getLogger(__name__)
 configure_logging()
-
-
-@router.get('/logs', status_code=status.HTTP_200_OK)
+    
+@router.get('/logs', response_model=PaginatedLogResponse)
 async def get_all_logs(
-    db: Annotated[sessionmaker, Depends(DI.get_db)], 
-    current_user: Annotated[any, Depends(auth.get_current_user)],
-    date_start: Optional[datetime] = Query(None, alias="date_start"),
-    date_end: Optional[datetime] = Query(None, alias="date_end")
+    filters: Annotated[LogFilterSchema, Depends()],
+    log_service: Annotated[LogService, Depends(DI.get_log_service)], 
+    current_user: Annotated[any, Depends(auth.get_current_user)]
 ):
-    logger.info(f'GET /logs: start')
-    logger.info(f'GET /logs: start_date={date_start}, end_date={date_end}')
-    feedback_alias = aliased(Feedback)
-
-    query = db.query(Log)
-
-    if date_start and date_end:
-        query = query.filter(Log.dateCreated.between(date_start, date_end))
-    elif date_start:
-        query = query.filter(Log.dateCreated >= date_start)
-    elif date_end:
-        query = query.filter(Log.dateCreated <= date_end)
-
-    query = query.outerjoin(feedback_alias, Log.id == feedback_alias.log_id)
-
-    logs_with_feedback = query.all()
-
-    combined_logs = []
-    for log in logs_with_feedback:
-        if log.feedback:
-            for feedback in log.feedback:
-                combined_logs.append(
-                    {
-                        "log_id": log.id,
-                        "llmPrompt": log.llmPrompt,
-                        "llmResponse": log.llmResponse,
-                        "llm_classifier": log.llm_classifier,
-                        "dateCreated": log.dateCreated,
-                        "userRequest": log.userRequest,
-                        "userName": log.userName,
-                        "query_type": log.query_type,
-                        "feedback_id": feedback.feedback_id,
-                        "userComment": feedback.userComment,
-                        "userScore": feedback.userScore,
-                        "manualEstimate": feedback.manualEstimate,
-                        "llmEstimate": feedback.llmEstimate,
-                    }
-                )
-        else:
-            combined_logs.append(
-                {
-                    "log_id": log.id,
-                    "llmPrompt": log.llmPrompt,
-                    "llmResponse": log.llmResponse,
-                    "llm_classifier": log.llm_classifier,
-                    "dateCreated": log.dateCreated,
-                    "userRequest": log.userRequest,
-                    "userName": log.userName,
-                    "query_type": log.query_type,
-                    "feedback_id": None,
-                    "userComment": None,
-                    "userScore": None,
-                    "manualEstimate": None,
-                    "llmEstimate": None,
-                }
-            )
-    return combined_logs
+    logger.info(f'GET /logs')
 
-
-@router.get('/log/{log_id}', status_code=status.HTTP_200_OK)
-async def get_log(db: Annotated[sessionmaker, Depends(DI.get_db)], 
-    current_user: Annotated[any, Depends(auth.get_current_user)], log_id):
-    log = db.query(Log).filter(Log.id == log_id).first()
-    if log is None:
-        raise LogNotFoundException(log_id)
-    return log
-
-
-@router.post('/log', status_code=status.HTTP_201_CREATED)
-async def create_log(log: LogCreate, db: Annotated[sessionmaker, Depends(DI.get_db)]):
-    logger.info("Handling POST request to /log")
     try:
-        new_log = Log(
-            llmPrompt=log.llmPrompt,
-            llmResponse=log.llmResponse,
-            llm_classifier=log.llm_classifier,
-            userRequest=log.userRequest,
-            userName=log.userName,
-        )
-
-        db.add(new_log)
-        db.commit()
-        db.refresh(new_log)
-
-        logger.info(f"Successfully created log with ID: {new_log.id}")
-        return new_log
-    except Exception as e:
-        logger.error(f"Error creating log: {str(e)}")
+        return log_service.get_list(filters)
+    except HTTPException as e:
         raise e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

schemas/entity.py

@@ -41,6 +41,7 @@ class EntityTextRequest(BaseModel):
     chunk_scores: Optional[dict[str, float]] = None
     include_tables: bool = True
     max_documents: Optional[int] = None
+    dataset_id: int
 
 
 class EntityTextResponse(BaseModel):

schemas/log.py

@@ -1,12 +1,51 @@
-from typing import Optional
+from datetime import datetime
+from typing import List, Optional
 
 from pydantic import BaseModel
 
 
-class LogCreate(BaseModel):
-    llmPrompt: Optional[str] = None
-    llmResponse: Optional[str] = None
-    userRequest: str
-    llm_classifier: Optional[str] = None
-    query_type: Optional[str] = None
-    userName: str
+class LogSchema(BaseModel):
+    id: int
+    date_created: datetime
+    user_request: Optional[str] = None
+    qe_result: Optional[str] = None
+    search_result: Optional[str] = None
+    llm_result: Optional[str] = None
+    llm_settings: Optional[str] = None
+    user_name: Optional[str] = None
+    error: Optional[str] = None
+    
+class LogCreateSchema(BaseModel):
+    user_request: Optional[str] = None
+    qe_result: Optional[str] = None
+    search_result: Optional[str] = None
+    llm_result: Optional[str] = None
+    llm_settings: Optional[str] = None
+    user_name: Optional[str] = None
+    error: Optional[str] = None
+
+class LogFilterSchema(BaseModel):
+    user_name: Optional[str] = None
+    date_from: Optional[datetime] = None
+    date_to: Optional[datetime] = None
+    
+    page: int = 1  # Номер страницы, по умолчанию 1
+    page_size: int = 50  # Размер страницы, по умолчанию 50
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "user_name": "demo",
+                "date_from": "2024-01-01T00:00:00",
+                "date_to": "2026-12-31T23:59:59",
+                "page": 1,
+                "page_size": 50
+            }
+        }
+
+class PaginatedLogResponse(BaseModel):
+    data: List[LogSchema]
+    total: int
+    page: int
+    page_size: int
+    total_pages: int