🔊 add logs

Signed-off-by: peter szemraj <peterszemraj@gmail.com>

constrained_generation.py

@@ -4,6 +4,7 @@
 
 import copy
 import logging
+
 logging.basicConfig(level=logging.INFO)
 import time
 from pathlib import Path
@@ -11,6 +12,7 @@ from pathlib import Path
 import yake
 from transformers import AutoTokenizer, PhrasalConstraint
 
+
 def get_tokenizer(model_name="gpt2", verbose=False):
     """
     get_tokenizer - returns a tokenizer object
@@ -164,6 +166,8 @@ def constrained_generation(
     -------
     response : str, generated text
     """
+    logging.debug(f" constraining generation with {locals()}")
+
     st = time.perf_counter()
     tokenizer = tokenizer or copy.deepcopy(pipeline.tokenizer)
     tokenizer.add_prefix_space = True
@@ -228,7 +232,9 @@ def constrained_generation(
             force_words_ids=force_words_ids if force_flexible is not None else None,
             max_length=None,
             max_new_tokens=max_generated_tokens,
-            min_length=min_generated_tokens + prompt_length if full_text else min_generated_tokens,
+            min_length=min_generated_tokens + prompt_length
+            if full_text
+            else min_generated_tokens,
             num_beams=num_beams,
             no_repeat_ngram_size=no_repeat_ngram_size,
             num_return_sequences=num_return_sequences,

converse.py

@@ -186,6 +186,7 @@ def gen_response(
         str, the generated text
 
     """
+    logging.debug(f"input args - gen_response() : {locals()}")
     input_len = len(pipeline.tokenizer(query).input_ids)
     if max_length + input_len > 1024:
         max_length = max(1024 - input_len, 8)

utils.py

@@ -7,6 +7,7 @@ from pathlib import Path
 import pprint as pp
 import re
 import shutil  # zipfile formats
+import logging
 from datetime import datetime
 from os.path import basename
 from os.path import getsize, join
@@ -383,3 +384,15 @@ def cleantxt_wrap(ugly_text, all_lower=False):
         return clean(ugly_text, lower=all_lower)
     else:
         return ugly_text
+
+
+def setup_logging(loglevel):
+    """Setup basic logging
+
+    Args:
+      loglevel (int): minimum loglevel for emitting messages
+    """
+    logformat = "[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
+    logging.basicConfig(
+        level=loglevel, stream=sys.stdout, format=logformat, datefmt="%Y-%m-%d %H:%M:%S"
+    )