UPDATE: cache_word_list

app.py

@@ -203,7 +203,8 @@ def generate(
     out_tokens: List[int] = []
     out_last = 0
 
-    output_cache = collections.deque(maxlen=5)
+    cache_word_list = []
+    cache_word_len = 4
 
     for i in range(max_tokens):
         for n in occurrence:
@@ -216,8 +217,8 @@ def generate(
 
         if token == 0 and token in request.stop_tokens:
             yield {
-                "content": "",
-                "tokens": out_tokens[out_last:],
+                "content": "".join(cache_word_list),
+                "tokens": out_tokens[out_last - cache_word_len :],
                 "finish_reason": "stop:token:0",
                 "state": model_state,
             }
@@ -231,11 +232,12 @@ def generate(
         )
 
         model_tokens.append(token)
+        out_tokens.append(token)
 
         if token in request.stop_tokens:
             yield {
-                "content": "",
-                "tokens": out_tokens[out_last:],
+                "content": "".join(cache_word_list),
+                "tokens": out_tokens[out_last - cache_word_len :],
                 "finish_reason": f"stop:token:{token}",
                 "state": model_state,
             }
@@ -244,8 +246,6 @@ def generate(
             gc.collect()
             return
 
-        out_tokens.append(token)
-
         for xxx in occurrence:
             occurrence[xxx] *= request.penalty_decay
         occurrence[token] = 1 + (occurrence[token] if token in occurrence else 0)
@@ -255,15 +255,15 @@ def generate(
         if "\ufffd" in tmp:
             continue
 
-        output_cache.append(tmp)
-        output_cache_str = "".join(output_cache)
+        cache_word_list.append(tmp)
+        output_cache_str = "".join(cache_word_list)
 
+        print(output_cache_str)
         for stop_words in request.stop:
             if stop_words in output_cache_str:
-
                 yield {
-                    "content": tmp.replace(stop_words, ""),
-                    "tokens": out_tokens[out_last:],
+                    "content": output_cache_str.replace(stop_words, ""),
+                    "tokens": out_tokens[out_last - cache_word_len :],
                     "finish_reason": f"stop:words:{stop_words}",
                     "state": model_state,
                 }
@@ -272,11 +272,12 @@ def generate(
                 gc.collect()
                 return
 
-        yield {
-            "content": tmp,
-            "tokens": out_tokens[out_last:],
-            "finish_reason": None,
-        }
+        if len(cache_word_list) > cache_word_len:
+            yield {
+                "content": cache_word_list.pop(0),
+                "tokens": out_tokens[out_last - cache_word_len :],
+                "finish_reason": None,
+            }
 
         out_last = i + 1