sleep for long quests

app.py

@@ -12,17 +12,21 @@ def ask(question, history, behavior):
     openai.request_times += 1
     print(f"request times {openai.request_times}: {datetime.datetime.now()}: {question}")
     try:
+        messages = [
+            {"role":"system", "content":content}
+            for content in behavior
+        ] + [
+            {"role":"user" if i%2==0 else "assistant", "content":content}
+            for i,content in enumerate(history + [question])
+        ]
+        length_messages = num_tokens_from_messages(messages)
+        time_penalty = (length_messages-1000)//10
+        if time_penalty>0:
+            print(f"sleep for {time_penalty:.2f}s for too long a quest: {length_messages}")
+            time.sleep(time_penalty)
         response = openai.ChatCompletion.create(
             model="gpt-3.5-turbo",
-            messages=forget_long_term(
-                [
-                    {"role":"system", "content":content}
-                    for content in behavior
-                ] + [
-                    {"role":"user" if i%2==0 else "assistant", "content":content}
-                    for i,content in enumerate(history + [question])
-                ]
-            )
+            messages=forget_long_term(messages)
         )["choices"][0]["message"]["content"]
         while response.startswith("\n"):
             response = response[1:]
@@ -30,6 +34,7 @@ def ask(question, history, behavior):
         print(e)
         response = 'Timeout! Please wait a few minutes and retry'
     history = history + [question, response]
+    record_dialogue(history)
     return history
 
 def num_tokens_from_messages(messages, model="gpt-3.5-turbo"):
@@ -162,4 +167,5 @@ with gr.Blocks() as demo:
     button_gen.click(fn=predict, inputs=[txt, state, behavior], outputs=[txt, state, chatbot])
     button_clr.click(fn=lambda :([],[]), inputs=None, outputs=[chatbot, state])
 
+demo.queue(concurrency_count=3, max_size=10)
 demo.launch()