ignore queries too long

app.py

@@ -85,7 +85,7 @@ def ask(query, timeout=600):
     answer = None
     dead_time = time.time() + timeout
     attempt_times = 0
-    while answer is None and time.time()<dead_time:
+    while answer is None and time.time()<dead_time and attempt_times<10:
         try:
             answer = openai.ChatCompletion.create(
                 model="gpt-3.5-turbo",
@@ -102,6 +102,8 @@ def ask(query, timeout=600):
                     openai.api_key = openai.api_keys[idx]
                     attempt_times += 0
                     print(f"switch api_key")
+                elif "Please reduce the length of the messages." in str(e):
+                    return None
                 else:
                     attempt_times += 1
                     wait_time = int(attempt_times*10)

merge_qas.py

@@ -3,28 +3,23 @@ import json
 import collections
 
 def read_qs():
-    qs = []
-    directory = "./dialogues_set"
+    qs, qas = [], []
+    directory = "./questions"
     filenames = [
-        'dialogues_film.json', 
-        'dialogues_jindong.json', 
-        'dialogues_music.json', 
-        'dialogues_natural.json', 
-        'dialogues_taobao.json', 
-        'dialogues_travel_kd.json'
+        'math_question.txt', 
+        'qa_question.txt', 
+        'summarization_question.txt',
     ]
     for filename in filenames:
         with open(f"{directory}/{filename}", "r", encoding="utf-8") as f:
             for idx,line in enumerate(f):
-                idx2query = json.loads(line)
-                query = idx2query[str(idx)]
-                qs.append(query)
+                qs.append(line.replace("生成摘要","生成中文摘要"))
     print(f"read {len(qs)} queries from files")
     return qs
 
 def read_qas():
     qas = []
-    directory = "./dialogues_set"
+    directory = "./questions"
     
     for filename in os.listdir(directory):
         if filename.endswith(".json") and "qas" in filename:
@@ -53,5 +48,5 @@ if __name__ == "__main__":
     qs = read_qs()
     qas = read_qas()
     qas = merge(qs, qas)
-    with open("./dialogues_set/qas.json", "w", encoding="utf-8") as f:
+    with open("./questions/qas.json", "w", encoding="utf-8") as f:
         f.write(json.dumps(qas, ensure_ascii=False, indent=2))

questions/qas.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3070a38076e44280b5e007a32255000d86488d5b52037a8a7a3917cdaec3450
+size 10170736