Update app.py

app.py

@@ -48,7 +48,7 @@ def parse_action(string: str):
 
 VERBOSE = True
 MAX_HISTORY = 100
-MAX_DATA = 1000
+MAX_DATA = 20000
 
 def format_prompt(message, history):
   prompt = "<s>"
@@ -302,21 +302,27 @@ def find_all(purpose,task,history, url):
             #rawp.append([tag.name for tag in soup.find_all()] )
             #print([tag.name for tag in soup.find_all()])
             rawp=(f'RAW TEXT RETURNED: {soup.text}')
+            cnt=0
+            cnt+=len(rawp)
             out.append(rawp)
             out.append("HTML fragments: ")
             q=("a","p","span","content","article")
             for p in soup.find_all(q):
+                cnt+=len(p.string)
+                cnt+=len(p)                
+                
                 out.append([{"string":p.string,"content":p}])
             c=0
             out = str(out)
             rl = len(out)
+            print (cnt)
             print(f'rl:: {rl}')
             #for ea in out:
             for i in str(out):
                 if i == " " or i=="," or i=="\n":
                     c +=1
             print (f'c:: {c}')
-            if rl > MAX_DATA:
+            if c > MAX_DATA:
                 print("compressing...")
                 rawp = compress_data(c,purpose,task,out)    
             print (rawp)