Update app.py

app.py

@@ -48,7 +48,7 @@ def parse_action(string: str):
 
 VERBOSE = True
 MAX_HISTORY = 100
-MAX_DATA = 10000
+MAX_DATA = 1000
 
 def format_prompt(message, history):
   prompt = "<s>"
@@ -289,7 +289,8 @@ def find_all(purpose,task,history, url):
     print (f"trying URL:: {url}")        
     try:
         if url != "" and url != None:    
-            rawp = []
+            #rawp = []
+            out = []
             source = requests.get(url)
             #source = urllib.request.urlopen(url).read()
             soup = bs4.BeautifulSoup(source.content,'lxml')
@@ -303,18 +304,22 @@ def find_all(purpose,task,history, url):
             print(soup.title.parent.name)
             #rawp.append([tag.name for tag in soup.find_all()] )
             print([tag.name for tag in soup.find_all()])
-            rawp=soup
+            rawp=(f'RAW TEXT RETURNED:\n*********\n{soup.text}\n*********\n')
+            out.append(rawp)
+            q=("a","p","span","content","article")
+            for p in soup.find_all(f'{q}'):
+                out.append([{q:p.string,"additional":z,"parent":p.parent.name,"previous":[b for b in p.previous],"first-child":[b.name for b in p.children],"content":p}])
             c=0
-            rl = len(rawp)
-            print(rl)
-            for i in str(rawp):
-                if i == " " or i==",":
-                    c +=1
-            
-            print (c)
+            rl = len(out)
+            print(f'rl:: {rl}')
+            for ea in out:
+                for i in str(ea):
+                    if i == " " or i==",":
+                        c +=1
+            print (f'c:: {c}')
             if rl > MAX_DATA:
                 print("compressing...")
-                rawp = compress_data(rl,purpose,task,rawp)    
+                rawp = compress_data(c,purpose,task,out)    
             print (rawp)
             history += "observation: the search results are:\n {}\n".format(rawp)
             task = "complete?"