use abstracts as a ranking signal

app.py

@@ -33,12 +33,18 @@ def remove_html(x):
 # deduplicate
 # search per query
 
+# options are abstract search
+# all search
 
-def search(term, limit=10, clean=True, strict=True):
+
+def search(term, limit=10, clean=True, strict=True, abstracts=True):
     term = clean_query(term, clean=clean, strict=strict)
     # heuristic, 2 searches strict and not? and then merge?
-    # https://api.scite.ai/search?mode=citations&term=unit%20testing%20software&limit=10&date_from=2000&date_to=2022&offset=0&supporting_from=1&contrasting_from=0&contrasting_to=0&user_slug=domenic-rosati-keW5&compute_aggregations=true
-    search = f"https://api.scite.ai/search?mode=citations&term={term}&limit={limit}&offset=0&user_slug=domenic-rosati-keW5&compute_aggregations=false"
+    # https://api.scite.ai/search?mode=all&term=unit%20testing%20software&limit=10&date_from=2000&date_to=2022&offset=0&supporting_from=1&contrasting_from=0&contrasting_to=0&user_slug=domenic-rosati-keW5&compute_aggregations=true
+    mode = 'all'
+    if not abstracts:
+        mode = 'citations'
+    search = f"https://api.scite.ai/search?mode={mode}&term={term}&limit={limit}&offset=0&user_slug=domenic-rosati-keW5&compute_aggregations=false"
     req = requests.get(
         search,
         headers={
@@ -49,13 +55,18 @@ def search(term, limit=10, clean=True, strict=True):
         req.json()
     except:
         return [], []
+
+    citation_contexts = [remove_html('\n'.join([cite['snippet'] for cite in doc['citations']])) for doc in req.json()['hits']]
     return (
-        [remove_html('\n'.join([cite['snippet'] for cite in doc['citations']])) for doc in req.json()['hits']],
+        citation_contexts,
         [(doc['doi'], doc['citations'], doc['title'])
-         for doc in req.json()['hits']]
+        for doc in req.json()['hits']]
     )
 
 
+
+
+
 def find_source(text, docs):
     for doc in docs:
         if text in remove_html(doc[1][0]['snippet']):
@@ -150,6 +161,9 @@ st.markdown("""
 """, unsafe_allow_html=True)
 
 with st.expander("Settings (strictness, context limit, top hits)"):
+    support_abstracts = st.radio(
+        "Use abstracts as a ranking signal (if the words are matched in the abstract then the document is more relevant)?",
+        ('yes', 'no'))
     strict_lenient_mix = st.radio(
         "Type of strict+lenient combination: Fallback or Mix? If fallback, strict is run first then if the results are less than context_lim we also search lenient. Mix will search them both and let reranking sort em out",
         ('fallback', 'mix'))
@@ -178,9 +192,9 @@ def run_query(query):
     # could also try fallback if there are no good answers by score...
     limit = top_hits_limit or 100
     context_limit = context_lim or 10
-    contexts_strict, orig_docs_strict = search(query, limit=limit, strict=True)
+    contexts_strict, orig_docs_strict = search(query, limit=limit, strict=True, abstracts=support_abstracts == 'yes')
     if strict_lenient_mix == 'fallback' and len(contexts_strict) < context_limit:
-        contexts_lenient, orig_docs_lenient = search(query, limit=limit, strict=False)
+        contexts_lenient, orig_docs_lenient = search(query, limit=limit, strict=False, abstracts=support_abstracts == 'yes')
         contexts = list(
             set(contexts_strict + contexts_lenient)
         )