Snapshot

.gitignore

@@ -2,3 +2,4 @@
 /.env
 /.venv/
 /__pycache__/
+/.vscode/

combine.py

@@ -0,0 +1,17 @@
+from functools import reduce
+from typing import Callable
+
+
+def combine[T](items: list[T], combine_fn: Callable[[T, T], T | None]) -> list[T]:
+    def fold_fn(acc: list[T], item: T) -> list[T]:
+        if not acc:
+            return [item]
+
+        combined = combine_fn(acc[-1], item)
+        if combined is not None:
+            return [*acc[:-1], combined]
+        return [*acc, item]
+
+    result = reduce(fold_fn, items, [])
+
+    return result

combine_test.py

@@ -0,0 +1,42 @@
+import pytest
+from combine import combine
+
+
+def test_empty_list():
+    assert combine([], lambda x, y: x + y) == []
+
+def test_single_item():
+    assert combine(["hello"], lambda x, y: x + y) == ["hello"]
+
+def test_two_items():
+    assert combine(["hello", "world"], lambda x, y: x + y) == ["helloworld"]
+
+def test_sum():
+    assert combine([1, 2, 3, 4], lambda x, y: x + y) == [10]
+
+
+def test_add_if_even():
+    def add_if_even(x: int, y: int) -> int | None:
+        if (x + y) % 2 == 0:
+            return x + y
+        return None
+
+    assert combine([1, 3, 1, 4], add_if_even) == [4, 1, 4]
+    assert combine([1, 3, 2, 4], add_if_even) == [10]
+
+
+def test_join_if_same_letter():
+    def join_if_same_letter(x: str, y: str) -> str | None:
+        if x[0] == y[0]:
+            return x + y
+        return None
+
+    assert combine(["hello", "hi", "home", "world", "welcome"], join_if_same_letter) == ["hellohihome", "worldwelcome"]
+
+
+def test_no_combinations():
+    def never_combine(x: int, y: int) -> None:
+        return None
+
+    input_list = [1, 2, 3, 4]
+    assert combine(input_list, never_combine) == input_list

completions.py

@@ -11,6 +11,9 @@ type Tokenizer = PreTrainedTokenizer | PreTrainedTokenizerFast
 def starts_with_space(token: str) -> bool:
     return token.startswith(chr(9601)) or token.startswith(chr(288))
 
+def is_newline(token: str) -> bool:
+    return len(token) == 1 and ord(token[0]) == 266
+
 def split_into_words(token_probs: list[tuple[int, float]], tokenizer: Tokenizer) -> list[Word]:
     words: list[Word] = []
     current_word: list[int] = []
@@ -18,25 +21,32 @@ def split_into_words(token_probs: list[tuple[int, float]], tokenizer: Tokenizer)
     current_word_first_token_index: int = 0
     all_tokens: list[int] = [token_id for token_id, _ in token_probs]
 
-    def append_current_word():
-        if current_word:
-            words.append(Word(current_word,
-                              tokenizer.decode(current_word),
+    def append_word(word):
+        if word:
+            words.append(Word(word,
+                              tokenizer.decode(word),
                               sum(current_log_probs),
                               all_tokens[:current_word_first_token_index]))
 
     for i, (token_id, logprob) in enumerate(token_probs):
         token: str = tokenizer.convert_ids_to_tokens([token_id])[0]
-        if not starts_with_space(token) and token.isalpha():
+        token_str = tokenizer.decode([token_id])
+        print(f"-- {token_id=} {token=} {token_str=} {token_str.isalpha()=} {token_str.isspace()=}")
+        if (not starts_with_space(token) and token_str.isalpha()):
             current_word.append(token_id)
             current_log_probs.append(logprob)
         else:
-            append_current_word()
+            append_word(current_word)
             current_word = [token_id]
             current_log_probs = [logprob]
             current_word_first_token_index = i
+            if is_newline(token):
+                append_word(current_word)
+                current_word = []
+                current_log_probs = []
+                current_word_first_token_index = i
 
-    append_current_word()
+    append_word(current_word)
 
     return words
 

frontend/src/components/App.tsx

@@ -22,6 +22,7 @@ export default function App() {
   const [wordlist, setWordlist] = useState("")
   const [showWholePrompt, setShowWholePrompt] = useState(false)
   const [text, setText] = useState("I just drove to the store to but eggs, but they had some.")
+  // const [text, setText] = useState("1\n2\n3\n4\n5\n")
   const [mode, setMode] = useState<"edit" | "check">("edit")
   const [words, setWords] = useState<Word[]>([])
   const [isLoading, setIsLoading] = useState(false)

frontend/src/components/WordChip.tsx

@@ -44,6 +44,22 @@ export function WordChip({
     setIsExpanded(false);  // Close the dropdown
   };
 
+  console.log(`word: ->${word}<-`);
+
+  let w1;
+  let w2;
+  let w3;
+  // if word contains a newline, render a <br />
+  if (word.includes("\n")) {
+    [w1, w3] = word.split("\n");
+    w2 = "\n";
+    console.log(`split: ${w1} | ${w2} | ${w3}`);
+  } else {
+    w1 = word;
+    w2 = "";
+    w3 = "";
+  }
+
   return (
     <span
       title={logprob.toFixed(2)}
@@ -51,7 +67,9 @@ export function WordChip({
       style={{ position: "relative", cursor: logprob < threshold ? "pointer" : "default" }}
       onClick={handleClick}
     >
-      {word}
+      {w1}
+      {w2 && <br />}
+      {w3}
       {isExpanded && (
         <div
           ref={dropdownRef}

main.py

@@ -34,3 +34,5 @@ def check(text: str):
     return CheckResponse(text=text, words=cached_check_text(text))
 
 app.mount("/", StaticFiles(directory="frontend/public", html=True))
+
+#%%

pyproject.toml

@@ -5,11 +5,13 @@ description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
+    "datasets>=3.0.1",
     "fastapi[standard]>=0.115.2",
     "huggingface-hub>=0.25.2",
     "ipykernel>=6.29.5",
     "ipywidgets>=8.1.5",
     "openai>=1.51.2",
+    "pytest>=8.3.3",
     "torch>=2.4.1",
     "transformers>=4.45.2",
 ]

requirements.txt

@@ -5,3 +5,4 @@ transformers
 torch
 huggingface_hub
 fastapi[standard]
+pytest