Update app.py

app.py

@@ -15,7 +15,6 @@ model = AutoModelForCausalLM.from_pretrained(
     "PY007/LiteChat-Preview",
     trust_remote_code=True,
     device_map="auto",
-    torch_dtype=torch.float16
 )
 model.eval()
 
@@ -58,14 +57,14 @@ def bot(history):
     # Take only the most recent context up to the max context length and prepend the
     # system prompt with the messages
     max_tokens = -max_context_length + max_new_tokens + max_sys_tokens
-    inputs = BatchEncoding({
-        k: torch.concat([system_prompt_tokens[k], msg_tokens[k][:, max_tokens:]], dim=-1)
-        for k in msg_tokens
-    }).to('cuda')
     # inputs = BatchEncoding({
     #     k: torch.concat([system_prompt_tokens[k], msg_tokens[k][:, max_tokens:]], dim=-1)
     #     for k in msg_tokens
-    # })
+    # }).to('cuda')
+    inputs = BatchEncoding({
+        k: torch.concat([system_prompt_tokens[k], msg_tokens[k][:, max_tokens:]], dim=-1)
+        for k in msg_tokens
+    })
     # Remove `token_type_ids` b/c it's not yet supported for LLaMA `transformers` models
     if inputs.get("token_type_ids", None) is not None:
         inputs.pop("token_type_ids")