MegaTronX commited on
Commit
75ce9b5
·
verified ·
1 Parent(s): bb819e5

Update model_runner.py

Browse files
Files changed (1) hide show
  1. model_runner.py +3 -4
model_runner.py CHANGED
@@ -13,10 +13,9 @@ class ModelRunner:
13
  )
14
  self.llm = Llama(
15
  model_path=model_path,
16
- flash_attn=True,
17
- n_gpu_layers=81,
18
- n_batch=1024,
19
- n_ctx=8192,
20
  )
21
  self.provider = LlamaCppPythonProvider(self.llm)
22
  self.agent = LlamaCppAgent(
 
13
  )
14
  self.llm = Llama(
15
  model_path=model_path,
16
+ n_gpu_layers=0, # ⛔ No GPU
17
+ n_batch=512, # safe default for CPU
18
+ n_ctx=4096 # lower context for CPU environments
 
19
  )
20
  self.provider = LlamaCppPythonProvider(self.llm)
21
  self.agent = LlamaCppAgent(