File size: 338 Bytes
1575bd1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 |
#!/bin/bash
set -ex
echo "--- Starting setup and server ---"
echo "Starting server..."
exec vllm serve ${HF_MODEL_PATH} \
--host 0.0.0.0 \
--port ${PORT} \
--tensor-parallel-size 1 \
--gpu-memory-utilization 0.8 \
--chat-template-content-format string \
--served-model-name dotsocr-model \
--trust-remote-code
|