fix: use fp16 dtype for sm75 (#1136)

zhyncs · web-flow · commit 9208591f05c3 · 2024-08-18T00:45:42.000+10:00
diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py
@@ -148,6 +148,11 @@ def load_model(self):
             f"[gpu={self.gpu_id}] Load weight begin. "
             f"avail mem={get_available_gpu_memory(self.gpu_id):.2f} GB"
         )
+        if torch.cuda.get_device_capability()[0] < 8:
+            logger.info(
+                "Compute capability below sm80 use float16 due to lack of bfloat16 support."
+            )
+            self.server_args.dtype = "float16"
 
         monkey_patch_vllm_dummy_weight_loader()
         device_config = DeviceConfig()
