From 3e94f8c75116377d4b1c32b8c674368a27fb2a77 Mon Sep 17 00:00:00 2001
From: Nadav Elyahu <88962733+nelyahu@users.noreply.github.com>
Date: Tue, 26 Dec 2023 18:00:11 +0200
Subject: [PATCH] deepspeed engine: fp16 support validation on init (#4843)

---
 deepspeed/runtime/engine.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/deepspeed/runtime/engine.py b/deepspeed/runtime/engine.py
index 9c9641a1c4cf..4f672f0ba5e2 100644
--- a/deepspeed/runtime/engine.py
+++ b/deepspeed/runtime/engine.py
@@ -1017,6 +1017,9 @@ def _supported_optims(self):
 
     # Validate configuration based on command line arguments
     def _do_sanity_check(self):
+        if self.fp16_enabled() and not get_accelerator().is_fp16_supported():
+            raise ValueError("Type fp16 is not supported.")
+
         expected_optim_types = self._supported_optims()
         expected_optim_types += [type(None), Callable]
         assert isinstance(self.client_optimizer, tuple(expected_optim_types)), \