fix eps for nTransformer

lucidrains · Oct 28, 2024 · 50f9a5e · 50f9a5e
1 parent f56477a
commit 50f9a5e
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 6 deletions.
diff --git a/nGPT_pytorch/nGPTExperimental.py b/nGPT_pytorch/nGPTExperimental.py
@@ -44,15 +44,14 @@ def l2norm(
     eps = None,
     groups = 1
 ):
-    eps = default(eps, 1e-5 if t.dtype == torch.float16 else 1e-10)
-
     if groups > 1:
         t = t.chunk(groups, dim = dim)
         t = torch.stack(t)
 
     if norm_eps == 0.:
-        out = F.normalize(t, dim = dim, p = 2, eps = eps)
+        out = F.normalize(t, dim = dim, p = 2)
     else:
+        eps = default(eps, 1e-5 if t.dtype == torch.float16 else 1e-10)
         norm = t.norm(dim = dim, keepdim = True)
         target_norm = norm.detach().clamp(min = 1. - norm_eps, max = 1. + norm_eps)
         divisor = norm / target_norm

diff --git a/nGPT_pytorch/nTransformer.py b/nGPT_pytorch/nTransformer.py
@@ -44,15 +44,15 @@ def l2norm(
     eps = None,
     groups = 1
 ):
-    eps = default(eps, 1e-5 if t.dtype == torch.float16 else 1e-10)
 
     if groups > 1:
         t = t.chunk(groups, dim = dim)
         t = torch.stack(t)
 
     if norm_eps == 0.:
-        out = F.normalize(t, dim = dim, p = 2, eps = eps)
+        out = F.normalize(t, dim = dim, p = 2)
     else:
+        eps = default(eps, 1e-5 if t.dtype == torch.float16 else 1e-10)
         norm = t.norm(dim = dim, keepdim = True)
         target_norm = norm.detach().clamp(min = 1. - norm_eps, max = 1. + norm_eps)
         divisor = norm / target_norm

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "nGPT-pytorch"
-version = "0.1.12"
+version = "0.1.14"
 description = "nGPT"
 authors = [
     { name = "Phil Wang", email = "lucidrains@gmail.com" }