Move to characters as major displacement

jloveric · Dec 1, 2023 · 61de75b · 61de75b
1 parent 9a8fb62
commit 61de75b
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/language_interpolation/networks.py b/language_interpolation/networks.py
@@ -346,7 +346,11 @@ def forward(self, x: Tensor) -> Tensor:
         # Scale the input to [-1, 1] where every token is bumped by 1/(2*max_context)
         # the 0th token is -1 and the nth token is 1
         # THIS LOOKS RIGHT!
-        xp = ((0.5 * (x + 1) + self.positional_embedding[: x.shape[1]])*2 - self.max_context)/self.max_context
+
+        # characters are small spacinb
+        # xp = ((0.5 * (x + 1) + self.positional_embedding[: x.shape[1]])*2 - self.max_context)/self.max_context
+        # characters are large spacing
+        xp = ((0.5 * (x + 1)*self.max_context + self.positional_embedding[: x.shape[1]]/(self.max_context-1))*2 - self.max_context)/self.max_context
 
         query = xp
         key = xp