From 61de75b04fec89aaeb5c5e937efedc14795efc85 Mon Sep 17 00:00:00 2001 From: jloveric Date: Thu, 30 Nov 2023 20:16:30 -0800 Subject: [PATCH] Move to characters as major displacement --- language_interpolation/networks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/language_interpolation/networks.py b/language_interpolation/networks.py index afa3174..44e51fa 100644 --- a/language_interpolation/networks.py +++ b/language_interpolation/networks.py @@ -346,7 +346,11 @@ def forward(self, x: Tensor) -> Tensor: # Scale the input to [-1, 1] where every token is bumped by 1/(2*max_context) # the 0th token is -1 and the nth token is 1 # THIS LOOKS RIGHT! - xp = ((0.5 * (x + 1) + self.positional_embedding[: x.shape[1]])*2 - self.max_context)/self.max_context + + # characters are small spacinb + # xp = ((0.5 * (x + 1) + self.positional_embedding[: x.shape[1]])*2 - self.max_context)/self.max_context + # characters are large spacing + xp = ((0.5 * (x + 1)*self.max_context + self.positional_embedding[: x.shape[1]]/(self.max_context-1))*2 - self.max_context)/self.max_context query = xp key = xp