use unk token instead of eos (#1198)

Signed-off-by: Dillon Laird <dillonalaird@gmail.com>

intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/llava_utils.py

@@ -315,13 +315,10 @@ def preprocess_v1(
     # Mask targets
     sep = conv.sep + conv.roles[1] + ": "
     for conversation, target in zip(conversations, targets):
-        # total_len = int(target.ne(tokenizer.pad_token_id).sum())
+        total_len = int(target.ne(tokenizer.pad_token_id).sum())
 
         rounds = conversation.split(conv.sep2)
 
-        total_len = int(target.ne(tokenizer.pad_token_id).sum()) + \
-                len([rou for rou in rounds if rou !=  ""])
-
         cur_len = 1
         target[:cur_len] = IGNORE_INDEX
         for i, rou in enumerate(rounds):

intel_extension_for_transformers/neural_chat/examples/finetuning/multi_modal/train.py

@@ -219,7 +219,7 @@ def make_inputs_require_grad(module, input, output):
             use_fast=use_fast
             )
 
-    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token = tokenizer.unk_token
 
     # set vision module
     model.get_model().initialize_vision_modules(