-
Notifications
You must be signed in to change notification settings - Fork 2
/
SampleCode.py
60 lines (44 loc) · 2.55 KB
/
SampleCode.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# Sample 1. Prompt Large Language Models (e.g., LLaMA) for BLI. Code grabbed from Prompt4BLI repo: "https://github.com/cambridgeltl/prompt4bli".
# This is the current state-of-the-art BLI approach.
from transformers import LlamaForCausalLM, LlamaTokenizer
tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b",padding_side='left')
tokenizer.pad_token_id = tokenizer.eos_token_id
model = LlamaForCausalLM.from_pretrained("huggyllama/llama-7b").cuda()
model.eval()
TXT = ["The word 'you' in Chinese is 你. The word 'he' in Chinese is"] # In-context learning: a '.' token will follow the target word.
input_encs = tokenizer.batch_encode_plus(TXT,padding=True,add_special_tokens=True,return_tensors="pt")
max_len = 5 + input_encs.input_ids.size(1)
input_encs_cuda = {k:v.cuda() for k,v in input_encs.items()}
outputs = model.generate(**input_encs_cuda,num_beams=5, num_return_sequences=5,max_length=max_len,output_scores=True,return_dict_in_generate=True,do_sample=False)
output_sents = [tokenizer.decode(o, skip_special_tokens=True) for o in outputs.sequences]
target_word = output_sents[0].split(TXT[0])[1].split()[0].split(".")[0]
# Sample 2. Procrustes. Code grabbed from ContrastiveBLI repo: "https://github.com/cambridgeltl/ContrastiveBLI". Please refer to Appendix A.3 of paper "Improving Word Translation via Two-Stage Contrastive Learning".
# A representative mapping-based BLI approach.
import numpy as np
import torch
def procrustes(X_src, Y_tgt):
# X_src: source WEs of size (n, d1), np.array
# Y_tgt: target WEs of size (n, d2), np.array
# n: number of word translation pairs for training
# d1 and d2: dimension of WEs. d1<=d2 (usually d1=d2), typically 300 for fastText WEs and 768 for mBERT WEs.
X_dim = X_src.shape[1]
Y_dim = Y_tgt.shape[1]
U, s, V = np.linalg.svd(np.dot(X_src.T, Y_tgt))
if X_dim == Y_dim:
return np.dot(U, V)
elif X_dim < Y_dim:
zeros_pad = np.zeros([X_dim,Y_dim-X_dim])
U = np.concatenate((U,zeros_pad),axis=1)
return np.dot(U, V)
else:
zeros_pad = np.zeros([X_dim-Y_dim, Y_dim])
V = np.concatenate((V,zeros_pad),axis=0)
return np.dot(U, V)
# Require: X_src of size (n, dim_X), Y_tgt of size (n, dim_Y), torch.tensor
# Output: X_src_mapped, torch.tensor
linear_weights = procrustes(X_src.numpy(), Y_tgt.numpy())
linear_weights = torch.tensor(linear_weights,dtype=torch.float32)
linear_layer = torch.nn.Linear(dim_X,dim_Y,bias=False)
linear_layer.weight.data = linear_weights.t()
linear_layer.requires_grad_(False)
X_src_mapped = linear_layer(X_src)