temp.ipynb

# Preparation
## Library
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from sklearn.model_selection import train_test_split

from torch.utils.data import TensorDataset, DataLoader
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from torch.utils.data.sampler import WeightedRandomSampler
from torch.optim import lr_scheduler

from sklearn.preprocessing import StandardScaler

from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report
from sklearn.metrics import f1_score, precision_score, recall_score
from transformers import AdamW, get_linear_schedule_with_warmup
from sklearn.inspection import permutation_importance


from fetchData import fetchdata, cv_events
import __MLP
# from __MLP import getSamplers, convert_df_to_unsqueezed_tensor, train_sequential, clf_report
import random

import __Preprocessing
from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier

pd.set_option('display.max_columns', None)
if torch.cuda.is_available():       
    device = torch.device("cuda")
    print(f'There are {torch.cuda.device_count()} GPU(s) available.')
    print('Device name:', torch.cuda.get_device_name(0))

else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")
seed = 42
if torch.cuda.is_available():
    torch.cuda.manual_seed_all(seed_value)


torch.manual_seed(42)
torch.cuda.manual_seed(42)
np.random.seed(42)
random.seed(42)
torch.backends.cudnn.deterministic=True
torch.backends.cudnn.benchmark = False

def _init_fn(worker_id):
    np.random.seed(int(seed))
## Data
# Final
pheme_sparse_final = pd.read_csv('./data/_PHEME_sparse.csv')
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
pheme_pos_final = pd.read_csv('./data/_PHEME_postags.csv')
pheme_thread_final_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')
pheme_thread_final_std = pd.read_csv('./data/_PHEME_thread_std.csv')

ext_pos_final = pd.read_csv('./data/_PHEMEext_postags.csv')
ext_sparse_final = pd.read_csv('./data/_PHEMEext_sparse.csv')
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target
ext_thread_final_avg = pd.read_csv('./data/_PHEMEext_thread_avg.csv')
ext_thread_final_std = pd.read_csv('./data/_PHEMEext_thread_std.csv')

pheme_bert_simple_normal = pd.read_csv('./data/_PHEME_Bert_final_brackets.csv')
pheme_bert_brackets_normal = pd.read_csv('./data/_PHEME_Bert_final_brackets_nrmzd.csv')
ext_bert_simple_normal = pd.read_csv('./data/_PHEMEext_Bert_final_brackets.csv')
ext_bert_brackets_normal = pd.read_csv('./data/_PHEMEext_Bert_final_brackets_nrmzd.csv')

pheme_event = pd.read_csv('./data/_PHEME_text.csv')['Event']
ext_event = pd.read_csv('./data/_PHEMEext_text.csv').Event
pheme_AVGw2v = pd.read_csv('./data/_PHEME_text_AVGw2v.csv').drop(['token'],axis=1)
ext_AVGw2v = pd.read_csv('./data/_PHEMEext_text_AVGw2v.csv').drop(['token'],axis=1)
print("pheme_sparse_final", pheme_sparse_final.shape)
print(pheme_sparse_final.columns, "\n")
print("pheme_pos_final", pheme_pos_final.shape)
x = zip(pheme_pos_final.columns.values, ['Interjection', 'Hashtag', 'coordinating conjunction', 'punctuation', 'at-mention', 'proper noun', 'adjective' 'determiner' 'other', 'nominal + verbal',
    'common noun', 'pronoun', 'pre/postposition/subordinating conjunction', 'adverb', 'nominal + possessive', 'verb particle', 'URL or email', 'verb incl. copula, auxiliaries',
 'existential there, predeterminers', 'proper noun + possessive', 'discourse marker'])
print(set(x), "\n")
print("pheme_thread_final", pheme_thread_final_avg.shape)
print(pheme_thread_final_avg.columns, "\n")
print("ext_bert_simple_normal", ext_bert_simple_normal.shape)
print(pheme_bert_simple_normal.columns, "\n")

# cv = cv_events(pd.concat([pheme_sparse, pheme_y, pheme_event],axis=1))
# X = cv[3][1].drop(['target', 'Event'],axis=1)
# y = cv[3][1].target
# val_X = cv[3][0].drop(['target', 'Event'],axis=1)
# val_y = cv[3][0].target
def set_seed(seed_value=42):
    """Set seed for reproducibility.
    """
    random.seed(seed_value)
    np.random.seed(seed_value)
    torch.manual_seed(seed_value)
    torch.cuda.manual_seed_all(seed_value)
# Feature Importance
def f_imp(X, y):
    forest = ExtraTreesClassifier(n_estimators=250,
                                random_state=3)

    forest.fit(X, y)
    importances = forest.feature_importances_
    std = np.std([tree.feature_importances_ for tree in forest.estimators_],
                axis=0)
    indices = np.argsort(importances)[::-1]

    # Print the feature ranking
    print("Feature ranking:")

    for f in range(X.shape[1]):
        print("%d. feature %d: %s (%f)" % (f + 1, indices[f], X.columns[indices[f]], importances[indices[f]]))

    # Plot the impurity-based feature importances of the forest
    plt.figure(figsize=(12, 7))
    plt.title("Feature importances")
    plt.bar( X.columns[indices], importances[indices], color="r")
    plt.tight_layout()
    plt.show()

def f_imp2(X, y):
    rf = RandomForestClassifier(random_state=42)
    rf.fit(X, y)
    result = permutation_importance(rf, X, y, n_repeats=10,
                                    random_state=42, n_jobs=2)
    sorted_idx = result.importances_mean.argsort()

    fig, ax = plt.subplots()
    fig.set_size_inches(15, 7)
    ax.boxplot(result.importances[sorted_idx].T,
            vert=False, labels=X.columns[sorted_idx])
    ax.set_title("Permutation Importances (test set)",fontdict={'fontsize': 'x-large', 'fontweight': 'medium'})
    ax.labelsize='x-large'
    fig.titlesize='x-large'
    fig.tight_layout()
    plt.show()
f_imp(pheme_pos_final, pheme_y)
f_imp(pheme_sparse_final, pheme_y)
f_imp2(pheme_pos_final, pheme_y)
f_imp2(pheme_sparse_final, pheme_y)
f_imp2(pheme_thread_final, pheme_y)
# Sparse-MLP
# import __MLP
pheme_sparse = pd.read_csv('./data/_PHEME_sparse.csv')
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
ext_sparse = pd.read_csv('./data/_PHEMEext_sparse.csv')
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target

scaler = StandardScaler()
pheme_scaled = pd.DataFrame(scaler.fit_transform(pheme_sparse))
ext_scaled = pd.DataFrame(scaler.transform(ext_sparse))

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_scaled,pheme_y,ext_scaled,ext_y)

train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
# tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_sparse,pheme_y,ext_sparse,ext_y)
# train_dataset = TensorDataset(tensor_x1,tensor_y1)
# test_dataset = TensorDataset(tensor_x2,tensor_y2)
# print(tensor_x1.size(),tensor_x2.size())
# print(tensor_y1.size(),tensor_y2.size())
batch_size = 16
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class sparse_model(nn.Module):
    def __init__(self):
        super(sparse_model, self).__init__() # 1*20
        self.fc1 = nn.Linear(36, 8, bias=True) # 420
        # self.fc2 = nn.Linear(12, 8, bias=True)
        self.fc3 = nn.Linear(8, 1,bias=True)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = self.drop_3(F.elu(self.fc1(x)))
        # x = self.drop_2(F.elu(self.fc2(x)))
        x = self.drop_3(self.fc3(x))
        # x = self.fc3(x)
        # x = self.fc3(x)
        return x
model_sparse = sparse_model()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model_sparse.parameters(), lr=0.01, momentum=0.9)
optimizer = optim.Adam(model_sparse.parameters(), lr=5e-5, eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_sparse_model.pt"
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_sparse, num_epochs=100, patience=15, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
def predict(model, criterion, val_dataloader, val_size):
    model.eval()
    val_label_list = []
    # val_preds_list = []
    running_val_preds = []

    with torch.no_grad():
        val_loss = 0.0
        val_corrects = 0
        f1_running = 0
        for j, val in enumerate(val_dataloader, 0):
            val_x, val_label = val
            val_x, val_label = val_x.float(), val_label.float()
            val_outputs = model(val_x)
            val_preds = val_outputs.squeeze(1) > 0.0
            f1_running += (f1_score(val_label, val_preds,zero_division=True) * val_x.size(0))
            v_loss = criterion(val_outputs, val_label.unsqueeze(1))
            val_loss += (v_loss.item() * val_x.size(0))
            val_corrects += torch.sum(val_preds == val_label)
            val_label_list.append(val_label)
            running_val_preds.append(val_preds)

    running_val_preds = torch.cat(running_val_preds, 0)
    val_label_list = torch.cat(val_label_list, 0)
    val_corrects = val_corrects
    val_loss = val_loss/val_size
    val_acc = val_corrects.double().numpy() / val_size
    f1_running /= val_size
    print("accuracy_score:\t\t%.4f" % val_acc)
    print('Precision Score:\t%.4f' % precision_score(val_label_list,running_val_preds))
    print('Recall Score:\t\t%.4f' % recall_score(val_label_list,running_val_preds))
    print("f1_score:\t\t%.4f" % f1_running)
    print("Test_loss:\t\t%.4f" % val_loss)

import __MLP
model = sparse_model()
PATH = './Model/state_dict_sparse_model.pt'
model.load_state_dict(torch.load(PATH))
predict(model_sparse, criterion, test_dataloader, test_size)
# W2V-MLP
pheme_AVGw2v = pd.read_csv('./data/_PHEME_text_AVGw2v.csv').drop(['token'],axis=1)
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target
ext_AVGw2v = pd.read_csv('./data/_PHEMEext_text_AVGw2v.csv').drop(['token'],axis=1)
tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_AVGw2v,pheme_y,ext_AVGw2v,ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class W2V_net(nn.Module):
    def __init__(self):
        super(W2V_net, self).__init__() # 1*20
        self.fc1 = nn.Linear(200, 12, bias=True) # 420
        # self.fc2 = nn.Linear(12, 1, bias=True)
        self.fc3 = nn.Linear(12, 1)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = self.F.elu(self.fc1(x))
        # x = self.drop_2(F.elu(self.fc2(x)))
        x = self.fc3(x)
        return x
model_w2v = W2V_net()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(task1_model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_w2v.parameters(), lr=5e-5, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  

train_loss = []
train_accuracy = []

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_w2v_model.pt"
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_w2v, num_epochs=100, patience=20, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = W2V_net()
PATH = './Model/state_dict_w2v_model.pt'
model.load_state_dict(torch.load(PATH))
predict(model, criterion, test_dataloader, test_size)
# BERT - MLP
ext_bert = pd.read_csv('./data/_PHEMEext_Bert_simple.csv')
pheme_bert = pd.read_csv('./data/_PHEME_Bert_simple.csv')

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_bert, pheme_y, ext_bert, ext_y)
# tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_bert, pheme_y, ext_bert, ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())

batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape)
print(tensor_y1.shape)
print("Train Size",train_size,"Test Size",test_size)
class BERT_net(nn.Module):
    def __init__(self):
        super(BERT_net, self).__init__() # 1*20
        self.fc1 = nn.Linear(768, 50, bias=True) # 420
        self.fc2 = nn.Linear(50, 8, bias=True)
        self.fc3 = nn.Linear(8, 1)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = F.elu(self.fc1(x))
        x = F.elu(self.fc2(x))
        x = self.fc3(x)
        return x
model_bert = BERT_net()
import torch.optim as optim

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_bert.parameters(), lr=5e-5, eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_bert_model.pt"
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_bert, num_epochs=100, patience=25, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = BERT_net()
PATH = './Model/state_dict_bert_model.pt'
model.load_state_dict(torch.load(PATH))
predict(model, criterion, test_dataloader, test_size)
# Thread
cv = __Preprocessing.cv_events(pd.concat([pheme_bert_simple, pheme_y, pheme_event],axis=1))
X = cv[3][1].drop(['target', 'Event'],axis=1)
y = cv[3][1].target
val_X = cv[3][0].drop(['target', 'Event'],axis=1)
val_y = cv[3][0].target

scaler = StandardScaler()
pheme_thread_scaled = pd.DataFrame(scaler.fit_transform(pheme_thread_avg))
ext_thread_scaled = pd.DataFrame(scaler.transform(ext_thread_avg))

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_thread_scaled,pheme_y,ext_thread_scaled,ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class thread_model(nn.Module):
    def __init__(self):
        super(thread_model, self).__init__() # 1*20
        self.fc1 = nn.Linear(38, 12, bias=True) # 420
        # self.fc2 = nn.Linear(12, 8, bias=True)
        self.fc3 = nn.Linear(12, 1)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = self.drop_3(F.elu(self.fc1(x)))
        # x = F.elu(self.fc2(x))
        x = self.drop_2(self.fc3(x))
        return x
model_thread = thread_model()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model_sparse.parameters(), lr=0.01, momentum=0.9)
optimizer = optim.Adam(model_thread.parameters(), lr=5e-2, eps=1e-8, weight_decay=1e-4)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_thread_model.pt"
def train1(model, num_epochs, criterion, optimizer, scheduler, train_loader, train_size, test_loader=None, test_size=None, patience=5, PATH='./state_dict_model.pt'):
    set_seed(42)
    train_loss = []
    patience_count = 0
    train_accuracy = []
    prev_loss = 10
    best_loss = 10.0
    val_corrects_list = []
    val_loss_list = []
    val_acc_list = []
    for epoch in range(num_epochs):  # loop over the dataset multiple times
        # print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        # print('-' * 10)
        running_corrects = 0.0
        running_loss = 0.0
        model.train()  # Set model to training mode
        for i, data in enumerate(train_loader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data
            inputs, labels = inputs.float(), labels.float()

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)

            #  _, predictions = torch.max(outputs.data, 1) won’t work if your output only contains a single output unit.
            # _, preds = torch.max(outputs, 1)
            # print(outputs.flatten().size())
            preds = outputs.squeeze(1) > 0.0

            loss = criterion(outputs, labels.unsqueeze(1))
            loss.backward()
            optimizer.step()

            # step function
            scheduler.step()

            running_loss += loss.item() * inputs.size(0)
            running_corrects += torch.sum(preds == labels.data)

        epoch_loss = running_loss / train_size
        # print(running_loss)
        # print(train_size)
        epoch_acc = running_corrects.double() / train_size
        train_loss.append(epoch_loss)
        train_accuracy.append(epoch_acc)

        if (epoch % 2 == 0):
            print('Epoch {}/{}\tTrain) Acc: {:.4f}, Loss: {:.4f}'.format(epoch,
                                                                         num_epochs - 1, epoch_acc, epoch_loss))

        if (test_loader != None):
            model.eval()
            with torch.no_grad():
                val_loss = 0.0
                val_corrects = 0
                val_preds_list = []
                val_label_list = []
                for j, val in enumerate(test_loader, 0):
                    val_x, val_label = val
                    val_x, val_label = val_x.float(), val_label.float()
                    val_outputs = model(val_x)
                    # _, val_preds = torch.max(val_outputs, 1)
                    val_preds = val_outputs.squeeze(1) > 0.0
                    # print("val_preds:\n", val_preds)
                    # print("val_labels:\n", val_label)

                    val_preds_list.append(val_preds)
                    val_label_list.append(val_label)
                    v_loss = criterion(val_outputs, val_label.unsqueeze(1))
                    val_loss += (v_loss.item() * val_x.size(0))
                    val_corrects += torch.sum(val_preds == val_label)
                    # print("val_corrects:\n", val_corrects)
                    # accuracy = (preds == b_labels).cpu().numpy().mean() * 100

                if (epoch % 2 == 0):
                    val_preds_list = torch.cat(val_preds_list, 0)
                    val_label_list = torch.cat(val_label_list, 0)
                    # print("\t\tValidation) Acc: {:.4f} Loss:{:.4f} F1 score: {:4f}".format(val_corrects/test_size, val_loss/test_size, f1_score(val_label_list,val_preds_list,average='macro')))
                    print("\t\tValidation) Acc: {:.4f} Loss:{:.4f}".format(
                        val_corrects/test_size, val_loss/test_size))
            # print("val_corrects:\n", val_corrects)

            val_corrects_list.append(val_corrects/test_size)
            val_loss_list.append(val_loss/test_size)
            val_acc = val_corrects.double() / test_size
            val_acc_list.append(val_acc)

        if epoch_loss < best_loss:
            # print("prev_loss: {:.5f}".format(prev_loss))
            # print("loss: {:.5f}".format(loss))
            print(
                "\t\tSaving the best model w/ loss {:.4f}".format(epoch_loss))
            torch.save(model.state_dict(), PATH)
            best_loss = epoch_loss
            patience_count = 0
        elif best_loss < epoch_loss:
            patience_count += 1
        if patience_count >= patience:
            print("Finishing the Model: Loss is not decreasing...")
            print(train_loss[-6:-1])
            return train_accuracy, train_loss, val_acc_list, val_loss_list
    return train_accuracy, train_loss, val_acc_list, val_loss_list

train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_thread, num_epochs=100, patience=5, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = __MLP.thread_model()
PATH = './Model/state_dict_thread_model.pt'
model.load_state_dict(torch.load(PATH))
__MLP.predict(model, criterion, test_dataloader, test_size)
# SPARSE + THREAD
pheme_sparse = pd.read_csv('./data/_PHEME_sparse.csv')
ext_sparse = pd.read_csv('./data/_PHEMEext_sparse.csv')
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target
pheme_thread_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')
ext_thread_avg= pd.read_csv('./data/_PHEMEext_thread_avg.csv')

scaler = StandardScaler()
pheme_temp = scaler.fit_transform(pd.concat([pheme_sparse, pheme_thread_avg],axis=1))
ext_temp = scaler.transform(pd.concat([ext_sparse, ext_thread_avg],axis=1))

# pheme_sparse_bert_spread = pd.concat([pheme_sparse, pheme_thread, pheme_bert],axis=1)
# ext_sparse_bert_spread = pd.concat([ext_sparse, ext_thread_log, ext_bert],axis=1)

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_temp, pheme_y, ext_temp, ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class sparse_thread_net(nn.Module):
    def __init__(self):
        super(sparse_thread_net, self).__init__()  # 1*20
        self.fc1 = nn.Linear(79, 15, bias=True)  # 420
        # self.fc2 = nn.Linear(12, 7, bias=True)
        self.fc3 = nn.Linear(15, 1, bias=True)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_5 = nn.Dropout(0.5)
        self.drop_2 = nn.Dropout(0.2)
        self.batchnorm = nn.BatchNorm1d(64)

    def forward(self, x):
        x = self.drop_2(F.elu(self.fc1(x)))
        # x = F.elu(self.fc1(x))
        x = self.drop_2(F.elu(self.fc3(x)))
        # x = F.elu(self.fc2(x))
        # x = self.fc3(x)
        return x

__MLP.set_seed(42)
model_sparse_thread = sparse_thread_net()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_sparse_thread.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []

epochs = 200

PATH = "./Model/state_dict_sparse_thread_model.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_sparse_thread, num_epochs=200, patience=15, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = sparse_thread_net()
PATH = './Model/state_dict_sparse_thread_model.pt'
model.load_state_dict(torch.load(PATH))
__MLP.predict(model, criterion, test_dataloader, test_size)
model = BERT_net()
PATH = './Model/state_dict_bert_model.pt'
model.load_state_dict(torch.load(PATH))
predict(model, criterion, test_dataloader, test_size)
pheme_thread_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')
ext_thread_avg= pd.read_csv('./data/_PHEMEext_thread_avg.csv')

scaler = StandardScaler()
pheme_thread_scaled = pd.DataFrame(scaler.fit_transform(pheme_thread_avg))
ext_thread_scaled = pd.DataFrame(scaler.transform(ext_thread_avg))

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_thread_scaled,pheme_y,ext_thread_scaled,ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class thread_model(nn.Module):
    def __init__(self):
        super(thread_model, self).__init__() # 1*20
        self.fc1 = nn.Linear(38, 12, bias=True) # 420
        # self.fc2 = nn.Linear(12, 8, bias=True)
        self.fc3 = nn.Linear(12, 1)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = self.drop_3(F.elu(self.fc1(x)))
        # x = F.elu(self.fc2(x))
        x = self.drop_2(self.fc3(x))
        return x
model_thread = thread_model()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model_sparse.parameters(), lr=0.01, momentum=0.9)
optimizer = optim.Adam(model_thread.parameters(), lr=5e-2, eps=1e-8, weight_decay=1e-4)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_thread_model.pt"
def train1(model, num_epochs, criterion, optimizer, scheduler, train_loader, train_size, test_loader=None, test_size=None, patience=5, PATH='./state_dict_model.pt'):
    set_seed(42)
    train_loss = []
    patience_count = 0
    train_accuracy = []
    prev_loss = 10
    best_loss = 10.0
    val_corrects_list = []
    val_loss_list = []
    val_acc_list = []
    for epoch in range(num_epochs):  # loop over the dataset multiple times
        # print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        # print('-' * 10)
        running_corrects = 0.0
        running_loss = 0.0
        model.train()  # Set model to training mode
        for i, data in enumerate(train_loader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data
            inputs, labels = inputs.float(), labels.float()

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)

            #  _, predictions = torch.max(outputs.data, 1) won’t work if your output only contains a single output unit.
            # _, preds = torch.max(outputs, 1)
            # print(outputs.flatten().size())
            preds = outputs.squeeze(1) > 0.0

            loss = criterion(outputs, labels.unsqueeze(1))
            loss.backward()
            optimizer.step()

            # step function
            scheduler.step()

            running_loss += loss.item() * inputs.size(0)
            running_corrects += torch.sum(preds == labels.data)

        epoch_loss = running_loss / train_size
        # print(running_loss)
        # print(train_size)
        epoch_acc = running_corrects.double() / train_size
        train_loss.append(epoch_loss)
        train_accuracy.append(epoch_acc)

        if (epoch % 2 == 0):
            print('Epoch {}/{}\tTrain) Acc: {:.4f}, Loss: {:.4f}'.format(epoch,
                                                                         num_epochs - 1, epoch_acc, epoch_loss))

        if (test_loader != None):
            model.eval()
            with torch.no_grad():
                val_loss = 0.0
                val_corrects = 0
                val_preds_list = []
                val_label_list = []
                for j, val in enumerate(test_loader, 0):
                    val_x, val_label = val
                    val_x, val_label = val_x.float(), val_label.float()
                    val_outputs = model(val_x)
                    # _, val_preds = torch.max(val_outputs, 1)
                    val_preds = val_outputs.squeeze(1) > 0.0
                    # print("val_preds:\n", val_preds)
                    # print("val_labels:\n", val_label)

                    val_preds_list.append(val_preds)
                    val_label_list.append(val_label)
                    v_loss = criterion(val_outputs, val_label.unsqueeze(1))
                    val_loss += (v_loss.item() * val_x.size(0))
                    val_corrects += torch.sum(val_preds == val_label)
                    # print("val_corrects:\n", val_corrects)
                    # accuracy = (preds == b_labels).cpu().numpy().mean() * 100

                if (epoch % 2 == 0):
                    val_preds_list = torch.cat(val_preds_list, 0)
                    val_label_list = torch.cat(val_label_list, 0)
                    # print("\t\tValidation) Acc: {:.4f} Loss:{:.4f} F1 score: {:4f}".format(val_corrects/test_size, val_loss/test_size, f1_score(val_label_list,val_preds_list,average='macro')))
                    print("\t\tValidation) Acc: {:.4f} Loss:{:.4f}".format(
                        val_corrects/test_size, val_loss/test_size))
            # print("val_corrects:\n", val_corrects)

            val_corrects_list.append(val_corrects/test_size)
            val_loss_list.append(val_loss/test_size)
            val_acc = val_corrects.double() / test_size
            val_acc_list.append(val_acc)

        if epoch_loss < best_loss:
            # print("prev_loss: {:.5f}".format(prev_loss))
            # print("loss: {:.5f}".format(loss))
            print(
                "\t\tSaving the best model w/ loss {:.4f}".format(epoch_loss))
            torch.save(model.state_dict(), PATH)
            best_loss = epoch_loss
            patience_count = 0
        elif best_loss < epoch_loss:
            patience_count += 1
        if patience_count >= patience:
            print("Finishing the Model: Loss is not decreasing...")
            print(train_loss[-6:-1])
            return train_accuracy, train_loss, val_acc_list, val_loss_list
    return train_accuracy, train_loss, val_acc_list, val_loss_list

train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_thread, num_epochs=100, patience=5, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = __MLP.thread_model()
PATH = './Model/state_dict_thread_model.pt'
model.load_state_dict(torch.load(PATH))
__MLP.predict(model, criterion, test_dataloader, test_size)
# Sparse + W2V -> MLP
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
pheme_sparse = scaler.fit_transform(pheme_sparse.values)
sparse_bert = np.concatenate([pheme_sparse, pheme_bert.values],axis=1)
ext_sparse = scaler.fit_transform(ext_sparse.values)
sparse_bert_ext = np.concatenate([ext_sparse, ext_bert.values],axis=1)

tensor_x1 = torch.Tensor(sparse_bert).unsqueeze(1)
tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)
train_dataset = TensorDataset(tensor_x1,tensor_y1)

tensor_x2 = torch.Tensor(sparse_bert_ext).unsqueeze(1)
tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
pheme_sparse_W2V = pd.concat([pheme_sparse, pheme_AVGw2v],axis=1)
ext_sparse_W2V = pd.concat([ext_sparse, ext_AVGw2v],axis=1)

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_sparse_W2V, pheme_y, ext_sparse_W2V, ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class W2V_sparse_net(nn.Module):
    def __init__(self):
        super(W2V_sparse_net, self).__init__() # 1*20
        self.fc1 = nn.Linear(229, 50, bias=True) # 420
        self.fc2 = nn.Linear(50, 8, bias=True)
        self.fc3 = nn.Linear(8, 1,bias=True)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)

    def forward(self, x):
        x = self.drop_2(F.elu(self.fc1(x)))
        x = self.drop_2(F.elu(self.fc2(x)))
        x = self.fc3(x)
        return x
set_seed(42)
model_w2vsparse = W2V_sparse_net()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_w2vsparse.parameters(), lr=5e-5, eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  

epochs = 100

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

PATH = "./Model/state_dict_w2v_sparse_model.pt"
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_w2vsparse, num_epochs=40, patience=8, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
# Sparse + BERT -> MLP
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
pheme_sparse = scaler.fit_transform(pheme_sparse.values)
sparse_bert = np.concatenate([pheme_sparse, pheme_bert_bracket.values],axis=1)
ext_sparse = scaler.fit_transform(ext_sparse.values)
sparse_bert_ext = np.concatenate([ext_sparse, ext_bert_bracket.values],axis=1)
# pheme_sparse_bert = pd.concat([pheme_sparse, pheme_bert],axis=1)
# ext_sparse_bert = pd.concat([ext_sparse, ext_bert],axis=1)

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(sparse_bert, pheme_y, sparse_bert_ext, ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class BERT_sparse_net(nn.Module):
    def __init__(self):
        super(BERT_sparse_net, self).__init__()  # 1*20
        self.fc1 = nn.Linear(804, 96, bias=True)  # 420
        self.fc2 = nn.Linear(96, 12, bias=True)
        self.fc3 = nn.Linear(12, 1, bias=True)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_1 = nn.Dropout(0.1)
        self.drop_2 = nn.Dropout(0.2)
        self.batchnorm = nn.BatchNorm1d(64)

    def forward(self, x):
        x = self.drop_3(F.elu(self.fc1(x)))
        x = self.drop_3(F.elu(self.fc2(x)))
        x = self.fc3(x)
        return x

__MLP.set_seed(42)
model_bertsparse = BERT_sparse_net()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_bertsparse.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []

epochs = 40

PATH = "./Model/state_dict_bert_sparse_model.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_bertsparse, num_epochs=40, patience=10, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
# Sparse + BERT -> MLP (Multi)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

# pheme_scaled = scaler.fit_transform(np.concatenate([pheme_sparse.values, pheme_thread.values],axis=1))
pheme_sparse_scaled = pd.DataFrame(scaler.fit_transform(pheme_sparse))
# Train = np.concatenate([train, pheme_bert.values],axis=1)

ext_sparse_scaled = pd.DataFrame(scaler.transform(ext_sparse))
# Test = np.concatenate([test, ext_bert.values],axis=1)
# tensor_x11 = torch.Tensor(pheme_scaled).unsqueeze(1)
tensor_x11 = torch.Tensor(pheme_sparse_scaled.values).unsqueeze(1)
tensor_x12 = torch.Tensor(pheme_bert.values).unsqueeze(1)
tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)
train_dataset = TensorDataset(tensor_x11, tensor_x12, tensor_y1)

tensor_x21 = torch.Tensor(ext_sparse_scaled.values).unsqueeze(1)
# tensor_x21 = torch.Tensor(ext_sparse_scaled).unsqueeze(1)
tensor_x22 = torch.Tensor(ext_bert.values).unsqueeze(1)
tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)
test_dataset = TensorDataset(tensor_x21, tensor_x22, tensor_y2)

# tensor_x11, tensor_x12, tensor_y1, tensor_x21, tensor_x22, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_sparse, pheme_y, ext_sparse, ext_y, pheme_bert, ext_bert)
# train_dataset = TensorDataset(tensor_x11, tensor_x12, tensor_y1)
# test_dataset = TensorDataset(tensor_x21, tensor_x22, tensor_y2)
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x21)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x11.shape,tensor_x21.shape)
print(tensor_x12.shape,tensor_x22.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class BERT_sparse_net_multi(nn.Module):
    def __init__(self):
        super(BERT_sparse_net_multi, self).__init__()  # 1*20

        self.layer1 = nn.Sequential(
            nn.Linear(36, 5, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(5, 2, bias=True),  # 420
        )
        self.layer2 = nn.Sequential(
            nn.Linear(768, 64, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(64, 8, bias=True),
            nn.ReLU()
        )

        self.layer3 = nn.Sequential(
            nn.Linear(10, 1)  # 420
        )

        self.drop_2 = nn.Dropout(0.2)

    def forward(self, sparse, embedding):
        x1 = self.layer1(sparse)
        x2 = self.layer2(embedding)
        
        x = torch.cat((x1.squeeze(1), x2.squeeze(1)), dim=1)
        x = self.layer3(x)
        return x
# _MLP.set_seed(42)
model_bertsparse_multi = BERT_sparse_net_multi()
criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_bertsparse_multi.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []

epochs = 40

PATH = "./Model/state_dict_bert_sparse_multi.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)
def predict(model, criterion, val_dataloader, val_size):
    model.eval()
    with torch.no_grad():
        val_loss = 0.0
        val_corrects = 0
        for j, val in enumerate(val_dataloader, 0):
            val_sparse, val_embedding, val_labels = val
            val_sparse, val_embedding, val_labels = val_sparse.float(), val_embedding.float(), val_labels.float().squeeze(1)
            val_outputs = model(val_sparse, val_embedding)
            val_preds = val_outputs.squeeze(1) > 0.0

            val_preds_list.append(val_preds)
            val_label_list.append(val_labels)
            v_loss = criterion(val_outputs, val_labels.unsqueeze(1))
            val_loss += (v_loss.item() * val_labels.size(0))
            val_corrects += torch.sum(val_preds == val_labels)

    val_preds_list = torch.cat(val_preds_list, 0)
    val_label_list = torch.cat(val_label_list, 0)
    val_corrects = val_corrects/val_size
    val_loss = val_loss/val_size
    val_acc = val_corrects.double() / val_size
    print("\t\tValidation) Acc: {:.4f} Loss:{:.4f}".format(
        val_corrects/val_size, val_loss/val_size))
    # print("\t\tValidation) Acc: {:.4f} Loss:{:.4f} F1 score: {:4f}".format(val_corrects/val_size, val_loss/test_size, f1_score(val_label_list,val_preds_list,average='macro')))

def set_seed(seed_value=42):
    """Set seed for reproducibility.
    """
    random.seed(seed_value)
    np.random.seed(seed_value)
    torch.manual_seed(seed_value)
    torch.cuda.manual_seed_all(seed_value)

def train3(model, num_epochs, criterion, optimizer, scheduler, train_loader, train_size, test_loader=None, test_size=None, patience=5, PATH='./state_dict_model.pt'):
    set_seed(42)
    train_loss = []
    patience_count = 0
    train_accuracy = []
    prev_loss = 10
    best_loss = 10.0
    val_corrects_list = []
    val_loss_list = []
    val_acc_list = []
    for epoch in range(num_epochs):  # loop over the dataset multiple times
        # print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        # print('-' * 10)
        running_corrects = 0.0
        running_loss = 0.0
        model.train()  # Set model to training mode
        for i, data in enumerate(train_loader, 0):
            # get the inputs; data is a list of [inputs, labels]
            sparse, embedding, labels = data
            sparse, embedding, labels = sparse.float(), embedding.float(), labels.float().squeeze(1)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(sparse, embedding)

            #  _, predictions = torch.max(outputs.data, 1) won’t work if your output only contains a single output unit.
            # _, preds = torch.max(outputs, 1)
            # print(outputs.flatten().size())
            preds = outputs.squeeze(1) > 0.0

            loss = criterion(outputs, labels.unsqueeze(1))
            loss.backward()
            optimizer.step()

            # step function
            scheduler.step()

            running_loss += loss.item() * sparse.size(0)
            running_corrects += torch.sum(preds == labels.data)

        epoch_loss = running_loss / train_size
        # print(running_loss)
        # print(train_size)
        epoch_acc = running_corrects.double() / train_size
        train_loss.append(epoch_loss)
        train_accuracy.append(epoch_acc)

        if (epoch % 2 == 0):
            print('Epoch {}/{}\tTrain) Acc: {:.4f}, Loss: {:.4f}'.format(epoch,
                                                                         num_epochs - 1, epoch_acc, epoch_loss))

        if (test_loader != None):
            model.eval()
            with torch.no_grad():
                val_loss = 0.0
                val_corrects = 0
                val_preds_list = []
                val_label_list = []
                for j, val in enumerate(test_loader, 0):
                    val_sparse, val_embedding, val_labels = data
                    val_sparse, val_embedding, val_labels = val_sparse.float(), val_embedding.float(), val_labels.float().squeeze(1)
                    val_outputs = model(val_sparse, val_embedding)
                    # _, val_preds = torch.max(val_outputs, 1)
                    # print("val_outputs size:",val_outputs.size())
                    # print("val_label size:",val_label.size())
                    val_preds = val_outputs.squeeze(1) > 0.0
                    # print("pred size:",val_preds.size())

                    # print("Length of val_preds:",val_preds.size())
                    val_preds_list.append(val_preds)
                    val_label_list.append(val_labels)
                    v_loss = criterion(val_outputs, val_labels.unsqueeze(1))
                    val_loss += (v_loss.item() * val_labels.size(0))
                    val_corrects += torch.sum(val_preds == val_labels)
                    # print("== length",val_preds == val_label)
                    # accuracy = (preds == b_labels).cpu().numpy().mean() * 100

                    # print("val_corrects:",val_corrects)
                # print("length of one batch")
                # print("FINAL val_corrects:",val_corrects)
                # print("test_size:",test_size)
                    
                if (epoch % 2 == 0):
                    val_preds_list = torch.cat(val_preds_list, 0)
                    val_label_list = torch.cat(val_label_list, 0)
                    # print("\t\tValidation) Acc: {:.4f} Loss:{:.4f} F1 score: {:4f}".format(val_corrects/test_size, val_loss/test_size, f1_score(val_label_list,val_preds_list,average='macro')))
                    print("\t\tValidation) Acc: {:.4f} Loss:{:.4f}".format(
                        val_corrects/test_size, val_loss/test_size))
            val_corrects_list.append(val_corrects/test_size)
            val_loss_list.append(val_loss/test_size)
            val_acc = val_corrects.double() / test_size
            val_acc_list.append(val_acc)

        if epoch_loss < best_loss:
            # print("prev_loss: {:.5f}".format(prev_loss))
            # print("loss: {:.5f}".format(loss))
            print(
                "\t\tSaving the best model w/ loss {:.4f}".format(epoch_loss))
            torch.save(model.state_dict(), PATH)
            best_loss = epoch_loss
            patience_count = 0
        elif best_loss < epoch_loss:
            patience_count += 1
        if patience_count >= patience:
            print("Finishing the Model: Loss is not decreasing...")
            print(train_loss[-6:-1])
            return train_accuracy, train_loss, val_acc_list, val_loss_list
    return train_accuracy, train_loss, val_acc_list, val_loss_list
train_acc, train_loss, val_acc, val_loss_list = train3(patience=8,model=model_bertsparse_multi, num_epochs=epochs, criterion=criterion, optimizer=optimizer,scheduler=scheduler, train_loader=train_dataloader, train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)
__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
predict(model_bertsparse, criterion, test_dataloader, test_size)
## Previous Models
self.layer1 = nn.Sequential(
    nn.Linear(768, 64, bias=True),  # 420
    nn.ReLU(),
    nn.Dropout(0.2),
    nn.Linear(64, 8, bias=True),
    nn.ReLU()
)

self.layer2 = nn.Sequential(
    nn.Linear(27, 5, bias=True),  # 420
    nn.ReLU(),
    nn.Dropout(0.2),
    nn.Linear(5, 2, bias=True),
    nn.ReLU()
)

self.layer3 = nn.Sequential(
    nn.Linear(10, 1)  # 420
)

Min of Training Loss: 0.471854
Max of Training Accuracy: 0.785764
Mean of Training Loss: 0.505716
Mean of Training Accuracy: 0.758512
----
Max of Testing Accuracy: 0.639175
Mean of Testing Loss: 0.351359
Mean of Testing Accuracy: 0.454525
self.layer1 = nn.Sequential(
    nn.Linear(768, 64, bias=True),  # 420
    nn.ReLU(),
    nn.Dropout(0.2),
    nn.Linear(64, 5, bias=True),
    nn.ReLU()
)

self.layer2 = nn.Sequential(
    nn.Linear(27, 5, bias=True),  # 420
    nn.ReLU(),
    nn.Dropout(0.2),
    nn.Linear(5, 2, bias=True),
    nn.ReLU()
)

self.layer3 = nn.Sequential(
    nn.Linear(7, 1)  # 420
)

Min of Training Loss: 0.467921
Max of Training Accuracy: 0.788694
Mean of Training Loss: 0.501997
Mean of Training Accuracy: 0.760402
----
Max of Testing Accuracy: 0.639175
Mean of Testing Loss: 0.348198
Mean of Testing Accuracy: 0.459259
# BERT + THREAD (Multi)
# BERT + SPARSE + THREAD (Multi)

scaler = StandardScaler()

# pheme_scaled = scaler.fit_transform(np.concatenate([pheme_sparse.values, pheme_thread_avg.values],axis=1))
# Train = np.concatenate([train, pheme_bert.values],axis=1)

# ext_sparse_scaled = scaler.transform(np.concatenate([ext_sparse.values, ext_thread_avg.values],axis=1))
# Test = np.concatenate([test, ext_bert.values],axis=1)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

X_train = pd.concat([pheme_sparse, pheme_thread], axis=1)
X_test = pd.concat([ext_sparse, ext_thread], axis=1)

# X_train = scaler.fit_transform(X_train.values)
X_train = np.concatenate([X_train, pheme_bert.values], axis=1)

# X_test = scaler.fit_transform(X_test.values)
X_test = np.concatenate([X_test, ext_bert.values],axis=1)

# Train = pd.concat([pheme_bert, pheme_sparse], axis=1)
# Test = pd.concat([ext_bert, ext_sparse], axis=1)

tensor_x1 = torch.Tensor(X_train).unsqueeze(1)
tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)
train_dataset = TensorDataset(tensor_x1,tensor_y1)

tensor_x2 = torch.Tensor(X_test).unsqueeze(1)
tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
pheme_temp = pd.concat([pheme_bert, pheme_thread_avg], axis=1)
ext_temp = pd.concat([ext_bert, ext_thread_avg], axis=1)

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_temp, pheme_y, ext_temp, ext_y)
train_dataset = TensorDataset(tensor_x1, tensor_y1)
test_dataset = TensorDataset(tensor_x2, tensor_y2)

batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x1)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)

# pheme_temp = pd.concat([pheme_sparse, pheme_thread_log], axis=1)
# ext_temp = pd.concat([ext_sparse, ext_thread_log], axis=1)

tensor_x1 = torch.Tensor(pheme_thread_avg.values).unsqueeze(1)
tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)

tensor_x2 = torch.Tensor(ext_thread_avg.values).unsqueeze(1)
tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)

tensor_x1_2= torch.Tensor(pheme_bert.values).unsqueeze(1)
tensor_x2_2 = torch.Tensor(ext_bert.values).unsqueeze(1)

train_dataset = TensorDataset(tensor_x1, tensor_x1_2, tensor_y1)
test_dataset = TensorDataset(tensor_x2, tensor_x2_2, tensor_y2)

batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_x1_2.shape,tensor_x2_2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)

class sparse_thread_net_multi(nn.Module):
    def __init__(self):
        super(sparse_thread_net_multi, self).__init__()  # 1*20


        self.layer1 = nn.Sequential(
            nn.Linear(38, 8, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(8, 2, bias=True),  # 420
        )
        self.layer2 = nn.Sequential(
            nn.Linear(768, 64, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(64, 8, bias=True),
            nn.ReLU()
        )

        self.layer3 = nn.Sequential(
            nn.Linear(10, 1)  # 420
        )

        self.drop_2 = nn.Dropout(0.2)

    def forward(self, sparse, embedding):
        x1 = self.layer1(sparse)
        x2 = self.layer2(embedding)
        
        x = torch.cat((x1.squeeze(1), x2.squeeze(1)), dim=1)
        x = self.layer3(x)
        return x

set_seed()
model_sparsethread_multi = sparse_thread_net_multi()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_sparsethread_multi.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []


epochs = 200

PATH = "./Model/state_dict_sparse_thread_model_multi.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)

train_acc, train_loss, val_acc, val_loss_list = train3(model=model_sparsethread_multi, num_epochs=epochs,patience=15, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader, train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
# BERT + SPARSE + THREAD
def convert_df_to_unsqueezed_tensor(trainX, trainY, testX, testY, TrainX2=None, TestX2=None):
    tensor_x1 = torch.Tensor(trainX).unsqueeze(1)
    tensor_y1 = torch.Tensor(trainY.values).unsqueeze(1)

    tensor_x2 = torch.Tensor(testX).unsqueeze(1)
    tensor_y2 = torch.Tensor(testY).unsqueeze(1)
    if ((TrainX2 is None) | (TestX2 is None)):
        
        return tensor_x1, tensor_y1, tensor_x2, tensor_y2
    elif (TrainX2 is not None & TestX2 is not None):
        tensor_x1_2= torch.Tensor(TrainX2.values).unsqueeze(1)
        tensor_x2_2 = torch.Tensor(TestX2.values).unsqueeze(1)
        return tensor_x1, tensor_x1_2, tensor_y1, tensor_x2, tensor_x2_2, tensor_y2
pheme_sparse = pd.read_csv('./data/_PHEME_sparse.csv')
ext_sparse = pd.read_csv('./data/_PHEMEext_sparse.csv')
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target
pheme_thread_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')
ext_thread_avg= pd.read_csv('./data/_PHEMEext_thread_avg.csv')
pheme_bert = pd.read_csv('./data/_PHEME_Bert_simple.csv')
ext_bert = pd.read_csv('./data/_PHEMEext_Bert_simple.csv')

# pheme_bert = pd.read_csv('./data/_PHEME_Bert_brackets.csv')
# ext_bert = pd.read_csv('./data/_PHEMEext_Bert_brackets.csv')

scaler = StandardScaler()
pheme_temp = scaler.fit_transform(pd.concat([pheme_sparse, pheme_thread_avg],axis=1))
ext_temp = scaler.transform(pd.concat([ext_sparse, ext_thread_avg],axis=1))

pheme_temp = np.concatenate([pheme_bert, pheme_temp],axis=1)
ext_temp = np.concatenate([ext_bert, ext_temp],axis=1)

# pheme_sparse_bert_spread = pd.concat([pheme_sparse, pheme_thread, pheme_bert],axis=1)
# ext_sparse_bert_spread = pd.concat([ext_sparse, ext_thread_log, ext_bert],axis=1)

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = convert_df_to_unsqueezed_tensor(pheme_temp, pheme_y, ext_temp, ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)
print(tensor_x1.size(),tensor_x2.size())
print(tensor_y1.size(),tensor_y2.size())
# # Train = pd.concat([pheme_bert, pheme_sparse], axis=1)
# # Test = pd.concat([ext_bert, ext_sparse], axis=1)

# tensor_x1 = torch.Tensor(sparse_bert).unsqueeze(1)
# tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)
# train_dataset = TensorDataset(tensor_x1,tensor_y1)

# tensor_x2 = torch.Tensor(sparse_bert_ext).unsqueeze(1)
# tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)
# test_dataset = TensorDataset(tensor_x2,tensor_y2)
batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class BERT_sparse_thread_net1(nn.Module):
    def __init__(self):
        super(BERT_sparse_thread_net1, self).__init__()  # 1*20
        self.fc1 = nn.Linear(842, 12, bias=True)  # 420
        # self.fc2 = nn.Linear(30, 12, bias=True)
        self.fc3 = nn.Linear(12, 1, bias=True)

        self.drop_3 = nn.Dropout(0.3)
        self.drop_4 = nn.Dropout(0.4)
        self.drop_2 = nn.Dropout(0.2)
        self.batchnorm = nn.BatchNorm1d(842)

    def forward(self, x):
        # x = F.elu(self.fc1(x))
        x = self.drop_3(F.elu(self.fc1(x)))
        # x = self.drop_2(F.elu(self.fc2(x)))
        # x = F.elu(self.fc2(x))
        x = self.fc3(x)
        return x

set_seed(42)
model_bertsparse = BERT_sparse_thread_net1()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_bertsparse.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []

epochs = 200

PATH = "./Model/state_dict_bert_thread_model.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)
train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model_bertsparse, num_epochs=200, patience=30, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader,train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)

__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)
model = BERT_sparse_thread_net1()
PATH = './Model/state_dict_bert_thread_model.pt'
model.load_state_dict(torch.load(PATH))
predict(model, criterion, test_dataloader, test_size)
# BERT + SPARSE + THREAD (Multi)
# pheme_scaled = scaler.fit_transform(np.concatenate([pheme_sparse.values, pheme_thread.values],axis=1))
# # Train = np.concatenate([train, pheme_bert.values],axis=1)

# ext_sparse_scaled = scaler.fit_transform(np.concatenate([ext_sparse.values, ext_thread.values],axis=1))
# # Test = np.concatenate([test, ext_bert.values],axis=1)
# from sklearn.preprocessing import StandardScaler
# scaler = StandardScaler()

# X_train = pd.concat([pheme_sparse, pheme_thread], axis=1)
# X_test = pd.concat([ext_sparse, ext_thread], axis=1)

# # X_train = scaler.fit_transform(X_train.values)
# X_train = np.concatenate([X_train, pheme_bert.values], axis=1)

# # X_test = scaler.fit_transform(X_test.values)
# X_test = np.concatenate([X_test, ext_bert.values],axis=1)

# # Train = pd.concat([pheme_bert, pheme_sparse], axis=1)
# # Test = pd.concat([ext_bert, ext_sparse], axis=1)

# tensor_x1 = torch.Tensor(X_train).unsqueeze(1)
# tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)
# train_dataset = TensorDataset(tensor_x1,tensor_y1)

# tensor_x2 = torch.Tensor(X_test).unsqueeze(1)
# tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)
# test_dataset = TensorDataset(tensor_x2,tensor_y2)
___
pheme_temp = pd.concat([pheme_sparse, pheme_thread_log], axis=1)
ext_temp = pd.concat([ext_sparse, ext_thread_log], axis=1)

tensor_x1 = torch.Tensor(pheme_temp.values).unsqueeze(1)
tensor_y1 = torch.Tensor(pheme_y.values).unsqueeze(1)

tensor_x2 = torch.Tensor(ext_temp.values).unsqueeze(1)
tensor_y2 = torch.Tensor(ext_y.values).unsqueeze(1)

tensor_x1_2= torch.Tensor(pheme_bert.values).unsqueeze(1)
tensor_x2_2 = torch.Tensor(ext_bert.values).unsqueeze(1)

train_dataset = TensorDataset(tensor_x1, tensor_x1_2, tensor_y1)
test_dataset = TensorDataset(tensor_x2, tensor_x2_2, tensor_y2)

batch_size = 16

# Initialize WeightedRandomSampler to deal with the unbalanced dataset
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)

train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

data = next(iter(train_dataloader))
print("mean: %s, std: %s" %(data[0].mean(), data[0].std()))

train_size = int(tensor_y1.size(0))
test_size = int(tensor_y2.size(0))

print(tensor_x1.shape,tensor_x2.shape)
print(tensor_x1_2.shape,tensor_x2_2.shape)
print(tensor_y1.shape,tensor_y2.shape)
print("Train Size",train_size,"Test Size",test_size)
class BERT_sparse_thread_net_multi(nn.Module):
    def __init__(self):
        super(BERT_sparse_thread_net_multi, self).__init__()  # 1*20


        self.layer1 = nn.Sequential(
            nn.Linear(65, 12, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(12, 2, bias=True),  # 420
        )
        self.layer2 = nn.Sequential(
            nn.Linear(768, 64, bias=True),  # 420
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(64, 8, bias=True),
            nn.ReLU()
        )

        self.layer3 = nn.Sequential(
            nn.Linear(10, 1)  # 420
        )

    def forward(self, sparse, embedding):
        x1 = self.layer1(sparse)
        x2 = self.layer2(embedding)
        
        x = torch.cat((x1.squeeze(1), x2.squeeze(1)), dim=1)
        x = self.layer3(x)
        return x
set_seed()
model_bertsparsethread_2 = BERT_sparse_thread_net_multi()

criterion = nn.BCEWithLogitsLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
optimizer = optim.Adam(model_bertsparsethread_2.parameters(), lr=5e-5,eps=1e-8, weight_decay=1e-6)
# scheduler = lr_scheduler.ExponentialLR(optimizer, gamma= 0.99)  
train_loss = []
train_accuracy = []


epochs = 60

PATH = "./Model/state_dict_bert_sparse_thread_model_multi.pt"

total_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0,  # Default value
                                            num_training_steps=total_steps)
train_acc, train_loss, val_acc, val_loss_list = train3(model=model_bertsparsethread_2, num_epochs=epochs, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader, train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)
__MLP.clf_report(train_loss, train_acc, val_loss_list, val_acc)

# TESTING
pheme_AVGw2v = pd.read_csv('./data/_PHEME_text_AVGw2v.csv').drop(['token'],axis=1)
pheme_sparse = pd.read_csv('./data/_PHEME_sparse.csv')
pheme_y = pd.read_csv('./data/_PHEME_target.csv').target
pheme_event = pd.read_csv('./data/_PHEME_text.csv')['Event']
pheme_bert = fetchdata('pheme','bert')
pheme_bert = pd.read_csv('./data/_PHEME_Bert_simple.csv')
pheme_bert_bracket = pd.read_csv('./data/_PHEME_Bert_brackets.csv')
pheme_thread =  fetchdata('pheme','thread')
pheme_thread_log = pd.read_csv('./data/_PHEME_thread_log.csv')
pheme_thread_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')


ext_AVGw2v = pd.read_csv('./data/_PHEMEext_text_AVGw2v.csv').drop(['token'],axis=1)
ext_sparse = pd.read_csv('./data/_PHEMEext_sparse.csv')
ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target
ext_event = pd.read_csv('./data/_PHEMEext_text.csv').Event
ext_bert = fetchdata('ext','bert')
ext_bert = pd.read_csv('./data/_PHEMEext_Bert_simple.csv')
ext_bert_bracket = pd.read_csv('./data/_PHEMEext_Bert_brackets.csv')

ext_thread = fetchdata('ext','thread')
ext_thread_log = pd.read_csv('./data/_PHEMEext_thread_log.csv')
ext_thread_avg= pd.read_csv('./data/_PHEMEext_thread_avg.csv')

rhi = pd.read_csv('./data/_RHI_text_AVGw2v.csv').drop(['token'],axis=1)
rhi_y = pd.read_csv('./data/_RHI_target.csv')
rhi_bert = fetchdata('rhi','bert')
model_paths = ["./Model/state_dict_sparse_model.pt",
               "./Model/state_dict_w2v_model.pt",
               "./Model/state_dict_bert_model.pt",
               "./Model/state_dict_thread_model.pt",
               "./Model/state_dict_w2v_sparse_model.pt", 
               "./Model/state_dict_bert_sparse_model.pt",
               "./Model/state_dict_bert_sparse_multi.pt",
               "./Model/state_dict_sparse_thread_model_multi.pt",
                "./Model/state_dict_bert_sparse_model.pt",
                "./Model/state_dict_bert_thread_model.pt",
                "./Model/state_dict_bert_sparse_thread_model_multi.pt"
               ]

# Sparse
#pheme_sparse -> pheme_scaled -> tensor
scaler = StandardScaler()
pheme_scaled = pd.DataFrame(scaler.fit_transform(pheme_sparse))
ext_scaled = pd.DataFrame(scaler.transform(ext_sparse))

tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(pheme_scaled,pheme_y,ext_scaled,ext_y)
train_dataset = TensorDataset(tensor_x1,tensor_y1)
test_dataset = TensorDataset(tensor_x2,tensor_y2)

batch_size = 16
train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, num_workers=2)
test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)

train_size, test_size = int(tensor_y1.size(0)), int(tensor_y2.size(0))

print("Training: ",tensor_x1.shape,"/", tensor_x2.shape)
print("Testing2: ",tensor_y1.shape,"/", tensor_y2.shape)
print("Train Size",train_size,"/ Test Size",test_size)
PATH = './Model/BERTweet_raw_to_fine_tune_SimplePrdep.pt'

# bert_classifier.load_state_dict(torch.load(PATH))
# testing_process(bert_classifier, X_val, y_val)
predict(model, criterion, val_dataloader, val_size)
# Compute predicted probabilities on the test set
probs = bert_predict(bert_classifier, val_dataloader)

# Evaluate the Bert classifier
evaluate_roc(probs, y_val)
PATH = './Model/BERTweet_raw_to_fine_tune_SimplePrdep.pt'
bert_classifier.load_state_dict(torch.load(PATH))
testing_process(bert_classifier, X_val, y_val)
PATH = './Model/BERTweet_raw_to_fine_tune_SimplePrep.pt'
bert_classifier.load_state_dict(torch.load(PATH))
testing_process(bert_classifier, X_val, y_val)
def predict(model, criterion, val_dataloader, val_size):
    model.eval()
    with torch.no_grad():
        val_loss = 0.0
        val_corrects = 0
        for j, val in enumerate(val_dataloader, 0):
            val_x, val_label = val
            val_x, val_label = val_x.float(), val_label.float()
            val_outputs = model(val_x)
            val_preds = val_outputs.squeeze(1) > 0.0

            val_preds_list.append(val_preds)
            val_label_list.append(val_label)
            v_loss = criterion(val_outputs, val_label.unsqueeze(1))
            val_loss += (v_loss.item() * val_x.size(0))
            val_corrects += torch.sum(val_preds == val_label)

    val_preds_list = torch.cat(val_preds_list, 0)
    val_label_list = torch.cat(val_label_list, 0)
    val_corrects = val_corrects/val_size
    val_loss = val_loss/test_size
    val_acc = val_corrects.double() / val_size

    print("\t\tValidation) Acc: {:.4f} Loss:{:.4f}".format(
        val_corrects/val_size, val_loss/test_size))
    
    print(classification_report(y_test, result))

    return val_acc, val_loss, val_corrects, val_preds_list, val_label_list