Added eval_results with additional metrics and train.sh script

models/eval/eval_results.json

@@ -1 +1 @@
-{"eval_loss": 0.6932736039161682, "eval_accuracy": 0.5, "eval_f1_macro": 0.4620003056568347, "eval_f1_weighted": 0.46200030565683464, "eval_runtime": 2.7749, "eval_samples_per_second": 240.007, "eval_steps_per_second": 7.568}
+{"eval_loss": 0.6932736039161682, "eval_accuracy": 0.5, "eval_f1_macro": 0.4620003056568347, "eval_f1_weighted": 0.46200030565683464, "eval_precision": 0.5, "eval_recall": 0.5, "eval_auroc": 0.5, "eval_runtime": 2.6124, "eval_samples_per_second": 254.941, "eval_steps_per_second": 8.039}

models/semi-supervised/finetune_nested/eval_results.json

@@ -1 +1 @@
-{"0.25": {"eval_loss": 0.4045407474040985, "eval_accuracy": 0.8948948948948949, "eval_f1_macro": 0.8948863636363636, "eval_f1_weighted": 0.8948863636363638, "eval_runtime": 2.3577, "eval_samples_per_second": 282.482, "eval_steps_per_second": 2.969, "epoch": 25.0}, "0.5": {"eval_loss": 0.44522979855537415, "eval_accuracy": 0.8933933933933934, "eval_f1_macro": 0.8933931530475364, "eval_f1_weighted": 0.8933931530475363, "eval_runtime": 2.4038, "eval_samples_per_second": 277.059, "eval_steps_per_second": 2.912, "epoch": 25.0}, "0.75": {"eval_loss": 0.5143420696258545, "eval_accuracy": 0.8828828828828829, "eval_f1_macro": 0.8828733766233766, "eval_f1_weighted": 0.8828733766233766, "eval_runtime": 2.3909, "eval_samples_per_second": 278.561, "eval_steps_per_second": 2.928, "epoch": 25.0}, "1.0": {"eval_loss": 0.5319307446479797, "eval_accuracy": 0.8828828828828829, "eval_f1_macro": 0.8828818267080297, "eval_f1_weighted": 0.8828818267080298, "eval_runtime": 2.4267, "eval_samples_per_second": 274.451, "eval_steps_per_second": 2.885, "epoch": 25.0}}
+{"0.25": {"eval_loss": 0.4045407474040985, "eval_accuracy": 0.8948948948948949, "eval_f1_macro": 0.8948863636363636, "eval_f1_weighted": 0.8948863636363638, "eval_precision": 0.8950231387513192, "eval_recall": 0.8948948948948949, "eval_auroc": 0.8948948948948949, "eval_runtime": 2.3628, "eval_samples_per_second": 281.865, "eval_steps_per_second": 2.963, "epoch": 25.0}, "0.5": {"eval_loss": 0.44522979855537415, "eval_accuracy": 0.8933933933933934, "eval_f1_macro": 0.8933931530475364, "eval_f1_weighted": 0.8933931530475363, "eval_precision": 0.8933969410576438, "eval_recall": 0.8933933933933934, "eval_auroc": 0.8933933933933934, "eval_runtime": 2.3526, "eval_samples_per_second": 283.09, "eval_steps_per_second": 2.975, "epoch": 25.0}, "0.75": {"eval_loss": 0.5143420696258545, "eval_accuracy": 0.8828828828828829, "eval_f1_macro": 0.8828733766233766, "eval_f1_weighted": 0.8828733766233766, "eval_precision": 0.8830072257854997, "eval_recall": 0.8828828828828829, "eval_auroc": 0.8828828828828829, "eval_runtime": 2.353, "eval_samples_per_second": 283.043, "eval_steps_per_second": 2.975, "epoch": 25.0}, "1.0": {"eval_loss": 0.5319307446479797, "eval_accuracy": 0.8828828828828829, "eval_f1_macro": 0.8828818267080297, "eval_f1_weighted": 0.8828818267080298, "eval_precision": 0.8828966947738648, "eval_recall": 0.882882882882883, "eval_auroc": 0.8828828828828829, "eval_runtime": 2.3732, "eval_samples_per_second": 280.639, "eval_steps_per_second": 2.95, "epoch": 25.0}}

models/supervised/finetune_nested/eval_results.json

@@ -1 +1 @@
-{"0.25": {"eval_loss": 0.691838800907135, "eval_accuracy": 0.5675675675675675, "eval_f1_macro": 0.5675675675675675, "eval_f1_weighted": 0.5675675675675675, "eval_runtime": 2.3496, "eval_samples_per_second": 283.455, "eval_steps_per_second": 2.979, "epoch": 25.0}, "0.5": {"eval_loss": 0.6798407435417175, "eval_accuracy": 0.6816816816816816, "eval_f1_macro": 0.6765360824742268, "eval_f1_weighted": 0.6765360824742268, "eval_runtime": 2.348, "eval_samples_per_second": 283.64, "eval_steps_per_second": 2.981, "epoch": 25.0}, "0.75": {"eval_loss": 0.3387678563594818, "eval_accuracy": 0.8843843843843844, "eval_f1_macro": 0.8843778676124103, "eval_f1_weighted": 0.8843778676124106, "eval_runtime": 2.3509, "eval_samples_per_second": 283.302, "eval_steps_per_second": 2.978, "epoch": 25.0}, "1.0": {"eval_loss": 0.2984059453010559, "eval_accuracy": 0.8993993993993994, "eval_f1_macro": 0.8992992789682137, "eval_f1_weighted": 0.8992992789682136, "eval_runtime": 2.3477, "eval_samples_per_second": 283.684, "eval_steps_per_second": 2.982, "epoch": 25.0}}
+{"0.25": {"eval_loss": 0.691838800907135, "eval_accuracy": 0.5675675675675675, "eval_f1_macro": 0.5675675675675675, "eval_f1_weighted": 0.5675675675675675, "eval_precision": 0.5675675675675675, "eval_recall": 0.5675675675675675, "eval_auroc": 0.5675675675675675, "eval_runtime": 2.3948, "eval_samples_per_second": 278.108, "eval_steps_per_second": 2.923, "epoch": 25.0}, "0.5": {"eval_loss": 0.6798407435417175, "eval_accuracy": 0.6816816816816816, "eval_f1_macro": 0.6765360824742268, "eval_f1_weighted": 0.6765360824742268, "eval_precision": 0.6940279102019589, "eval_recall": 0.6816816816816818, "eval_auroc": 0.6816816816816816, "eval_runtime": 2.3669, "eval_samples_per_second": 281.381, "eval_steps_per_second": 2.957, "epoch": 25.0}, "0.75": {"eval_loss": 0.3387678563594818, "eval_accuracy": 0.8843843843843844, "eval_f1_macro": 0.8843778676124103, "eval_f1_weighted": 0.8843778676124106, "eval_precision": 0.8844710636455477, "eval_recall": 0.8843843843843844, "eval_auroc": 0.8843843843843844, "eval_runtime": 2.3506, "eval_samples_per_second": 283.327, "eval_steps_per_second": 2.978, "epoch": 25.0}, "1.0": {"eval_loss": 0.2984059453010559, "eval_accuracy": 0.8993993993993994, "eval_f1_macro": 0.8992992789682137, "eval_f1_weighted": 0.8992992789682136, "eval_precision": 0.9009941329856584, "eval_recall": 0.8993993993993994, "eval_auroc": 0.8993993993993994, "eval_runtime": 2.3504, "eval_samples_per_second": 283.362, "eval_steps_per_second": 2.978, "epoch": 25.0}}

models/supervised/transfer_nested/eval_results.json

@@ -1 +1 @@
-{"0.25": {"eval_loss": 0.6932699680328369, "eval_accuracy": 0.496996996996997, "eval_f1_macro": 0.4623790648080097, "eval_f1_weighted": 0.4623790648080097, "eval_runtime": 2.43, "eval_samples_per_second": 274.07, "eval_steps_per_second": 2.881, "epoch": 25.0}, "0.5": {"eval_loss": 0.6928256154060364, "eval_accuracy": 0.524024024024024, "eval_f1_macro": 0.48958361035425635, "eval_f1_weighted": 0.48958361035425635, "eval_runtime": 2.359, "eval_samples_per_second": 282.326, "eval_steps_per_second": 2.967, "epoch": 25.0}, "0.75": {"eval_loss": 0.6927742958068848, "eval_accuracy": 0.521021021021021, "eval_f1_macro": 0.4888841308066313, "eval_f1_weighted": 0.4888841308066313, "eval_runtime": 2.3592, "eval_samples_per_second": 282.303, "eval_steps_per_second": 2.967, "epoch": 25.0}, "1.0": {"eval_loss": 0.6926871538162231, "eval_accuracy": 0.5195195195195195, "eval_f1_macro": 0.48930296756383707, "eval_f1_weighted": 0.48930296756383707, "eval_runtime": 2.4071, "eval_samples_per_second": 276.681, "eval_steps_per_second": 2.908, "epoch": 25.0}}
+{"0.25": {"eval_loss": 0.6932699680328369, "eval_accuracy": 0.496996996996997, "eval_f1_macro": 0.4623790648080097, "eval_f1_weighted": 0.4623790648080097, "eval_precision": 0.49595520357594014, "eval_recall": 0.49699699699699695, "eval_auroc": 0.49699699699699695, "eval_runtime": 2.3519, "eval_samples_per_second": 283.169, "eval_steps_per_second": 2.976, "epoch": 25.0}, "0.5": {"eval_loss": 0.6928256154060364, "eval_accuracy": 0.524024024024024, "eval_f1_macro": 0.48958361035425635, "eval_f1_weighted": 0.48958361035425635, "eval_precision": 0.532905138339921, "eval_recall": 0.524024024024024, "eval_auroc": 0.524024024024024, "eval_runtime": 2.3686, "eval_samples_per_second": 281.18, "eval_steps_per_second": 2.955, "epoch": 25.0}, "0.75": {"eval_loss": 0.6927742958068848, "eval_accuracy": 0.521021021021021, "eval_f1_macro": 0.4888841308066313, "eval_f1_weighted": 0.4888841308066313, "eval_precision": 0.5280843373493975, "eval_recall": 0.5210210210210211, "eval_auroc": 0.521021021021021, "eval_runtime": 2.35, "eval_samples_per_second": 283.403, "eval_steps_per_second": 2.979, "epoch": 25.0}, "1.0": {"eval_loss": 0.6926871538162231, "eval_accuracy": 0.5195195195195195, "eval_f1_macro": 0.48930296756383707, "eval_f1_weighted": 0.48930296756383707, "eval_precision": 0.5255715045188729, "eval_recall": 0.5195195195195195, "eval_auroc": 0.5195195195195196, "eval_runtime": 2.3534, "eval_samples_per_second": 282.994, "eval_steps_per_second": 2.974, "epoch": 25.0}}

train.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+python /teamspace/studios/this_studio/npr-sentiment/src/model_pipeline.py eval 
+python /teamspace/studios/this_studio/npr-sentiment/src/model_pipeline.py  finetune --nested-splits --batch-size 105 --num-epochs 25
+python /teamspace/studios/this_studio/npr-sentiment/src/model_pipeline.py  transfer --nested-splits --batch-size 105 --num-epochs 25
+python /teamspace/studios/this_studio/npr-sentiment/src/model_pipeline.py  finetune --nested-splits --weak-label-path "/teamspace/studios/this_studio/npr-sentiment/data/weak_labelled/log_reg_weaklabels.parquet" --batch-size 105 --num-epochs 25