-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathget_full_model_components_over_time.py
195 lines (162 loc) · 6.59 KB
/
get_full_model_components_over_time.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
from typing import Tuple, List
from functools import partial
import os
import argparse
import yaml
import numpy as np
import pandas as pd
import torch
from utils.data_processing import (
load_edge_scores_into_dictionary,
get_ckpts,
get_ckpts
)
from utils.backup_analysis import load_model
from utils.data_utils import generate_data_and_caches
from utils.component_evaluation import (
#evaluate_direct_effect_heads,
#filter_name_movers,
#evaluate_s2i_candidates,
evaluate_induction_scores_full_model
)
def print_gpu_memory_usage(label="", device="cuda:0"):
allocated = torch.cuda.memory_allocated(device) / (1024 ** 3) # Convert bytes to GB
reserved = torch.cuda.memory_reserved(device) / (1024 ** 3)
print(f"{label} - Memory Allocated: {allocated:.2f} GB, Memory Reserved: {reserved:.2f} GB")
def get_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Download & assess model checkpoints")
parser.add_argument(
"-c",
"--config",
default=None,
help="Path to config file",
)
return parser.parse_args()
def read_config(config_path):
with open(config_path, "r") as f:
config = yaml.load(f, Loader=yaml.FullLoader)
return config
def process_args():
# Returns a namespace of arguments either from a config file or from the command line
args = get_args()
if args.config is not None:
config = read_config(args.config)
for key, value in config.items():
setattr(args, key, value)
# Placeholder to revisit when we want to add different model seed variants
if not args.variant:
setattr(args, "canonical_model", True)
else:
setattr(args, "canonical_model", False)
return args
def get_ckpts(schedule):
if schedule == "all":
ckpts = (
[0]
+ [2**i for i in range(10)]
+ [i * 1000 for i in range(1, 144)]
)
elif schedule == "linear":
ckpts = [i * 1000 for i in range(1, 144)]
elif schedule == "exponential":
ckpts = [
round((2**i) / 1000) * 1000 if 2**i > 1000 else 2**i
for i in range(18)
]
elif schedule == "exp_plus_detail":
ckpts = (
[2**i for i in range(10)]
+ [i * 1000 for i in range(1, 16)]
+ [i * 5000 for i in range(3, 14)]
+ [i * 10000 for i in range(7, 15)]
)
elif schedule == "late_start_exp_plus_detail":
ckpts = (
[i * 4000 for i in range(1, 16)]
+ [i * 5000 for i in range(3, 14)]
+ [i * 10000 for i in range(7, 15)]
)
elif schedule == "late_start_all":
ckpts = (
[2**i for i in range(8, 10)]
+ [i * 1000 for i in range(1, 144)]
)
elif schedule == "sparse":
ckpts = (
[2**i for i in range(8, 10)]
+ [i * 1000 for i in range(1, 10)]
+ [i * 5000 for i in range(2, 10)]
+ [i * 10000 for i in range(5, 10)]
+ [i * 20000 for i in range(5, 8)]
+ [143000]
)
elif schedule == "custom":
ckpts = []
else:
ckpts = [10000, 143000]
return ckpts
def main(args):
torch.set_grad_enabled(False)
config = read_config(args.config)
print(config)
TASK = config['task']
BASE_MODEL = config['base_model']
VARIANT = config['variant']
MODEL_SHORTNAME =BASE_MODEL if not VARIANT else VARIANT[11:]
CACHE = config['cache']
DATASET_SIZE = config['dataset_size']
BATCH_SIZE = config['batch_size']
CHECKPOINT_SCHEDULE = get_ckpts(config['checkpoint_schedule'])
if 'device' in config:
DEVICE = config['device']
else:
DEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'
overwrite = config['overwrite']
print(f"{config}")
print(f"Checkpoint schedule: {CHECKPOINT_SCHEDULE}")
# load model and dataset
model = load_model(BASE_MODEL, VARIANT, 143000, CACHE, DEVICE, large_model=False)
model.tokenizer.add_bos_token = False
#ioi_dataset, abc_dataset = generate_data_and_caches(model, DATASET_SIZE, verbose=True, prepend_bos=True)
for checkpoint in CHECKPOINT_SCHEDULE:
# check if file exists; if not, create
if not os.path.exists(f'results/components/{MODEL_SHORTNAME}/full_model_components_over_time.pt'):
os.makedirs(f'results/components/{MODEL_SHORTNAME}', exist_ok=True)
components_over_time = dict()
heads_over_time = dict()
else:
components_over_time = torch.load(f'results/components/{MODEL_SHORTNAME}/full_model_components_over_time.pt')
#heads_over_time = torch.load(f'results/components/{MODEL_SHORTNAME}/heads_over_time.pt')
if checkpoint in components_over_time and not overwrite:
print(f"Checkpoint {checkpoint} already processed")
continue
print(f"Processing checkpoint {checkpoint}")
model = load_model(BASE_MODEL, VARIANT, checkpoint, CACHE, DEVICE, large_model=True)
print_gpu_memory_usage("After loading first checkpoint model")
component_scores = dict()
#model_heads = dict()
#component_scores['direct_effect_scores'] = evaluate_direct_effect_heads(model, checkpoint_df, ioi_dataset, verbose=False, cuda_device=int(DEVICE[-1]), batch_size=BATCH_SIZE)
# if component_scores['direct_effect_scores'] is not None:
# nmh_list = filter_name_movers(component_scores['direct_effect_scores'], copy_score_threshold=10)
# else:
# nmh_list = []
# model_heads['nmh'] = nmh_list
# print(f"Found {len(nmh_list)} NMHs")
# print(nmh_list)
# if len(nmh_list) > 0:
# component_scores['s2i_scores'], s2i_list = evaluate_s2i_candidates(model, checkpoint_df, ioi_dataset, nmh_list, batch_size=BATCH_SIZE, verbose=False)
# print(f"Found {len(s2i_list)} S2I heads")
# print(s2i_list)
# else:
# component_scores['s2i_scores'] = None
# s2i_list = []
# model_heads['s2i'] = s2i_list
component_scores['tertiary_head_scores'] = evaluate_induction_scores_full_model(model)
components_over_time[checkpoint] = component_scores
#heads_over_time[checkpoint] = model_heads
torch.save(components_over_time, f'results/components/{MODEL_SHORTNAME}/full_model_components_over_time.pt')
#torch.save(heads_over_time, f'results/components/{MODEL_SHORTNAME}/heads_over_time.pt')
return components_over_time
if __name__ == "__main__":
args = process_args()
main(args)