diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_1024dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_1024dp.yaml new file mode 100644 index 0000000..0e1f3ae --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_1024dp.yaml @@ -0,0 +1,79 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 1024 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 5 + eval_each_epochs: 2 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_128dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_128dp.yaml new file mode 100644 index 0000000..7aa7ae2 --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_128dp.yaml @@ -0,0 +1,79 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 128 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 17 + eval_each_epochs: 8 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_16dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_16dp.yaml new file mode 100644 index 0000000..c745856 --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_16dp.yaml @@ -0,0 +1,79 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 16 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 65 + eval_each_epochs: 32 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_2dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_2dp.yaml new file mode 100644 index 0000000..200dda2 --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_2dp.yaml @@ -0,0 +1,80 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + # n_seeds_stage2: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 2 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 129 + eval_each_epochs: 64 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_32dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_32dp.yaml new file mode 100644 index 0000000..c21f31e --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_32dp.yaml @@ -0,0 +1,79 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 32 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 33 + eval_each_epochs: 16 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_4dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_4dp.yaml new file mode 100644 index 0000000..3d6472f --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_4dp.yaml @@ -0,0 +1,80 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + # n_seeds_stage2: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 4 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 129 + eval_each_epochs: 64 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/configs/password_locked/sample_efficiency/2stage_unlock_512dp.yaml b/configs/password_locked/sample_efficiency/2stage_unlock_512dp.yaml new file mode 100644 index 0000000..fb996a0 --- /dev/null +++ b/configs/password_locked/sample_efficiency/2stage_unlock_512dp.yaml @@ -0,0 +1,79 @@ +data_arguments: + dataset: "cvdb" + block_size: 24 + label_block_size: 4 + + +model_arguments: + seq2seq: False + max_new_tokens: 18 + # config_name: "gpt2" + # config_name: "t5-small" + config_name: "EleutherAI/pythia-70m" + # config_name: "EleutherAI/pythia-160m" + separate_token_per_var: False # only used for numeric experiments + + +training_arguments: + output_dir: 'experiments/temp' + bf16: True + per_device_train_batch_size: 512 + per_device_eval_batch_size: 2048 + optim: "adafactor" + # optim: "lion_32bit" + overwrite_output_dir: True + auto_find_batch_size: True + save_strategy: "no" + load_best_model_at_end: False + evaluation_strategy: 'epoch' + do_train: True + do_eval: True + do_sweeps: False + # n_sweeps: 5 + save_each_epochs: 0 + eval_each_epochs: 1 + eval_callback_type: "pipeline" # pipeline or generate + # weight_decay: 0.0001 + + +experiment_arguments: # common experiment arguments + define_experiment: False + numeric_experiment: True + name_prefix: "sample_efficiency_fromScratch" + n_stages: 2 + n_seeds: 5 + start_seed: 1010 + slurm: True + n_gpu_hours: 3 + + +define_experiment_arguments: + def_order: "tve" + + +numeric_experiment_arguments: + # Args for pwd composition experiment below + pwd_locked_experiment: True + n_datapoints: 200000 + max_unlocking_datapoints: 512 + max_x: 10 + n_func_in_chain: 2 + fn_input_len: 4 + nfunc: 32 + n_fns_to_lock: 16 + n_fns_to_unlock: 1 + + +# overrides specified parameters +first_stage_arguments: + train_subset: 'stage2' + num_train_epochs: 5 + gradient_accumulation_steps: 1 + +second_stage_arguments: + train_subset: 'stage3' + num_train_epochs: 9 + eval_each_epochs: 4 + gradient_accumulation_steps: 1 + dont_save_in_the_end: True + save_each_epochs: 0 \ No newline at end of file diff --git a/data_generation/pwd_locked_composition.py b/data_generation/pwd_locked_composition.py index f8cfb49..ecff28e 100644 --- a/data_generation/pwd_locked_composition.py +++ b/data_generation/pwd_locked_composition.py @@ -101,12 +101,12 @@ def make_ifPwdThen_fns(rng, functions, n_fns_to_lock=2) -> List[IfPwdElseFunctio returns a list of locked functions and a list of passwords (pwd is empty if the function is not locked) """ # shuffle the functions and lock the first n_fns_to_lock - # shuffled_fns = rng.sample(functions, len(functions)) + shuffled_fns = rng.sample(functions, len(functions)) # currently the first half of the fns are permutations and the second half are bijections # order them s.t. the first quarter are bijections, then the second quarter are permutations, then again bijections, and so on - m = len(functions) // 4 - shuffled_fns = functions[:m] + functions[2*m:3*m] + functions[m:2*m] + functions[3*m:] + # m = len(functions) // 4 + # shuffled_fns = functions[:m] + functions[2*m:3*m] + functions[m:2*m] + functions[3*m:] assert len(shuffled_fns) == len(functions) out = []