Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugfix parameter loading through experiment configs #252

Merged
merged 2 commits into from
Feb 18, 2020
Merged

Bugfix parameter loading through experiment configs #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
69c6890
Bugfix parameter loading through config, adjust configs, add bert2.0 …
Timoeller Feb 18, 2020
03ceab0
Adjust configs
Timoeller Feb 18, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions experiments/german-bert2.0-eval/germEval14_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"general": {
"cache_dir": {"value": null, "default": "", "desc": "Path for storing pre-trained models downloaded from s3."},
"data_dir": {"value": null, "default": "data/germeval14", "desc": "Input directory for downstream task. Should contain train + test (+ dev) files."},
"output_dir": {"value": null, "default": "saved_models", "desc": "Output directory where model predictions and checkpoints will be saved."},

"cuda": {"value": null, "default": true, "desc": "CUDA flag, uses CUDA if available."},
"local_rank": {"value": null, "default": -1, "desc": "If local_rank == -1 -> multiGPU mode on one machine, other values signal distributed computation across several nodes (apex install required)."},
"use_amp": {"value": null, "default": false, "desc": "Automatic mixed precision with APEX. Must be set to null to disable or to any optimisation level (see apex documentation). 'O1' is recommended."},
"seed": {"value": null, "default": 42, "desc": "Random seed for initializations."}
},

"task": {
"name": {"value": null, "default": "GermEval14", "desc": "Name of task."},
"type": {"value": null, "default": "ner"},
"language": {"value": null, "default": "de"},
"do_eval": {"value": null, "default": true, "desc": "Whether to run eval on the dev set."},
"do_train": {"value": null, "default": true, "desc": "Whether to run training. Can be used to only evaluate on an already trained model."},

"processor_name": {"value": null, "default": "NERProcessor", "desc": "A Dataprocessor that is suited for tabular data. Needs special data parameters defined."},
"dev_split": {"value": null, "default": 0.1, "desc": "Split a dev set from the training set using dev_split as proportion."},
"train_filename": {"value": null, "default": "train.txt", "desc": "Filename for training."},
"dev_filename": {"value": null, "default": null, "desc": "Filename for development. Missing in case of GermEval2018."},
"test_filename": {"value": null, "default": "test.txt", "desc": "Filename for testing. It is the submission file from competition."},
"delimiter": {"value": null, "default": " ", "desc": "Delimiter used to seprate columns in input data."},
"label_list": {"value": null, "default": ["[PAD]", "X", "O", "B-MISC", "I-MISC", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC", "B-OTH", "I-OTH"], "desc": ""},
"metric": {"value": null, "default": "seq_f1", "desc": "Metric used. A f1 scored tailored to sequences of labels."}
},

"parameter": {
"model": {"value": "bert-base-german-cased", "default": null, "desc": "Bert pre-trained model selected in the list: bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese."},
"prediction_head": {"value": null, "default": "TokenClassificationHead", "desc": "Kind of prediction head we use on top of Language Model"},
"lm_output_type": {"value": null, "default": "per_token", "desc": "Language Model output."},
"lower_case": {"value": null, "default": false, "desc": "Set to true if you are using an uncased model."},
"max_seq_len": {"value": null, "default": 128, "desc": "The maximum total input sequence length after WordPiece tokenization. Some GNAD texts even extend beyond 512 tokens."},
"balance_classes": {"value": null, "default": false, "desc": "Balance classes using weighted CrossEntropyLoss."},

"epochs": {"value": null, "default": 4, "desc": "Total number of training epochs to perform."},
"batch_size": {"value": null, "default": 64, "desc": "Total batch size for training on a single V100 GPU."},
"gradient_accumulation_steps": {"value": null, "default": 1, "desc": "Number of updates steps (batches) to accumulate before performing a backward/update pass."},
"embeds_dropout_prob": {"value": null, "default": 0.1, "desc": "Strength of dropout to be applied to the word embeddings generated by the language model."},
"layer_dims": {"value": null, "default": [768, 15], "desc": "Cannot do experiments on this value, since it is already a list. Dimensions of the prediction head. Needs to be of type String, otherwise it gets iterated over."}
},

"optimizer": {
"learning_rate": {"value": null, "default": 5e-5, "desc": "The learning rate for the optimizer."},
"optimizer_opts": {"value": null, "default": null, "desc": "Additional optimizer config."},
"schedule_opts": {"value": null, "default": {"name": "LinearWarmup", "warmup_proportion": 0.4}, "desc": "opts for lr schedule"}
},

"logging": {
"eval_every": {"value": null, "default": 60, "desc": "Steps per training loop (batches) required for evaluation on dev set. Set to 0 when you do not want to do evaluation on dev set during training."},
"mlflow_url": {"value": "https://public-mlflow.deepset.ai/", "default": null, "desc": "Mlflow server for tracking experiments (e.g. http://80.123.45.167:5000/)"},
"mlflow_nested": {"value": null, "default": true, "desc": "Nesting mlflow experiments. For doing multiple runs across a set of hyperparameters."},

"mlflow_experiment": {"value": "Public_FARM", "default": null, "desc": "Experiment name used for mlflow"},
"mlflow_run_name": {"value": "germeval14 by config", "default": null, "desc": "Name of the particular run for mlflow"}
}
}







61 changes: 61 additions & 0 deletions experiments/german-bert2.0-eval/germEval18Coarse_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"general": {
"cache_dir": {"value": null, "default": "", "desc": "Path for storing pre-trained models downloaded from s3."},
"data_dir": {"value": null, "default": "data/germeval18", "desc": "Input directory for downstream task. Should contain train + test (+ dev) files."},
"output_dir": {"value": null, "default": "saved_models", "desc": "Output directory where model predictions and checkpoints will be saved."},

"cuda": {"value": null, "default": true, "desc": "CUDA flag, uses CUDA if available."},
"local_rank": {"value": null, "default": -1, "desc": "If local_rank == -1 -> multiGPU mode on one machine, other values signal distributed computation across several nodes (apex install required)."},
"use_amp": {"value": null, "default": null, "desc": "Automatic mixed precision with APEX. Must be set to null to disable or to any optimisation level (see apex documentation). 'O1' is recommended."},
"seed": {"value": null, "default": 42, "desc": "Random seed for initializations."}
},

"task": {
"type": {"value": null, "default": "text_classification"},
"language": {"value": null, "default": "de"},
"name": {"value": null, "default": "GermEval18Coarse", "desc": "GermEval18Coarse: binary offensive language detection."},
"do_eval": {"value": null, "default": true, "desc": "Whether to run eval on the dev set."},
"do_train": {"value": null, "default": true, "desc": "Whether to run training. Can be used to only evaluate on an already trained model."},

"processor_name": {"value": null, "default": "TextClassificationProcessor","desc": "A Dataprocessor that is suited for tabular data. Needs special data parameters defined."},
"dev_split": {"value": null, "default": 0.1, "desc": "Split a dev set from the training set using dev_split as proportion."},
"train_filename": {"value": null, "default": "train.tsv", "desc": "Filename for training."},
"dev_filename": {"value": null, "default": null, "desc": "Filename for development. Missing in case of GermEval2018."},
"test_filename": {"value": null, "default": "test.tsv", "desc": "Filename for testing. It is the submission file from competition."},
"delimiter": {"value": null, "default": "\t", "desc": "Filename for testing. It is the submission file from competition."},
"columns": {"value": null, "default": ["text", "label", "unused"], "desc": "Columns specifying position of text and labels in data files."},
"label_list": {"value": null, "default": ["OTHER", "OFFENSE"], "desc": "List of possible labels."},
"metric": {"value": null, "default": "f1_macro", "desc": "Metric used. The competition uses macro averaged f1 score."},
"label_column_name":{"value": null, "default": "coarse_label", "desc":"Name of field that the label comes from in datasource"},
"skiprows": {"value": null, "default": null, "desc":""}
},
"parameter": {
"model": {"value": "bert-base-german-cased", "default": null, "desc": "Bert pre-trained model selected in the list: bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese."},
"prediction_head": {"value": null, "default": "TextClassificationHead", "desc": "Kind of prediction head we use on top of Language Model"},
"lm_output_type": {"value": null, "default": "per_sequence", "desc": "Language Model output."},
"lower_case": {"value": null, "default": false, "desc": "Set to true if you are using an uncased model."},
"max_seq_len": {"value": null, "default": 150, "desc": "The maximum total input sequence length after WordPiece tokenization. 128 was too short for some texts"},
"balance_classes": {"value": null, "default": true, "desc": "Balance classes using weighted CrossEntropyLoss. Original train set from GermEval18 is skewed and the final evaluation is macro averaged, so we need to balance for optimal performance.."},

"epochs": {"value": null, "default": 2.0, "desc": "Total number of training epochs to perform."},
"batch_size": {"value": null, "default": 48, "desc": "Total batch size for training on a single V100 GPU."},
"gradient_accumulation_steps": {"value": null, "default": 1, "desc": "Number of updates steps (batches) to accumulate before performing a backward/update pass."},
"embeds_dropout_prob": {"value": null, "default": 0.1, "desc": "Strength of dropout to be applied to the word embeddings generated by the language model."},
"layer_dims": {"value": null, "default": [768, 2], "desc": "Cannot do experiments on this value, since it is already a list. Dimensions of the prediction head. Needs to be of type String, otherwise it gets iterated over."}
},
"optimizer": {
"learning_rate": {"value": null, "default": 2e-5, "desc": "The initial learning rate for AdamW."},
"optimizer_opts": {"value": null, "default": null, "desc": "Additional optimizer config."},
"schedule_opts": {"value": null, "default": {"name": "LinearWarmup", "warmup_proportion": 0.2}, "desc": "opts for lr schedule"}
},
"logging": {
"eval_every": {"value": null, "default": 50, "desc": "Steps per training loop (batches) required for evaluation on dev set. Set to 0 when you do not want to do evaluation on dev set during training."},
"mlflow_url": {"value": "https://public-mlflow.deepset.ai/", "default": null, "desc": "Mlflow server for tracking experiments (e.g. http://80.123.45.167:5000/)"},
"mlflow_nested": {"value": null, "default": true, "desc": "Nesting mlflow experiments. For doing multiple runs across a set of hyperparameters."},

"mlflow_experiment": {"value": "Public_FARM", "default": null, "desc": "Experiment name used for mlflow"},
"mlflow_run_name": {"value": "germeval coarse by config", "default": null, "desc": "Name of the particular run for mlflow"}
}
}


62 changes: 62 additions & 0 deletions experiments/german-bert2.0-eval/germEval18Fine_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
{
"general": {
"cache_dir": {"value": null, "default": "", "desc": "Path for storing pre-trained models downloaded from s3."},
"data_dir": {"value": null, "default": "data/germeval18", "desc": "Input directory for downstream task. Should contain train + test (+ dev) files."},
"output_dir": {"value": null, "default": "saved_models", "desc": "Output directory where model predictions and checkpoints will be saved."},

"cuda": {"value": null, "default": true, "desc": "CUDA flag, uses CUDA if available."},
"local_rank": {"value": null, "default": -1, "desc": "If local_rank == -1 -> multiGPU mode on one machine, other values signal distributed computation across several nodes (apex install required)."},
"use_amp": {"value": null, "default": null, "desc": "Automatic mixed precision with APEX. Must be set to null to disable or to any optimisation level (see apex documentation). 'O1' is recommended."},
"seed": {"value": null, "default": 42, "desc": "Random seed for initializations."}
},

"task": {
"type": {"value": null, "default": "text_classification"},
"language": {"value": null, "default": "de"},
"name": {"value": null, "default": "GermEval18Fine", "desc": "GermEval18Fine: Finegrained multiclass offensive language detection, 4 classes."},
"do_eval": {"value": null, "default": true, "desc": "Whether to run eval on the dev set."},
"do_train": {"value": null, "default": true, "desc": "Whether to run training. Can be used to only evaluate on an already trained model."},

"processor_name": {"value": null, "default": "TextClassificationProcessor","desc": "A Dataprocessor that is suited for tabular data. Needs special data parameters defined."},
"dev_split": {"value": null, "default": 0.1, "desc": "Split a dev set from the training set using dev_split as proportion."},
"train_filename": {"value": null, "default": "train.tsv", "desc": "Filename for training."},
"dev_filename": {"value": null, "default": null, "desc": "Filename for development. Missing in case of GermEval2018."},
"test_filename": {"value": null, "default": "test.tsv", "desc": "Filename for testing. It is the submission file from competition."},
"delimiter": {"value": null, "default": "\t", "desc": "Filename for testing. It is the submission file from competition."},
"columns": {"value": null, "default": ["text", "unused", "label"], "desc": "Columns specifying position of text and labels in data files."},
"label_list": {"value": null, "default": ["OTHER", "INSULT", "ABUSE", "PROFANITY"],"desc": "List of possible labels."},
"metric": {"value": null, "default": "f1_macro", "desc": "Metric used. The competition uses macro averaged f1 score."},
"label_column_name":{"value": null, "default": "fine_label", "desc":"Name of field that the label comes from in datasource"},
"skiprows": {"value": null, "default": null, "desc":""}
},

"parameter": {
"model": {"value": "bert-base-german-cased", "default": null, "desc": "Bert pre-trained model selected in the list: bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese."},
"prediction_head": {"value": null, "default": "TextClassificationHead", "desc": "Kind of prediction head we use on top of Language Model"},
"lm_output_type": {"value": null, "default": "per_sequence", "desc": "Language Model output."},
"lower_case": {"value": null, "default": false, "desc": "Set to true if you are using an uncased model."},
"max_seq_len": {"value": null, "default": 150, "desc": "The maximum total input sequence length after WordPiece tokenization. 128 was too short for some texts"},
"balance_classes": {"value": null, "default": true, "desc": "Balance classes using weighted CrossEntropyLoss. Original train set from GermEval18 is skewed and the final evaluation is macro averaged, so we need to balance for optimal performance.."},

"epochs": {"value": null, "default": 3.0, "desc": "Total number of training epochs to perform."},
"batch_size": {"value": null, "default": 48, "desc": "Total batch size for training on a single V100 GPU."},
"gradient_accumulation_steps": {"value": null, "default": 1, "desc": "Number of updates steps (batches) to accumulate before performing a backward/update pass."},
"embeds_dropout_prob": {"value": null, "default": 0.1, "desc": "Strength of dropout to be applied to the word embeddings generated by the language model."},
"layer_dims": {"value": null, "default": [768, 4], "desc": "Cannot do experiments on this value, since it is already a list. Dimensions of the prediction head. Needs to be of type String, otherwise it gets iterated over."}
},
"optimizer": {
"learning_rate": {"value": null, "default": 2e-5, "desc": "The initial learning rate for AdamW."},
"optimizer_opts": {"value": null, "default": null, "desc": "Additional optimizer config."},
"schedule_opts": {"value": null, "default": {"name": "LinearWarmup", "warmup_proportion": 0.2}, "desc": "opts for lr schedule"}
},
"logging": {
"eval_every": {"value": null, "default": 50, "desc": "Steps per training loop (batches) required for evaluation on dev set. Set to 0 when you do not want to do evaluation on dev set during training."},
"mlflow_url": {"value": "https://public-mlflow.deepset.ai/", "default": null, "desc": "Mlflow server for tracking experiments (e.g. http://80.123.45.167:5000/)"},
"mlflow_nested": {"value": null, "default": true, "desc": "Nesting mlflow experiments. For doing multiple runs across a set of hyperparameters."},

"mlflow_experiment": {"value": "Public_FARM", "default": null, "desc": "Experiment name used for mlflow"},
"mlflow_run_name": {"value": "germeval fine by config", "default": null, "desc": "Name of the particular run for mlflow"}
}
}


Loading