-
Notifications
You must be signed in to change notification settings - Fork 0
/
leaderboard_generation_format.json
85 lines (82 loc) · 9.33 KB
/
leaderboard_generation_format.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
{
"Reinforcement Learning Energy Leaderboard" : {
"filter" : null,
"description" : "Reinforcement Learning Experiments, tracking efficiency versus performance of various implementations and environments. Environments listed on the left. Click on them to see algorithmic performance (and click further to get details on hardware used, among other details).<br/><br/> <div class=\"alert alert-success fade in\"> <strong>Call for contributions!</strong> We're always looking for more data to find the most efficient settings! Please <a href=\"https://github.com/Breakend/RL-Energy-Leaderboard\">send us a pull request with your runs!</a></div>",
"executive_summary_variables" : ["total_power", "exp_len_hours", "cpu_hours", "gpu_hours", "estimated_carbon_impact_kg"],
"child_experiments" :
{
"PongNoFrameskip-v4 Experiments" : {
"filter" : "(Pong)",
"description" : "PongNoFrameskip-v4 expeirments. Evaluate on separate environments every 250k timesteps in parallel (see code for details), run for 5M timesteps (roughly 23.15 hrs of experience).",
"executive_summary_variables" : ["AverageReturnPerkWh", "AverageReturn", "AsymptoticReturn", "total_power", "exp_len_hours", "cpu_hours", "gpu_hours", "estimated_carbon_impact_kg"],
"executive_summary_ordering_variable" : "AverageReturnPerkWh",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval",
"executive_summary_plots" : [
{"x" : "total_power", "y" : "AverageReturn"},
{"x" : "total_power", "y" : "AsymptoticReturn"},
{"x" : "exp_len_hours", "y" : "AverageReturn"},
{"x" : "exp_len_hours", "y" : "AsymptoticReturn"}
],
"child_experiments" :
{
"PPO2 (stable_baselines, default settings)" : {
"filter" : "(ppo2)",
"description" : "PPO2 experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo ppo2 --env PongNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_ppo2_PongNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_ppo2_PongNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/ppo2.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"A2C (stable_baselines, default settings)" : {
"filter" : "(.*stable_baseline.*.a2c.*)",
"description" : "A2C experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo a2c --env PongNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_a2c_PongNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_a2c_PongNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/a2c.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"DQN (stable_baselines, default settings)" : {
"filter" : "(.*stable_baseline.*.dqn.*)",
"description" : "DQN experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo dqn --env PongNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_dqn_PongNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_dqn_PongNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/dqn.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"A2C+Vtrace (cule, default settings)" : {
"filter" : "(.*vtrace.*.a2c.*)|(.*a2c.*.vtrace.*)",
"description" : "A2C+Vtrace using default settings from here: https://github.com/NVlabs/cule .<br/> Exact command run: python vtrace_main.py --env-name PongNoFrameskip-v4 --normalize --use-cuda-env --num-ales 1200 --num-steps 20 --num-steps-per-update 1 --num-minibatches 20 --t-max 5000000 --evaluation-episodes 25 --evaluation-interval 250000 --seed 4 --output-filename ./cule_results/Pong_seed_4_logs_a2c_vtrace_cule/eval.csv --log-dir ./cule_results/Pong_seed_4_logs_a2c_vtrace_cule/",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
}
}
},
"BreakoutNoFrameskip-v4 Experiments" : {
"filter" : "(Breakout)",
"description" : "BreakoutNoFrameskip-v4 expeirments. Evaluate on separate environments every 250k timesteps in parallel (see code for details), run for 5M timesteps (roughly 23.15 hrs of experience).",
"executive_summary_variables" : ["AverageReturnPerkWh", "AverageReturn", "AsymptoticReturn", "total_power", "exp_len_hours", "cpu_hours", "gpu_hours", "estimated_carbon_impact_kg"],
"executive_summary_ordering_variable" : "AverageReturnPerkWh",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval",
"executive_summary_plots" : [
{"x" : "total_power", "y" : "AverageReturn"},
{"x" : "total_power", "y" : "AsymptoticReturn"},
{"x" : "exp_len_hours", "y" : "AverageReturn"},
{"x" : "exp_len_hours", "y" : "AsymptoticReturn"}
],
"child_experiments" :
{
"PPO2 (stable_baselines, default settings)" : {
"filter" : "(ppo2)",
"description" : "PPO2 experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo ppo2 --env BreakoutNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_ppo2_BreakoutNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_ppo2_BreakoutNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/ppo2.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"A2C (stable_baselines, default settings)" : {
"filter" : "(.*stable_baseline.*.a2c.*)",
"description" : "A2C experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo a2c --env BreakoutNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_a2c_BreakoutNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_a2c_BreakoutNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/a2c.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"DQN (stable_baselines, default settings)" : {
"filter" : "(.*stable_baseline.*.dqn.*)",
"description" : "DQN experiments using default settings of stable_baselines repository.<br/>See https://github.com/araffin/rl-baselines-zoo/.<br/>Exact command run: python train.py --ignore-hardware --algo dqn --env BreakoutNoFrameskip-v4 --tensorboard-log ./stable_baselines_results/seed_0_dqn_BreakoutNoFrameSkip-v4_gpu_normal/ -f ./stable_baselines_results/seed_0_dqn_BreakoutNoFrameSkip-v4_gpu_normal/ --seed 0 --n-timesteps 5000000 --hparam_file hyperparams/dqn.yml ",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
},
"A2C+Vtrace (cule, default settings)" : {
"filter" : "(.*vtrace.*.a2c.*)|(.*a2c.*.vtrace.*)",
"description" : "A2C+Vtrace using default settings from here: https://github.com/NVlabs/cule .<br/> Exact command run: python vtrace_main.py --env-name BreakoutNoFrameskip-v4 --normalize --use-cuda-env --num-ales 1200 --num-steps 20 --num-steps-per-update 1 --num-minibatches 20 --t-max 5000000 --evaluation-episodes 25 --evaluation-interval 250000 --seed 4 --output-filename ./cule_results/Breakout_seed_4_logs_a2c_vtrace_cule/eval.csv --log-dir ./cule_results/Breakout_seed_4_logs_a2c_vtrace_cule/",
"extra_files_processor" : "rlenergyleaderboard.processors.load_rl_eval"
}
}
}
}
}
}