From 924cd2ec215bacb5ab5a1dc41b2fe82b263ec8ba Mon Sep 17 00:00:00 2001
From: Alexis Asseman <alexis@semiotic.ai>
Date: Tue, 15 Nov 2022 17:01:39 -0800
Subject: [PATCH] feat: updating experiment configs

Signed-off-by: Alexis Asseman <alexis@semiotic.ai>
---
 .../configs/10rollingppo_isa_nopullback.json  | 118 ++++++++++++++++++
 simulation/configs/1rollingppo_noisy.json     |  26 ++++
 .../configs/1rollingppo_noisy_cyclic.json     |  26 ++++
 .../configs/1rollingppo_noisy_nopullback.json |  26 ++++
 .../1rollingppo_noisy_shared_cyclic.json      |  27 ++++
 .../1rollingppo_noisy_shared_cyclic_zero.json |  27 ++++
 ...o_noisy_shared_cyclic_zero_nopullback.json |  27 ++++
 ...ic1rl_agents_noisy_shared_nopullback.json} |   0
 .../configs/3deterministic_agents_isa.json    |  26 ++++
 .../3different_agents_isa_nopullback.json     |  52 ++++++++
 .../3different_rollingppo_isa_nopullback.json |  41 ++++++
 ...tic1rl_agents_noisy_shared_nopullback.json |  46 +++++++
 ...ollmem_agents_noisy_shared_nopullback.json |  42 +++++++
 13 files changed, 484 insertions(+)
 create mode 100644 simulation/configs/10rollingppo_isa_nopullback.json
 create mode 100644 simulation/configs/1rollingppo_noisy.json
 create mode 100644 simulation/configs/1rollingppo_noisy_cyclic.json
 create mode 100644 simulation/configs/1rollingppo_noisy_nopullback.json
 create mode 100644 simulation/configs/1rollingppo_noisy_shared_cyclic.json
 create mode 100644 simulation/configs/1rollingppo_noisy_shared_cyclic_zero.json
 create mode 100644 simulation/configs/1rollingppo_noisy_shared_cyclic_zero_nopullback.json
 rename simulation/configs/{3deterministic1rl_agents_noisy_shared.json => 3deterministic1rl_agents_noisy_shared_nopullback.json} (100%)
 create mode 100644 simulation/configs/3deterministic_agents_isa.json
 create mode 100644 simulation/configs/3different_agents_isa_nopullback.json
 create mode 100644 simulation/configs/3different_rollingppo_isa_nopullback.json
 create mode 100644 simulation/configs/3heuristic1rl_agents_noisy_shared_nopullback.json
 create mode 100644 simulation/configs/3rollmem_agents_noisy_shared_nopullback.json

diff --git a/simulation/configs/10rollingppo_isa_nopullback.json b/simulation/configs/10rollingppo_isa_nopullback.json
new file mode 100644
index 0000000..a05620c
--- /dev/null
+++ b/simulation/configs/10rollingppo_isa_nopullback.json
@@ -0,0 +1,118 @@
+{
+    "agents": {
+        "rolling_ppo_1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_2": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_3": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_4": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_5": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_6": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_7": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_8": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_9": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_10": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        }
+    },
+    "environment": {
+        "isa": {
+        }
+    }
+}
diff --git a/simulation/configs/1rollingppo_noisy.json b/simulation/configs/1rollingppo_noisy.json
new file mode 100644
index 0000000..f15fe41
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy.json
@@ -0,0 +1,26 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": true
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-7,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisySharedSubgraph": {
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/1rollingppo_noisy_cyclic.json b/simulation/configs/1rollingppo_noisy_cyclic.json
new file mode 100644
index 0000000..04e706c
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy_cyclic.json
@@ -0,0 +1,26 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisyCyclicZeroQueriesSubgraph": {
+            "cycle": 1000,
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/1rollingppo_noisy_nopullback.json b/simulation/configs/1rollingppo_noisy_nopullback.json
new file mode 100644
index 0000000..3baef07
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy_nopullback.json
@@ -0,0 +1,26 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-7,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisySharedSubgraph": {
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/1rollingppo_noisy_shared_cyclic.json b/simulation/configs/1rollingppo_noisy_shared_cyclic.json
new file mode 100644
index 0000000..0de1349
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy_shared_cyclic.json
@@ -0,0 +1,27 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": true
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisyCyclicSharedSubgraph": {
+            "cycle": 200,
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/1rollingppo_noisy_shared_cyclic_zero.json b/simulation/configs/1rollingppo_noisy_shared_cyclic_zero.json
new file mode 100644
index 0000000..ef70f7b
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy_shared_cyclic_zero.json
@@ -0,0 +1,27 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": true
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 5e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisyCyclicSharedSubgraph": {
+            "cycle": 200,
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/1rollingppo_noisy_shared_cyclic_zero_nopullback.json b/simulation/configs/1rollingppo_noisy_shared_cyclic_zero_nopullback.json
new file mode 100644
index 0000000..9614cc1
--- /dev/null
+++ b/simulation/configs/1rollingppo_noisy_shared_cyclic_zero_nopullback.json
@@ -0,0 +1,27 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 5e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            },
+            "num_instances": 1
+        }
+    },
+    "environment": {
+        "NoisyCyclicSharedSubgraph": {
+            "cycle": 200,
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/3deterministic1rl_agents_noisy_shared.json b/simulation/configs/3deterministic1rl_agents_noisy_shared_nopullback.json
similarity index 100%
rename from simulation/configs/3deterministic1rl_agents_noisy_shared.json
rename to simulation/configs/3deterministic1rl_agents_noisy_shared_nopullback.json
diff --git a/simulation/configs/3deterministic_agents_isa.json b/simulation/configs/3deterministic_agents_isa.json
new file mode 100644
index 0000000..6f2bb4d
--- /dev/null
+++ b/simulation/configs/3deterministic_agents_isa.json
@@ -0,0 +1,26 @@
+{
+    "agents": {
+        "deterministic1": {
+            "action": {
+                "type": "deterministic",
+                "initial_value": 3e-5
+            }
+        },
+        "deterministic2": {
+            "action": {
+                "type": "deterministic",
+                "initial_value": 4e-5
+            }
+        },
+        "deterministic3": {
+            "action": {
+                "type": "deterministic",
+                "initial_value": 4.99e-5
+            }
+        }
+    },
+    "environment": {
+        "isa": {
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/3different_agents_isa_nopullback.json b/simulation/configs/3different_agents_isa_nopullback.json
new file mode 100644
index 0000000..a3b9158
--- /dev/null
+++ b/simulation/configs/3different_agents_isa_nopullback.json
@@ -0,0 +1,52 @@
+{
+    "agents": {
+        "vpg1": {
+            "policy": {
+                "type": "vpg"
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            }
+        },
+        "ppo1": {
+            "policy": {
+                "type": "ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            }
+        },
+        "rolling_ppo1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1e-6,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            }
+        }
+    },
+    "environment": {
+        "isa": {
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/3different_rollingppo_isa_nopullback.json b/simulation/configs/3different_rollingppo_isa_nopullback.json
new file mode 100644
index 0000000..efc4525
--- /dev/null
+++ b/simulation/configs/3different_rollingppo_isa_nopullback.json
@@ -0,0 +1,41 @@
+{
+    "agents": {
+        "rolling_ppo_1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1.0e-06,
+                "initial_stddev": 2e-7
+            }
+        },
+        "rolling_ppo_2": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rolling_ppo_3": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 3.0e-06,
+                "initial_stddev": 3e-7
+            }
+        }
+    },
+    "environment": {
+        "isa": {
+        }
+    }
+}
diff --git a/simulation/configs/3heuristic1rl_agents_noisy_shared_nopullback.json b/simulation/configs/3heuristic1rl_agents_noisy_shared_nopullback.json
new file mode 100644
index 0000000..0697558
--- /dev/null
+++ b/simulation/configs/3heuristic1rl_agents_noisy_shared_nopullback.json
@@ -0,0 +1,46 @@
+{
+    "agents": {
+        "random_scaled1": {
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1.25e-06,
+                "initial_stddev": 1e-7
+            }
+        },
+        "random_scaled2": {
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.5e-06,
+                "initial_stddev": 1e-7
+            }
+        },
+        "random_scaled3": {
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 1e-7
+            }
+        },
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "buffer_max_size": 10,
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 2.0e-06,
+                "initial_stddev": 1e-7
+            },
+            "optimizer": {
+                "type": "adam",
+                "lr": 0.01
+            }
+        }
+    },
+    "environment": {
+        "NoisySharedSubgraph": {
+            "noise": true
+        }
+    }
+}
\ No newline at end of file
diff --git a/simulation/configs/3rollmem_agents_noisy_shared_nopullback.json b/simulation/configs/3rollmem_agents_noisy_shared_nopullback.json
new file mode 100644
index 0000000..d5a2990
--- /dev/null
+++ b/simulation/configs/3rollmem_agents_noisy_shared_nopullback.json
@@ -0,0 +1,42 @@
+{
+    "agents": {
+        "rollmem1": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rollmem2": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1.0e-06,
+                "initial_stddev": 4e-7
+            }
+        },
+        "rollmem3": {
+            "policy": {
+                "type": "rolling_ppo",
+                "graceful_init_pull": false
+            },
+            "action": {
+                "type": "scaled_gaussian",
+                "initial_mean": 1.0e-06,
+                "initial_stddev": 4e-7
+            }
+        }
+    },
+    "environment": {
+        "NoisySharedSubgraph": {
+            "noise": true
+        }
+    }
+}
\ No newline at end of file