From a78e735a20f15237b92e6ae02248dc78f57acb32 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 5 Mar 2022 12:37:28 -0800
Subject: [PATCH 01/49] [scheduler] 25/28 tests passing for NP2.

---
 core/threaded/scheduler_NP2.c      | 109 +++++++++++++++++++++
 core/threaded/worker_assignments.h | 151 +++++++++++++++++++++++++++++
 core/threaded/worker_states.h      | 116 ++++++++++++++++++++++
 3 files changed, 376 insertions(+)
 create mode 100644 core/threaded/scheduler_NP2.c
 create mode 100644 core/threaded/worker_assignments.h
 create mode 100644 core/threaded/worker_states.h

diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
new file mode 100644
index 000000000..a8fe5203e
--- /dev/null
+++ b/core/threaded/scheduler_NP2.c
@@ -0,0 +1,109 @@
+/**
+ * This is a non-priority-driven scheduler. See scheduler.h for documentation.
+ */
+
+
+#ifndef NUMBER_OF_WORKERS
+#define NUMBER_OF_WORKERS 1
+#endif // NUMBER_OF_WORKERS
+
+#include <assert.h>
+
+#include "scheduler.h"
+#include "../utils/pqueue_support.h"
+#include "scheduler_sync_tag_advance.c"
+#include "worker_assignments.h"
+#include "worker_states.h"
+
+#ifndef MAX_REACTION_LEVEL
+#define MAX_REACTION_LEVEL INITIAL_REACT_QUEUE_SIZE
+#endif
+
+static bool init_called = false;
+static bool should_stop = false;
+
+extern lf_mutex_t mutex;
+
+///////////////////////// Scheduler Private Functions ///////////////////////////
+
+/**
+ * @brief Increment the level currently being executed, and the tag if need be.
+ * 
+ * Sleep thereafter if that is what the current worker ought to do.
+ * @param worker The number of the calling worker.
+ */
+static void advance_level(size_t worker) {
+    lf_mutex_lock(&mutex);  // FIXME: do this elsewhere?
+    if (try_increment_level()) {
+        // There is no need to actually acquire a lock here. Due to the use of an atomic to
+        // determine which worker advances level, only one worker can be doing this.
+        if (_lf_sched_advance_tag_locked()) {
+            should_stop = true;
+            worker_states_never_sleep_again();
+            lf_mutex_unlock(&mutex);
+            return;
+        }
+    }
+    size_t num_workers_busy = get_num_workers_busy();
+    size_t level_snapshot = worker_states_awaken_locked(num_workers_busy);
+    lf_mutex_unlock(&mutex);
+    if (num_workers_busy < worker && num_workers_busy) {
+        printf("DEBUG: %ld sleeps; others should be busy.\n", worker);
+        worker_states_sleep(worker, level_snapshot);
+    }
+}
+
+///////////////////// Scheduler Init and Destroy API /////////////////////////
+
+void lf_sched_init(size_t number_of_workers, sched_params_t* params) {
+    // TODO: Instead of making this a no-op, crash the program. If this gets called twice, then that
+    // is a bug that should be fixed.
+    if (init_called) return;
+    worker_states_init(number_of_workers);
+    worker_assignments_init(number_of_workers, params);
+    init_called = true;
+}
+
+void lf_sched_free() {
+    worker_states_free();
+    worker_assignments_free();
+}
+
+///////////////////////// Scheduler Worker API ///////////////////////////////
+
+reaction_t* lf_sched_get_ready_reaction(int worker_number) {
+    assert(worker_number >= 0);
+    reaction_t* ret;
+    while (!(ret = worker_assignments_get(worker_number))) {
+        printf("DEBUG: %d failed to get.\n", worker_number);
+        size_t level_counter_snapshot = level_counter;
+        if (worker_assignments_finished_with_level(worker_number)) {
+            // TODO: Advance level all the way to the next level with at least one reaction?
+            advance_level(worker_number);
+            printf("%d !\n", worker_number);
+        } else {
+            //printf("DEBUG: %d sleeps; finished early.\n", worker_number);
+            worker_states_sleep(worker_number, level_counter_snapshot);
+        }
+        if (should_stop) {
+            //printf("DEBUG: %d should stop.\n", worker_number);
+            return NULL;
+        }
+        //printf("DEBUG: %d will try to get.\n", worker_number);
+    }
+    printf("%d <- %p @ %ld\n", worker_number, ret, LEVEL(ret->index));
+    return (reaction_t*) ret;
+}
+
+void lf_sched_done_with_reaction(size_t worker_number, reaction_t* done_reaction) {
+    assert(worker_number >= 0);
+    assert(done_reaction->status != inactive);
+    done_reaction->status = inactive;
+}
+
+void lf_sched_trigger_reaction(reaction_t* reaction, int worker_number) {
+    assert(worker_number >= -1);
+    if (reaction == NULL) return; // FIXME: When does this happen again? In federated execution?
+    if (!lf_bool_compare_and_swap(&reaction->status, inactive, queued)) return;
+    worker_assignments_put(reaction);
+}
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
new file mode 100644
index 000000000..8aca1689f
--- /dev/null
+++ b/core/threaded/worker_assignments.h
@@ -0,0 +1,151 @@
+
+#include "scheduler.h"
+
+static reaction_t**** reactions_by_worker_by_level;
+static size_t** num_reactions_by_worker_by_level;
+static size_t* max_num_workers_by_level;
+static size_t* num_workers_by_level;
+static size_t num_levels;
+static size_t max_num_workers;
+
+// The following apply to the current level.
+static size_t current_level;
+static size_t num_workers_busy;
+static size_t* num_reactions_by_worker;
+static reaction_t*** reactions_by_worker;
+static size_t num_workers;
+
+// A counter of the number of reactions triggered. No function should depend on the precise
+// correctness of this value. Race conditions when accessing this value are acceptable.
+static size_t reactions_triggered_counter = 0;
+
+/**
+ * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
+ * impossible.
+ * 
+ * @param level The new current level.
+ */
+static void set_level(size_t level) {
+    assert(level < num_levels);
+    assert(0 <= level);
+    current_level = level;
+    num_workers_busy = num_workers_by_level[level];
+    num_reactions_by_worker = num_reactions_by_worker_by_level[level];
+    reactions_by_worker = reactions_by_worker_by_level[level];
+    num_workers = num_workers_by_level[level];
+}
+
+void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
+    num_levels = params->num_reactions_per_level_size;
+    max_num_workers = number_of_workers;
+    reactions_by_worker_by_level = (reaction_t****) malloc(sizeof(reaction_t***) * num_levels);
+    num_reactions_by_worker_by_level = (size_t**) malloc(sizeof(size_t*) * num_levels);
+    num_workers_by_level = (size_t*) malloc(sizeof(size_t) * num_levels);
+    max_num_workers_by_level = (size_t*) malloc(sizeof(size_t) * num_levels);
+    for (size_t level = 0; level < num_levels; level++) {
+        size_t num_reactions = params->num_reactions_per_level[level];
+        size_t num_workers = num_reactions < max_num_workers ? num_reactions : max_num_workers;
+        max_num_workers_by_level[level] = num_workers;
+        num_workers_by_level[level] = max_num_workers_by_level[level];
+        reactions_by_worker_by_level[level] = (reaction_t***) malloc(
+            sizeof(reaction_t**) * max_num_workers
+        );
+        num_reactions_by_worker_by_level[level] = (size_t*) calloc(max_num_workers, sizeof(size_t));
+        for (size_t worker = 0; worker < max_num_workers; worker++) {
+            reactions_by_worker_by_level[level][worker] = (reaction_t**) malloc(
+                sizeof(reaction_t*) * (1 + ((worker < num_workers) ? num_reactions : 0))
+            );  // Warning: This wastes space.
+            reactions_by_worker_by_level[level][worker][0] = NULL;
+        }
+    }
+    set_level(0);
+}
+
+void worker_assignments_free() {
+    for (size_t level = 0; level < num_levels; level++) {
+        for (size_t worker = 0; worker < max_num_workers_by_level[level]; worker++) {
+            free(reactions_by_worker_by_level[level][worker]);
+        }
+        free(reactions_by_worker_by_level[level]);
+        free(num_reactions_by_worker_by_level[level]);
+    }
+    free(max_num_workers_by_level);
+    free(num_workers_by_level);
+}
+
+/**
+ * @brief Get a reaction for the given worker to execute. If no such reaction exists, consider the
+ * given worker to be NOT BUSY. Workers that are not busy must not continue to request reactions.
+ * 
+ * @param worker A worker requesting work.
+ * @return reaction_t* A reaction to execute, or NULL if no such reaction exists.
+ */
+reaction_t* worker_assignments_get(size_t worker) {
+    assert(worker >= 0);
+    // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
+    assert(num_reactions_by_worker[worker] >= 0);
+    // The following is correct because of the NULL that precedes each reactions array. Note that it
+    // temporarily leaves num_reactions_by_worker[worker] in an inconsistent state.
+    // printf("DEBUG: %ld %ld\n", worker, num_reactions_by_worker[worker]);
+    return reactions_by_worker[worker][num_reactions_by_worker[worker]--];
+}
+
+/**
+ * @brief Record that worker is finished working on the current level.
+ * 
+ * @param worker The number of a worker.
+ * @return true If this is the last worker to finish working on the current level.
+ * @return false If at least one other worker is still working on the current level.
+ */
+bool worker_assignments_finished_with_level(size_t worker) {
+    assert(worker >= 0);
+    // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
+    assert(num_workers_busy > 0 || worker >= num_workers);
+    num_reactions_by_worker[worker] = 0;
+    bool ret = !lf_atomic_add_fetch(&num_workers_busy, -(worker < num_workers));
+    return ret;
+}
+
+/**
+ * @brief Trigger the given reaction.
+ * 
+ * @param reaction A reaction to be executed in the current tag.
+ */
+void worker_assignments_put(reaction_t* reaction) {
+    size_t level = LEVEL(reaction->index);
+    assert(level > current_level || current_level == 0);
+    assert(level < num_levels);
+    size_t worker = (reactions_triggered_counter++) % num_workers_by_level[level];
+    assert(worker >= 0 && worker <= num_workers);
+    size_t num_preceding_reactions_plus_one = lf_atomic_add_fetch(
+        &num_reactions_by_worker_by_level[level][worker],
+        1
+    );
+    printf("%p -> %ld @ %ld\n", reaction, worker, level);
+    reactions_by_worker_by_level[level][worker][num_preceding_reactions_plus_one] = reaction;
+}
+
+/**
+ * @brief Get the number of workers that should currently be working.
+ * 
+ * @return size_t The number of workers that should currently be working.
+ */
+size_t get_num_workers_busy() {
+    return num_workers_busy;
+}
+
+/**
+ * @brief Increment the level currently being processed by the workers.
+ * 
+ * @return true If the level was already at the maximum and was reset to zero.
+ * @return false Otherwise.
+ */
+bool try_increment_level() {
+    assert(num_workers_busy == 0);
+    if (current_level + 1 == num_levels) {
+        set_level(0);
+        return true;
+    }
+    set_level(current_level + 1);
+    return false;
+}
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
new file mode 100644
index 000000000..29b1d2747
--- /dev/null
+++ b/core/threaded/worker_states.h
@@ -0,0 +1,116 @@
+
+#ifndef NUMBER_OF_WORKERS
+#define NUMBER_OF_WORKERS 1
+#endif // NUMBER_OF_WORKERS
+
+#include "scheduler.h"
+#include "../platform.h"
+
+static size_t num_awakened = 0;
+static lf_cond_t* worker_conds;
+
+static bool worker_states_sleep_forbidden = false;
+
+extern size_t current_level;
+extern size_t** num_reactions_by_worker_by_level;
+extern size_t max_num_workers;
+extern lf_mutex_t mutex;
+
+/**
+ * The level counter is a number that changes whenever the current level changes.
+ *
+ * This number must have a very long period in the sense that if it changes, the probability that it
+ * is checked at a time in the future that is selected from some "reasonable" distribution, the
+ * probability that it will have returned to the same value is vanishingly small.
+ */
+static size_t level_counter = 0;
+
+/**
+ * @brief Return the index of the condition variable used by worker.
+ * 
+ * This function is nondecreasing, and the least element of its image is zero.
+ * 
+ * @param worker A worker number.
+ * @return size_t The index of the condition variable used by worker.
+ */
+static size_t cond_of(size_t worker) {
+    // Note: __builtin_clz with GCC might be preferred, or fls (?).
+    int ret = 0;
+    while (worker) {
+        ret++;
+        worker >>= 1;
+    }
+    return ret;
+}
+
+void worker_states_init(size_t number_of_workers) {
+    size_t greatest_worker_number = number_of_workers - 1;
+    size_t num_conds = cond_of(greatest_worker_number) + 1;
+    worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
+    for (int i = 0; i < num_conds; i++) {
+        lf_cond_init(worker_conds + i);
+    }
+}
+
+void worker_states_free() {
+    // FIXME: Why do the condition variables and mutexes not need to be freed?
+    free(worker_conds);
+}
+
+/**
+ * @brief Awaken the workers scheduled to work on the current level.
+ * 
+ * @param num_to_awaken The number of workers to awaken.
+ * @return A snapshot of the level counter after awakening the workers.
+ */
+size_t worker_states_awaken_locked(size_t num_to_awaken) {
+    assert(num_to_awaken <= max_num_workers);
+    num_awakened = num_to_awaken;
+    size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
+    size_t max_cond = cond_of(greatest_worker_number_to_awaken);
+    for (int cond = 0; cond <= max_cond; cond++) {
+        printf("DEBUG: broadcasting to cond %d.\n", cond);
+        lf_cond_broadcast(worker_conds + cond);
+    }
+    size_t ret = ++level_counter;
+    return ret;
+}
+
+/**
+ * @brief Wake up all workers and forbid them from ever sleeping again.
+ * 
+ * This is intended to coordinate shutdown (without leaving any dangling threads behind).
+ */
+void worker_states_never_sleep_again() {
+    worker_states_sleep_forbidden = true;
+    lf_mutex_lock(&mutex);
+    worker_states_awaken_locked(max_num_workers);
+    lf_mutex_unlock(&mutex);
+}
+
+/**
+ * @brief Make the given worker go to sleep.
+ *
+ * This should be called by the given worker when the worker will do nothing for the remainder of
+ * the execution of the current level.
+ *
+ * @param worker The number of the calling worker.
+ * @param level_counter_snapshot The value of the level counter at the time of the decision to
+ * sleep.
+ */
+void worker_states_sleep(size_t worker, size_t level_counter_snapshot) {
+    // printf("DEBUG: worker=%ld\n", worker);
+    assert(worker < max_num_workers);
+    size_t cond = cond_of(worker);
+    printf("DEBUG: %ld tries to get mutex so it can sleep.\n", worker);
+    lf_mutex_lock(&mutex);
+    printf("DEBUG: %ld has mutex and will try to sleep.\n", worker);
+    if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
+        do {
+            printf("DEBUG: %ld is going to sleep with cond %ld.\n", worker, cond);
+            lf_cond_wait(worker_conds + cond, &mutex);
+        } while (worker >= num_awakened);
+        printf("DEBUG: %ld is awakened.\n", worker);
+    }
+    lf_mutex_unlock(&mutex);
+}

From 505c5eb3fb92c40196cb9ec51e88af8bf2fbe3cb Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 5 Mar 2022 16:24:29 -0800
Subject: [PATCH 02/49] [scheduler] Tests passing for NP2.

---
 core/threaded/scheduler_NP2.c      | 27 ++++++++-----------
 core/threaded/worker_assignments.h | 42 ++++++++++++++++++------------
 core/threaded/worker_states.h      |  9 +------
 3 files changed, 38 insertions(+), 40 deletions(-)

diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index a8fe5203e..164fe995c 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -32,8 +32,7 @@ extern lf_mutex_t mutex;
  * Sleep thereafter if that is what the current worker ought to do.
  * @param worker The number of the calling worker.
  */
-static void advance_level(size_t worker) {
-    lf_mutex_lock(&mutex);  // FIXME: do this elsewhere?
+static void advance_level_and_unlock(size_t worker) {
     if (try_increment_level()) {
         // There is no need to actually acquire a lock here. Due to the use of an atomic to
         // determine which worker advances level, only one worker can be doing this.
@@ -46,10 +45,10 @@ static void advance_level(size_t worker) {
     }
     size_t num_workers_busy = get_num_workers_busy();
     size_t level_snapshot = worker_states_awaken_locked(num_workers_busy);
-    lf_mutex_unlock(&mutex);
-    if (num_workers_busy < worker && num_workers_busy) {
-        printf("DEBUG: %ld sleeps; others should be busy.\n", worker);
-        worker_states_sleep(worker, level_snapshot);
+    if (num_workers_busy < worker && num_workers_busy) {  // FIXME: Is this branch still necessary?
+        worker_states_sleep_and_unlock(worker, level_snapshot);
+    } else {
+        lf_mutex_unlock(&mutex);
     }
 }
 
@@ -74,24 +73,20 @@ void lf_sched_free() {
 reaction_t* lf_sched_get_ready_reaction(int worker_number) {
     assert(worker_number >= 0);
     reaction_t* ret;
-    while (!(ret = worker_assignments_get(worker_number))) {
-        printf("DEBUG: %d failed to get.\n", worker_number);
+    while (!(ret = worker_assignments_get_or_lock(worker_number))) {
+        // printf("%d failed to get.\n", worker_number);
         size_t level_counter_snapshot = level_counter;
-        if (worker_assignments_finished_with_level(worker_number)) {
+        if (worker_assignments_finished_with_level_locked(worker_number)) {
             // TODO: Advance level all the way to the next level with at least one reaction?
-            advance_level(worker_number);
-            printf("%d !\n", worker_number);
+            advance_level_and_unlock(worker_number);
+            // printf("%d !\n", worker_number);
         } else {
-            //printf("DEBUG: %d sleeps; finished early.\n", worker_number);
-            worker_states_sleep(worker_number, level_counter_snapshot);
+            worker_states_sleep_and_unlock(worker_number, level_counter_snapshot);
         }
         if (should_stop) {
-            //printf("DEBUG: %d should stop.\n", worker_number);
             return NULL;
         }
-        //printf("DEBUG: %d will try to get.\n", worker_number);
     }
-    printf("%d <- %p @ %ld\n", worker_number, ret, LEVEL(ret->index));
     return (reaction_t*) ret;
 }
 
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 8aca1689f..c4ca0771b 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -19,6 +19,8 @@ static size_t num_workers;
 // correctness of this value. Race conditions when accessing this value are acceptable.
 static size_t reactions_triggered_counter = 0;
 
+extern lf_mutex_t mutex;
+
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
  * impossible.
@@ -53,9 +55,8 @@ void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
         num_reactions_by_worker_by_level[level] = (size_t*) calloc(max_num_workers, sizeof(size_t));
         for (size_t worker = 0; worker < max_num_workers; worker++) {
             reactions_by_worker_by_level[level][worker] = (reaction_t**) malloc(
-                sizeof(reaction_t*) * (1 + ((worker < num_workers) ? num_reactions : 0))
+                sizeof(reaction_t*) * ((worker < num_workers) ? num_reactions : 0)
             );  // Warning: This wastes space.
-            reactions_by_worker_by_level[level][worker][0] = NULL;
         }
     }
     set_level(0);
@@ -74,20 +75,27 @@ void worker_assignments_free() {
 }
 
 /**
- * @brief Get a reaction for the given worker to execute. If no such reaction exists, consider the
- * given worker to be NOT BUSY. Workers that are not busy must not continue to request reactions.
+ * @brief Get a reaction for the given worker to execute. If no such reaction exists, claim the
+ * mutex.
  * 
  * @param worker A worker requesting work.
  * @return reaction_t* A reaction to execute, or NULL if no such reaction exists.
  */
-reaction_t* worker_assignments_get(size_t worker) {
+reaction_t* worker_assignments_get_or_lock(size_t worker) {
     assert(worker >= 0);
     // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
     assert(num_reactions_by_worker[worker] >= 0);
-    // The following is correct because of the NULL that precedes each reactions array. Note that it
-    // temporarily leaves num_reactions_by_worker[worker] in an inconsistent state.
-    // printf("DEBUG: %ld %ld\n", worker, num_reactions_by_worker[worker]);
-    return reactions_by_worker[worker][num_reactions_by_worker[worker]--];
+    if (num_reactions_by_worker[worker]) {
+        reaction_t* ret = reactions_by_worker[worker][--num_reactions_by_worker[worker]];
+        // printf("%ld <- %p @ %lld\n", worker, ret, LEVEL(ret->index));
+        return ret;
+    }
+    lf_mutex_lock(&mutex);
+    if (!num_reactions_by_worker[worker]) {
+        return NULL;
+    }
+    lf_mutex_unlock(&mutex);
+    return reactions_by_worker[worker][--num_reactions_by_worker[worker]];
 }
 
 /**
@@ -97,13 +105,14 @@ reaction_t* worker_assignments_get(size_t worker) {
  * @return true If this is the last worker to finish working on the current level.
  * @return false If at least one other worker is still working on the current level.
  */
-bool worker_assignments_finished_with_level(size_t worker) {
+bool worker_assignments_finished_with_level_locked(size_t worker) {
     assert(worker >= 0);
     // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
     assert(num_workers_busy > 0 || worker >= num_workers);
-    num_reactions_by_worker[worker] = 0;
-    bool ret = !lf_atomic_add_fetch(&num_workers_busy, -(worker < num_workers));
-    return ret;
+    assert(num_reactions_by_worker[worker] != 1);
+    assert(num_reactions_by_worker[worker] == (((size_t) 0) - 1) || num_reactions_by_worker[worker] == 0);
+    num_workers_busy -= worker < num_workers;
+    return !num_workers_busy;
 }
 
 /**
@@ -113,16 +122,17 @@ bool worker_assignments_finished_with_level(size_t worker) {
  */
 void worker_assignments_put(reaction_t* reaction) {
     size_t level = LEVEL(reaction->index);
+    assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
     size_t worker = (reactions_triggered_counter++) % num_workers_by_level[level];
     assert(worker >= 0 && worker <= num_workers);
-    size_t num_preceding_reactions_plus_one = lf_atomic_add_fetch(
+    size_t num_preceding_reactions = lf_atomic_fetch_add(
         &num_reactions_by_worker_by_level[level][worker],
         1
     );
-    printf("%p -> %ld @ %ld\n", reaction, worker, level);
-    reactions_by_worker_by_level[level][worker][num_preceding_reactions_plus_one] = reaction;
+    // printf("%p -> %ld @ %ld[%ld]\n", reaction, worker, level, num_preceding_reactions);
+    reactions_by_worker_by_level[level][worker][num_preceding_reactions] = reaction;
 }
 
 /**
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 29b1d2747..25b04e97c 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -69,7 +69,6 @@ size_t worker_states_awaken_locked(size_t num_to_awaken) {
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
     size_t max_cond = cond_of(greatest_worker_number_to_awaken);
     for (int cond = 0; cond <= max_cond; cond++) {
-        printf("DEBUG: broadcasting to cond %d.\n", cond);
         lf_cond_broadcast(worker_conds + cond);
     }
     size_t ret = ++level_counter;
@@ -98,19 +97,13 @@ void worker_states_never_sleep_again() {
  * @param level_counter_snapshot The value of the level counter at the time of the decision to
  * sleep.
  */
-void worker_states_sleep(size_t worker, size_t level_counter_snapshot) {
-    // printf("DEBUG: worker=%ld\n", worker);
+void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_snapshot) {
     assert(worker < max_num_workers);
     size_t cond = cond_of(worker);
-    printf("DEBUG: %ld tries to get mutex so it can sleep.\n", worker);
-    lf_mutex_lock(&mutex);
-    printf("DEBUG: %ld has mutex and will try to sleep.\n", worker);
     if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
         do {
-            printf("DEBUG: %ld is going to sleep with cond %ld.\n", worker, cond);
             lf_cond_wait(worker_conds + cond, &mutex);
         } while (worker >= num_awakened);
-        printf("DEBUG: %ld is awakened.\n", worker);
     }
     lf_mutex_unlock(&mutex);
 }

From 086e7d0eb165bb48e0257602dfeb8898a56319a2 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 5 Mar 2022 23:05:29 -0800
Subject: [PATCH 03/49] [scheduler] First possible performance improvement.

A timer event seems to be occasionally dropped in TimeLimitThreaded.
---
 core/threaded/data_collection.h    | 49 ++++++++++++++++++++++++++++++
 core/threaded/scheduler_NP2.c      |  2 --
 core/threaded/worker_assignments.h |  8 ++++-
 3 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 core/threaded/data_collection.h

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
new file mode 100644
index 000000000..155a0371c
--- /dev/null
+++ b/core/threaded/data_collection.h
@@ -0,0 +1,49 @@
+
+#ifndef NUMBER_OF_WORKERS
+#define NUMBER_OF_WORKERS 1
+#endif // NUMBER_OF_WORKERS
+
+#include "scheduler.h"
+
+static interval_t* start_times_by_level;
+static interval_t* execution_times_by_level;
+extern size_t num_levels;
+extern size_t max_num_workers;
+
+#define OPTIMAL_NANOSECONDS_WORK 65536
+
+void data_collection_init(sched_params_t* params) {
+    start_times_by_level = (interval_t*) calloc(
+        params->num_reactions_per_level_size, sizeof(interval_t)
+    );
+    execution_times_by_level = (interval_t*) calloc(
+        params->num_reactions_per_level_size, sizeof(interval_t)
+    );
+}
+
+void data_collection_free() {
+    free(start_times_by_level);
+    free(execution_times_by_level);
+}
+
+void data_collection_start_level(size_t level) {
+    start_times_by_level[level] = get_physical_time();
+}
+
+void data_collection_end_level(size_t level) {
+    if (start_times_by_level[level]) {
+        execution_times_by_level[level] = (
+            3 * execution_times_by_level[level]
+            + get_physical_time() - start_times_by_level[level]
+        ) >> 2;
+    }
+}
+
+void data_collection_compute_number_of_workers(size_t* num_workers_by_level) {
+    for (size_t level = 0; level < num_levels; level++) {
+        size_t ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
+        num_workers_by_level[level] = (ideal_number_of_workers < 1) ? 1 : (
+            (ideal_number_of_workers > max_num_workers) ? max_num_workers : ideal_number_of_workers
+        );
+    }
+}
diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index 164fe995c..c4214bf13 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -34,8 +34,6 @@ extern lf_mutex_t mutex;
  */
 static void advance_level_and_unlock(size_t worker) {
     if (try_increment_level()) {
-        // There is no need to actually acquire a lock here. Due to the use of an atomic to
-        // determine which worker advances level, only one worker can be doing this.
         if (_lf_sched_advance_tag_locked()) {
             should_stop = true;
             worker_states_never_sleep_again();
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index c4ca0771b..af3f6196b 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -21,6 +21,8 @@ static size_t reactions_triggered_counter = 0;
 
 extern lf_mutex_t mutex;
 
+#include "data_collection.h"
+
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
  * impossible.
@@ -30,11 +32,13 @@ extern lf_mutex_t mutex;
 static void set_level(size_t level) {
     assert(level < num_levels);
     assert(0 <= level);
+    data_collection_end_level(current_level);
     current_level = level;
     num_workers_busy = num_workers_by_level[level];
     num_reactions_by_worker = num_reactions_by_worker_by_level[level];
     reactions_by_worker = reactions_by_worker_by_level[level];
     num_workers = num_workers_by_level[level];
+    data_collection_start_level(current_level);
 }
 
 void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
@@ -59,6 +63,7 @@ void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
             );  // Warning: This wastes space.
         }
     }
+    data_collection_init(params);
     set_level(0);
 }
 
@@ -72,6 +77,7 @@ void worker_assignments_free() {
     }
     free(max_num_workers_by_level);
     free(num_workers_by_level);
+    data_collection_free();
 }
 
 /**
@@ -148,11 +154,11 @@ size_t get_num_workers_busy() {
  * @brief Increment the level currently being processed by the workers.
  * 
  * @return true If the level was already at the maximum and was reset to zero.
- * @return false Otherwise.
  */
 bool try_increment_level() {
     assert(num_workers_busy == 0);
     if (current_level + 1 == num_levels) {
+        data_collection_compute_number_of_workers(num_workers_by_level);
         set_level(0);
         return true;
     }

From 9a40c6e6807533b7b308e83df269e37adcfe8b31 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 00:02:48 -0800
Subject: [PATCH 04/49] [scheduler] Include assert.h.

---
 core/threaded/data_collection.h    | 1 +
 core/threaded/worker_assignments.h | 1 +
 core/threaded/worker_states.h      | 1 +
 3 files changed, 3 insertions(+)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 155a0371c..8495c0b90 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -3,6 +3,7 @@
 #define NUMBER_OF_WORKERS 1
 #endif // NUMBER_OF_WORKERS
 
+#include <assert.h>
 #include "scheduler.h"
 
 static interval_t* start_times_by_level;
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index af3f6196b..2ac7af449 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -1,4 +1,5 @@
 
+#include <assert.h>
 #include "scheduler.h"
 
 static reaction_t**** reactions_by_worker_by_level;
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 25b04e97c..ef47db51a 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -3,6 +3,7 @@
 #define NUMBER_OF_WORKERS 1
 #endif // NUMBER_OF_WORKERS
 
+#include <assert.h>
 #include "scheduler.h"
 #include "../platform.h"
 

From 5885ec8eb3a63f780621d334f22cf45ffa6f0b08 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 12:27:54 -0800
Subject: [PATCH 05/49] [scheduler] SortedLinkList: Make C++ compiler happy.

---
 .../{data_collection.h => data_collection.c}  | 15 ++++++++-----
 core/threaded/scheduler_NP2.c                 |  4 ++--
 ...ker_assignments.h => worker_assignments.c} | 21 ++++++++++++-------
 .../{worker_states.h => worker_states.c}      | 15 ++++++++-----
 4 files changed, 35 insertions(+), 20 deletions(-)
 rename core/threaded/{data_collection.h => data_collection.c} (78%)
 rename core/threaded/{worker_assignments.h => worker_assignments.c} (92%)
 rename core/threaded/{worker_states.h => worker_states.c} (90%)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.c
similarity index 78%
rename from core/threaded/data_collection.h
rename to core/threaded/data_collection.c
index 8495c0b90..051b3ffe7 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.c
@@ -1,4 +1,7 @@
 
+#ifndef DATA_COLLECTION
+#define DATA_COLLECTION
+
 #ifndef NUMBER_OF_WORKERS
 #define NUMBER_OF_WORKERS 1
 #endif // NUMBER_OF_WORKERS
@@ -13,7 +16,7 @@ extern size_t max_num_workers;
 
 #define OPTIMAL_NANOSECONDS_WORK 65536
 
-void data_collection_init(sched_params_t* params) {
+static void data_collection_init(sched_params_t* params) {
     start_times_by_level = (interval_t*) calloc(
         params->num_reactions_per_level_size, sizeof(interval_t)
     );
@@ -22,16 +25,16 @@ void data_collection_init(sched_params_t* params) {
     );
 }
 
-void data_collection_free() {
+static void data_collection_free() {
     free(start_times_by_level);
     free(execution_times_by_level);
 }
 
-void data_collection_start_level(size_t level) {
+static void data_collection_start_level(size_t level) {
     start_times_by_level[level] = get_physical_time();
 }
 
-void data_collection_end_level(size_t level) {
+static void data_collection_end_level(size_t level) {
     if (start_times_by_level[level]) {
         execution_times_by_level[level] = (
             3 * execution_times_by_level[level]
@@ -40,7 +43,7 @@ void data_collection_end_level(size_t level) {
     }
 }
 
-void data_collection_compute_number_of_workers(size_t* num_workers_by_level) {
+static void data_collection_compute_number_of_workers(size_t* num_workers_by_level) {
     for (size_t level = 0; level < num_levels; level++) {
         size_t ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
         num_workers_by_level[level] = (ideal_number_of_workers < 1) ? 1 : (
@@ -48,3 +51,5 @@ void data_collection_compute_number_of_workers(size_t* num_workers_by_level) {
         );
     }
 }
+
+#endif
diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index c4214bf13..6a248302b 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -12,8 +12,8 @@
 #include "scheduler.h"
 #include "../utils/pqueue_support.h"
 #include "scheduler_sync_tag_advance.c"
-#include "worker_assignments.h"
-#include "worker_states.h"
+#include "worker_assignments.c"
+#include "worker_states.c"
 
 #ifndef MAX_REACTION_LEVEL
 #define MAX_REACTION_LEVEL INITIAL_REACT_QUEUE_SIZE
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.c
similarity index 92%
rename from core/threaded/worker_assignments.h
rename to core/threaded/worker_assignments.c
index 2ac7af449..d45db3c5b 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.c
@@ -1,4 +1,7 @@
 
+#ifndef WORKER_ASSIGNMENTS
+#define WORKER_ASSIGNMENTS
+
 #include <assert.h>
 #include "scheduler.h"
 
@@ -22,7 +25,7 @@ static size_t reactions_triggered_counter = 0;
 
 extern lf_mutex_t mutex;
 
-#include "data_collection.h"
+#include "data_collection.c"
 
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
@@ -42,7 +45,7 @@ static void set_level(size_t level) {
     data_collection_start_level(current_level);
 }
 
-void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
+static void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
     num_levels = params->num_reactions_per_level_size;
     max_num_workers = number_of_workers;
     reactions_by_worker_by_level = (reaction_t****) malloc(sizeof(reaction_t***) * num_levels);
@@ -68,7 +71,7 @@ void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
     set_level(0);
 }
 
-void worker_assignments_free() {
+static void worker_assignments_free() {
     for (size_t level = 0; level < num_levels; level++) {
         for (size_t worker = 0; worker < max_num_workers_by_level[level]; worker++) {
             free(reactions_by_worker_by_level[level][worker]);
@@ -88,7 +91,7 @@ void worker_assignments_free() {
  * @param worker A worker requesting work.
  * @return reaction_t* A reaction to execute, or NULL if no such reaction exists.
  */
-reaction_t* worker_assignments_get_or_lock(size_t worker) {
+static reaction_t* worker_assignments_get_or_lock(size_t worker) {
     assert(worker >= 0);
     // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
     assert(num_reactions_by_worker[worker] >= 0);
@@ -112,7 +115,7 @@ reaction_t* worker_assignments_get_or_lock(size_t worker) {
  * @return true If this is the last worker to finish working on the current level.
  * @return false If at least one other worker is still working on the current level.
  */
-bool worker_assignments_finished_with_level_locked(size_t worker) {
+static bool worker_assignments_finished_with_level_locked(size_t worker) {
     assert(worker >= 0);
     // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
     assert(num_workers_busy > 0 || worker >= num_workers);
@@ -127,7 +130,7 @@ bool worker_assignments_finished_with_level_locked(size_t worker) {
  * 
  * @param reaction A reaction to be executed in the current tag.
  */
-void worker_assignments_put(reaction_t* reaction) {
+static void worker_assignments_put(reaction_t* reaction) {
     size_t level = LEVEL(reaction->index);
     assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
@@ -147,7 +150,7 @@ void worker_assignments_put(reaction_t* reaction) {
  * 
  * @return size_t The number of workers that should currently be working.
  */
-size_t get_num_workers_busy() {
+static size_t get_num_workers_busy() {
     return num_workers_busy;
 }
 
@@ -156,7 +159,7 @@ size_t get_num_workers_busy() {
  * 
  * @return true If the level was already at the maximum and was reset to zero.
  */
-bool try_increment_level() {
+static bool try_increment_level() {
     assert(num_workers_busy == 0);
     if (current_level + 1 == num_levels) {
         data_collection_compute_number_of_workers(num_workers_by_level);
@@ -166,3 +169,5 @@ bool try_increment_level() {
     set_level(current_level + 1);
     return false;
 }
+
+#endif
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.c
similarity index 90%
rename from core/threaded/worker_states.h
rename to core/threaded/worker_states.c
index ef47db51a..41f7b5c8a 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.c
@@ -1,4 +1,7 @@
 
+#ifndef WORKER_STATES
+#define WORKER_STATES
+
 #ifndef NUMBER_OF_WORKERS
 #define NUMBER_OF_WORKERS 1
 #endif // NUMBER_OF_WORKERS
@@ -44,7 +47,7 @@ static size_t cond_of(size_t worker) {
     return ret;
 }
 
-void worker_states_init(size_t number_of_workers) {
+static void worker_states_init(size_t number_of_workers) {
     size_t greatest_worker_number = number_of_workers - 1;
     size_t num_conds = cond_of(greatest_worker_number) + 1;
     worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
@@ -53,7 +56,7 @@ void worker_states_init(size_t number_of_workers) {
     }
 }
 
-void worker_states_free() {
+static void worker_states_free() {
     // FIXME: Why do the condition variables and mutexes not need to be freed?
     free(worker_conds);
 }
@@ -64,7 +67,7 @@ void worker_states_free() {
  * @param num_to_awaken The number of workers to awaken.
  * @return A snapshot of the level counter after awakening the workers.
  */
-size_t worker_states_awaken_locked(size_t num_to_awaken) {
+static size_t worker_states_awaken_locked(size_t num_to_awaken) {
     assert(num_to_awaken <= max_num_workers);
     num_awakened = num_to_awaken;
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
@@ -81,7 +84,7 @@ size_t worker_states_awaken_locked(size_t num_to_awaken) {
  * 
  * This is intended to coordinate shutdown (without leaving any dangling threads behind).
  */
-void worker_states_never_sleep_again() {
+static void worker_states_never_sleep_again() {
     worker_states_sleep_forbidden = true;
     lf_mutex_lock(&mutex);
     worker_states_awaken_locked(max_num_workers);
@@ -98,7 +101,7 @@ void worker_states_never_sleep_again() {
  * @param level_counter_snapshot The value of the level counter at the time of the decision to
  * sleep.
  */
-void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_snapshot) {
+static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_snapshot) {
     assert(worker < max_num_workers);
     size_t cond = cond_of(worker);
     if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
@@ -108,3 +111,5 @@ void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_snapshot
     }
     lf_mutex_unlock(&mutex);
 }
+
+#endif

From 746af9a7448ae57cbeb0e221e92ca41d36274ce6 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 15:46:43 -0800
Subject: [PATCH 06/49] [scheduler] Make compiler happy.

---
 core/threaded/worker_assignments.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/threaded/worker_assignments.c b/core/threaded/worker_assignments.c
index d45db3c5b..7d268f8bf 100644
--- a/core/threaded/worker_assignments.c
+++ b/core/threaded/worker_assignments.c
@@ -9,8 +9,8 @@ static reaction_t**** reactions_by_worker_by_level;
 static size_t** num_reactions_by_worker_by_level;
 static size_t* max_num_workers_by_level;
 static size_t* num_workers_by_level;
-static size_t num_levels;
-static size_t max_num_workers;
+size_t num_levels;
+size_t max_num_workers;
 
 // The following apply to the current level.
 static size_t current_level;

From a65531d6fb1a17eb37cde705ce8e0a4dc7103485 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 16:23:25 -0800
Subject: [PATCH 07/49] [scheduler] Reduce expensive calls to
 get_physical_time.

TODO: compute_number_of_workers should be reduced correspondingly?
---
 core/threaded/worker_assignments.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/core/threaded/worker_assignments.c b/core/threaded/worker_assignments.c
index 7d268f8bf..745594575 100644
--- a/core/threaded/worker_assignments.c
+++ b/core/threaded/worker_assignments.c
@@ -34,15 +34,24 @@ extern lf_mutex_t mutex;
  * @param level The new current level.
  */
 static void set_level(size_t level) {
+    static size_t data_collection_counter = 0;
+    static bool collecting_data = true;
     assert(level < num_levels);
     assert(0 <= level);
-    data_collection_end_level(current_level);
+    if (collecting_data) data_collection_end_level(current_level);
+    if (level == 0) {
+        data_collection_counter++;
+        int shift = 2 << (data_collection_counter > 8);
+        collecting_data = data_collection_counter == (
+            (data_collection_counter >> shift) << shift
+        );
+    }
     current_level = level;
     num_workers_busy = num_workers_by_level[level];
     num_reactions_by_worker = num_reactions_by_worker_by_level[level];
     reactions_by_worker = reactions_by_worker_by_level[level];
     num_workers = num_workers_by_level[level];
-    data_collection_start_level(current_level);
+    if (collecting_data) data_collection_start_level(current_level);
 }
 
 static void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
@@ -135,6 +144,7 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
+    // TODO: Hashing by a pointer to the reaction will let us cheaply simulate ``worker affinity''.
     size_t worker = (reactions_triggered_counter++) % num_workers_by_level[level];
     assert(worker >= 0 && worker <= num_workers);
     size_t num_preceding_reactions = lf_atomic_fetch_add(

From 022b8747af400fef8d69ede5a4f13836f4c5e8c0 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 16:43:33 -0800
Subject: [PATCH 08/49] [scheduler] Do not use more workers than there are
 reactions.

---
 core/threaded/data_collection.c    |  9 +++++++--
 core/threaded/worker_assignments.c | 12 ++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/core/threaded/data_collection.c b/core/threaded/data_collection.c
index 051b3ffe7..e0cc6b910 100644
--- a/core/threaded/data_collection.c
+++ b/core/threaded/data_collection.c
@@ -43,11 +43,16 @@ static void data_collection_end_level(size_t level) {
     }
 }
 
-static void data_collection_compute_number_of_workers(size_t* num_workers_by_level) {
+static void data_collection_compute_number_of_workers(
+    size_t* num_workers_by_level,
+    size_t* max_num_workers_by_level
+) {
     for (size_t level = 0; level < num_levels; level++) {
         size_t ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
+        size_t max_reasonable_num_workers = max_num_workers_by_level[level];
         num_workers_by_level[level] = (ideal_number_of_workers < 1) ? 1 : (
-            (ideal_number_of_workers > max_num_workers) ? max_num_workers : ideal_number_of_workers
+            (ideal_number_of_workers > max_reasonable_num_workers) ? max_reasonable_num_workers :
+            ideal_number_of_workers
         );
     }
 }
diff --git a/core/threaded/worker_assignments.c b/core/threaded/worker_assignments.c
index 745594575..c1b67a06b 100644
--- a/core/threaded/worker_assignments.c
+++ b/core/threaded/worker_assignments.c
@@ -9,8 +9,8 @@ static reaction_t**** reactions_by_worker_by_level;
 static size_t** num_reactions_by_worker_by_level;
 static size_t* max_num_workers_by_level;
 static size_t* num_workers_by_level;
-size_t num_levels;
-size_t max_num_workers;
+static size_t num_levels;
+static size_t max_num_workers;
 
 // The following apply to the current level.
 static size_t current_level;
@@ -41,7 +41,7 @@ static void set_level(size_t level) {
     if (collecting_data) data_collection_end_level(current_level);
     if (level == 0) {
         data_collection_counter++;
-        int shift = 2 << (data_collection_counter > 8);
+        int shift = 3 << (data_collection_counter > 8);
         collecting_data = data_collection_counter == (
             (data_collection_counter >> shift) << shift
         );
@@ -70,9 +70,9 @@ static void worker_assignments_init(size_t number_of_workers, sched_params_t* pa
             sizeof(reaction_t**) * max_num_workers
         );
         num_reactions_by_worker_by_level[level] = (size_t*) calloc(max_num_workers, sizeof(size_t));
-        for (size_t worker = 0; worker < max_num_workers; worker++) {
+        for (size_t worker = 0; worker < max_num_workers_by_level[level]; worker++) {
             reactions_by_worker_by_level[level][worker] = (reaction_t**) malloc(
-                sizeof(reaction_t*) * ((worker < num_workers) ? num_reactions : 0)
+                sizeof(reaction_t*) * num_reactions
             );  // Warning: This wastes space.
         }
     }
@@ -172,7 +172,7 @@ static size_t get_num_workers_busy() {
 static bool try_increment_level() {
     assert(num_workers_busy == 0);
     if (current_level + 1 == num_levels) {
-        data_collection_compute_number_of_workers(num_workers_by_level);
+        data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
         set_level(0);
         return true;
     }

From ece68751986a4bf90cb9cded57c1fb20663a5183 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 16:57:13 -0800
Subject: [PATCH 09/49] [scheduler] Try again to keep C++ and C compilers both
 happy.

---
 core/threaded/{data_collection.c => data_collection.h}      | 0
 core/threaded/scheduler_NP2.c                               | 4 ++--
 .../threaded/{worker_assignments.c => worker_assignments.h} | 6 +++++-
 core/threaded/{worker_states.c => worker_states.h}          | 0
 4 files changed, 7 insertions(+), 3 deletions(-)
 rename core/threaded/{data_collection.c => data_collection.h} (100%)
 rename core/threaded/{worker_assignments.c => worker_assignments.h} (98%)
 rename core/threaded/{worker_states.c => worker_states.h} (100%)

diff --git a/core/threaded/data_collection.c b/core/threaded/data_collection.h
similarity index 100%
rename from core/threaded/data_collection.c
rename to core/threaded/data_collection.h
diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index 6a248302b..c4214bf13 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -12,8 +12,8 @@
 #include "scheduler.h"
 #include "../utils/pqueue_support.h"
 #include "scheduler_sync_tag_advance.c"
-#include "worker_assignments.c"
-#include "worker_states.c"
+#include "worker_assignments.h"
+#include "worker_states.h"
 
 #ifndef MAX_REACTION_LEVEL
 #define MAX_REACTION_LEVEL INITIAL_REACT_QUEUE_SIZE
diff --git a/core/threaded/worker_assignments.c b/core/threaded/worker_assignments.h
similarity index 98%
rename from core/threaded/worker_assignments.c
rename to core/threaded/worker_assignments.h
index c1b67a06b..e0d78ab50 100644
--- a/core/threaded/worker_assignments.c
+++ b/core/threaded/worker_assignments.h
@@ -2,6 +2,10 @@
 #ifndef WORKER_ASSIGNMENTS
 #define WORKER_ASSIGNMENTS
 
+#ifndef NUMBER_OF_WORKERS
+#define NUMBER_OF_WORKERS 1
+#endif // NUMBER_OF_WORKERS
+
 #include <assert.h>
 #include "scheduler.h"
 
@@ -25,7 +29,7 @@ static size_t reactions_triggered_counter = 0;
 
 extern lf_mutex_t mutex;
 
-#include "data_collection.c"
+#include "data_collection.h"
 
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
diff --git a/core/threaded/worker_states.c b/core/threaded/worker_states.h
similarity index 100%
rename from core/threaded/worker_states.c
rename to core/threaded/worker_states.h

From ae97fbb50279718b21a20787c08e8dd373071803 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 6 Mar 2022 17:26:55 -0800
Subject: [PATCH 10/49] [scheduler] Small adjustments.

---
 core/threaded/data_collection.h    | 2 +-
 core/threaded/worker_assignments.h | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index e0cc6b910..998364fd2 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -14,7 +14,7 @@ static interval_t* execution_times_by_level;
 extern size_t num_levels;
 extern size_t max_num_workers;
 
-#define OPTIMAL_NANOSECONDS_WORK 65536
+#define OPTIMAL_NANOSECONDS_WORK 32768
 
 static void data_collection_init(sched_params_t* params) {
     start_times_by_level = (interval_t*) calloc(
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index e0d78ab50..bfb77ce37 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -16,6 +16,8 @@ static size_t* num_workers_by_level;
 static size_t num_levels;
 static size_t max_num_workers;
 
+static bool collecting_data = true;
+
 // The following apply to the current level.
 static size_t current_level;
 static size_t num_workers_busy;
@@ -39,7 +41,6 @@ extern lf_mutex_t mutex;
  */
 static void set_level(size_t level) {
     static size_t data_collection_counter = 0;
-    static bool collecting_data = true;
     assert(level < num_levels);
     assert(0 <= level);
     if (collecting_data) data_collection_end_level(current_level);
@@ -176,7 +177,7 @@ static size_t get_num_workers_busy() {
 static bool try_increment_level() {
     assert(num_workers_busy == 0);
     if (current_level + 1 == num_levels) {
-        data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
+        if (collecting_data) data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
         set_level(0);
         return true;
     }

From c34ef01c9d1cbd49aa688d9827c6b8b6fcd38dfe Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Mon, 7 Mar 2022 01:04:47 -0800
Subject: [PATCH 11/49] [scheduler] Try to dynamically optimize num workers.

---
 core/threaded/data_collection.h    | 43 +++++++++++++++++++++++-------
 core/threaded/worker_assignments.h | 16 +++--------
 2 files changed, 37 insertions(+), 22 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 998364fd2..45e9f50ba 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -11,18 +11,26 @@
 
 static interval_t* start_times_by_level;
 static interval_t* execution_times_by_level;
+static interval_t* execution_times_mins;
+static size_t* execution_times_argmins;
+static size_t data_collection_counter = 0;
+static bool collecting_data = false;
+static int experimental_jitter = 0;
+
 extern size_t num_levels;
 extern size_t max_num_workers;
 
 #define OPTIMAL_NANOSECONDS_WORK 32768
 
 static void data_collection_init(sched_params_t* params) {
-    start_times_by_level = (interval_t*) calloc(
-        params->num_reactions_per_level_size, sizeof(interval_t)
-    );
-    execution_times_by_level = (interval_t*) calloc(
-        params->num_reactions_per_level_size, sizeof(interval_t)
-    );
+    size_t num_levels = params->num_reactions_per_level_size;
+    start_times_by_level = (interval_t*) calloc(num_levels, sizeof(interval_t));
+    execution_times_by_level = (interval_t*) calloc(num_levels, sizeof(interval_t));
+    execution_times_mins = (interval_t*) calloc(num_levels, sizeof(interval_t));
+    execution_times_argmins = (size_t*) malloc(num_levels * sizeof(size_t));
+    for (size_t i = 0; i < num_levels; i++) {
+        execution_times_argmins[i] = max_num_workers;
+    }
 }
 
 static void data_collection_free() {
@@ -31,15 +39,29 @@ static void data_collection_free() {
 }
 
 static void data_collection_start_level(size_t level) {
-    start_times_by_level[level] = get_physical_time();
+    if (collecting_data) start_times_by_level[level] = get_physical_time();
 }
 
-static void data_collection_end_level(size_t level) {
-    if (start_times_by_level[level]) {
+static void data_collection_end_level(size_t level, size_t num_workers) {
+    if (collecting_data && start_times_by_level[level]) {
         execution_times_by_level[level] = (
             3 * execution_times_by_level[level]
             + get_physical_time() - start_times_by_level[level]
         ) >> 2;
+        interval_t score = execution_times_by_level[level] + execution_times_by_level[
+            (level + num_levels - 1) % num_levels
+        ];
+        if (!execution_times_mins[level] | (score < execution_times_mins[level])) {
+            execution_times_mins[level] = score;
+            execution_times_argmins[level] = num_workers;
+        }
+    }
+    if (level == 0) {
+        data_collection_counter++;
+        int shift = (data_collection_counter > 8) << 3;
+        size_t shifted = data_collection_counter >> shift;
+        collecting_data = data_collection_counter == (shifted << shift);
+        experimental_jitter = ((int) (shifted % 3)) - 1;
     }
 }
 
@@ -47,8 +69,11 @@ static void data_collection_compute_number_of_workers(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level
 ) {
+    if (!collecting_data) return;
     for (size_t level = 0; level < num_levels; level++) {
         size_t ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
+        ideal_number_of_workers = (ideal_number_of_workers + execution_times_argmins[level]) >> 1;
+        ideal_number_of_workers += experimental_jitter;
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
         num_workers_by_level[level] = (ideal_number_of_workers < 1) ? 1 : (
             (ideal_number_of_workers > max_reasonable_num_workers) ? max_reasonable_num_workers :
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index bfb77ce37..d45a5a7bd 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -16,8 +16,6 @@ static size_t* num_workers_by_level;
 static size_t num_levels;
 static size_t max_num_workers;
 
-static bool collecting_data = true;
-
 // The following apply to the current level.
 static size_t current_level;
 static size_t num_workers_busy;
@@ -40,23 +38,15 @@ extern lf_mutex_t mutex;
  * @param level The new current level.
  */
 static void set_level(size_t level) {
-    static size_t data_collection_counter = 0;
     assert(level < num_levels);
     assert(0 <= level);
-    if (collecting_data) data_collection_end_level(current_level);
-    if (level == 0) {
-        data_collection_counter++;
-        int shift = 3 << (data_collection_counter > 8);
-        collecting_data = data_collection_counter == (
-            (data_collection_counter >> shift) << shift
-        );
-    }
+    data_collection_end_level(current_level, num_workers);
     current_level = level;
     num_workers_busy = num_workers_by_level[level];
     num_reactions_by_worker = num_reactions_by_worker_by_level[level];
     reactions_by_worker = reactions_by_worker_by_level[level];
     num_workers = num_workers_by_level[level];
-    if (collecting_data) data_collection_start_level(current_level);
+    data_collection_start_level(current_level);
 }
 
 static void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
@@ -177,7 +167,7 @@ static size_t get_num_workers_busy() {
 static bool try_increment_level() {
     assert(num_workers_busy == 0);
     if (current_level + 1 == num_levels) {
-        if (collecting_data) data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
+        data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
         set_level(0);
         return true;
     }

From 10736a0971eb1793e2749a44c85feae762b9b812 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 8 Mar 2022 00:07:07 -0800
Subject: [PATCH 12/49] [scheduler] First possible improvement due to runtime
 analysis.

---
 core/threaded/data_collection.h | 36 +++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 45e9f50ba..ec3eebdcc 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -15,12 +15,14 @@ static interval_t* execution_times_mins;
 static size_t* execution_times_argmins;
 static size_t data_collection_counter = 0;
 static bool collecting_data = false;
+static bool completing_experiment = false;
 static int experimental_jitter = 0;
 
 extern size_t num_levels;
 extern size_t max_num_workers;
 
 #define OPTIMAL_NANOSECONDS_WORK 32768
+#define STOP_USING_OPTIMAL_NANOSECONDS_WORK 15
 
 static void data_collection_init(sched_params_t* params) {
     size_t num_levels = params->num_reactions_per_level_size;
@@ -51,7 +53,7 @@ static void data_collection_end_level(size_t level, size_t num_workers) {
         interval_t score = execution_times_by_level[level] + execution_times_by_level[
             (level + num_levels - 1) % num_levels
         ];
-        if (!execution_times_mins[level] | (score < execution_times_mins[level])) {
+        if (!execution_times_mins[level] | (score < execution_times_mins[level]) | (num_workers == execution_times_argmins[level])) {
             execution_times_mins[level] = score;
             execution_times_argmins[level] = num_workers;
         }
@@ -61,23 +63,41 @@ static void data_collection_end_level(size_t level, size_t num_workers) {
         int shift = (data_collection_counter > 8) << 3;
         size_t shifted = data_collection_counter >> shift;
         collecting_data = data_collection_counter == (shifted << shift);
-        experimental_jitter = ((int) (shifted % 3)) - 1;
+        if (collecting_data) {
+            experimental_jitter = ((int) (shifted % 3)) - 1;
+            completing_experiment = experimental_jitter;
+            // printf("%d\n", shifted % 3);
+        }
+        if (completing_experiment && !collecting_data) {
+            collecting_data = true;
+            experimental_jitter = 0;
+            completing_experiment = false;
+        }
     }
 }
 
+static size_t restrict_to_range(size_t start_inclusive, size_t end_inclusive, size_t value) {
+    if (value < start_inclusive) return start_inclusive;
+    if (value > end_inclusive) return end_inclusive;
+    return value;
+}
+
 static void data_collection_compute_number_of_workers(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level
 ) {
     if (!collecting_data) return;
     for (size_t level = 0; level < num_levels; level++) {
-        size_t ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
-        ideal_number_of_workers = (ideal_number_of_workers + execution_times_argmins[level]) >> 1;
-        ideal_number_of_workers += experimental_jitter;
+        size_t ideal_number_of_workers;
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
-        num_workers_by_level[level] = (ideal_number_of_workers < 1) ? 1 : (
-            (ideal_number_of_workers > max_reasonable_num_workers) ? max_reasonable_num_workers :
-            ideal_number_of_workers
+        if (data_collection_counter < STOP_USING_OPTIMAL_NANOSECONDS_WORK) {
+            ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
+        } else {
+            ideal_number_of_workers = execution_times_argmins[level] + experimental_jitter;
+        }
+        // printf("level=%ld, num_workers=%ld, jitter=%d\n", level, ideal_number_of_workers, experimental_jitter);
+        num_workers_by_level[level] = restrict_to_range(
+            1, max_reasonable_num_workers, ideal_number_of_workers
         );
     }
 }

From bb37043f06f810ec850ca7fd376fdc917b2b484f Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 8 Mar 2022 18:14:30 -0800
Subject: [PATCH 13/49] [scheduler] Dirty patch on the heuristic.

---
 core/threaded/data_collection.h | 34 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index ec3eebdcc..3dad0bb85 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -22,7 +22,8 @@ extern size_t num_levels;
 extern size_t max_num_workers;
 
 #define OPTIMAL_NANOSECONDS_WORK 32768
-#define STOP_USING_OPTIMAL_NANOSECONDS_WORK 15
+#define STOP_USING_OPTIMAL_NANOSECONDS_WORK 5
+#define SLOW_EXPERIMENTS 256
 
 static void data_collection_init(sched_params_t* params) {
     size_t num_levels = params->num_reactions_per_level_size;
@@ -46,33 +47,28 @@ static void data_collection_start_level(size_t level) {
 
 static void data_collection_end_level(size_t level, size_t num_workers) {
     if (collecting_data && start_times_by_level[level]) {
-        execution_times_by_level[level] = (
-            3 * execution_times_by_level[level]
-            + get_physical_time() - start_times_by_level[level]
-        ) >> 2;
+        execution_times_by_level[level] = get_physical_time() - start_times_by_level[level];
         interval_t score = execution_times_by_level[level] + execution_times_by_level[
             (level + num_levels - 1) % num_levels
         ];
         if (!execution_times_mins[level] | (score < execution_times_mins[level]) | (num_workers == execution_times_argmins[level])) {
+            // printf(
+            //     "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
+            //     execution_times_argmins[level], execution_times_mins[level],
+            //     num_workers, score,
+            //     level
+            // );
             execution_times_mins[level] = score;
             execution_times_argmins[level] = num_workers;
         }
     }
     if (level == 0) {
         data_collection_counter++;
-        int shift = (data_collection_counter > 8) << 3;
+        int shift = (data_collection_counter > SLOW_EXPERIMENTS) << 3;
         size_t shifted = data_collection_counter >> shift;
-        collecting_data = data_collection_counter == (shifted << shift);
-        if (collecting_data) {
-            experimental_jitter = ((int) (shifted % 3)) - 1;
-            completing_experiment = experimental_jitter;
-            // printf("%d\n", shifted % 3);
-        }
-        if (completing_experiment && !collecting_data) {
-            collecting_data = true;
-            experimental_jitter = 0;
-            completing_experiment = false;
-        }
+        completing_experiment = !completing_experiment & collecting_data;
+        collecting_data = completing_experiment | (data_collection_counter == (shifted << shift));
+        experimental_jitter = ((int) (shifted % 3)) - 1;
     }
 }
 
@@ -93,7 +89,9 @@ static void data_collection_compute_number_of_workers(
         if (data_collection_counter < STOP_USING_OPTIMAL_NANOSECONDS_WORK) {
             ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
         } else {
-            ideal_number_of_workers = execution_times_argmins[level] + experimental_jitter;
+            ideal_number_of_workers = execution_times_argmins[level];
+            if (!completing_experiment) ideal_number_of_workers += experimental_jitter;
+            // printf("Assigning %ld @ %ld.\n", ideal_number_of_workers, level);
         }
         // printf("level=%ld, num_workers=%ld, jitter=%d\n", level, ideal_number_of_workers, experimental_jitter);
         num_workers_by_level[level] = restrict_to_range(

From 14664164bb210a4d2eae4828a4ae638cfdee66bb Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 19 Mar 2022 14:22:34 -0700
Subject: [PATCH 14/49] [scheduler] Advance level as far as necessary at once.

Marked improvement observed locally for RadixSort, Counting.
---
 core/threaded/scheduler_NP2.c      |  4 +---
 core/threaded/worker_assignments.h | 19 +++++++++++--------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index c4214bf13..bf13fef9a 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -2,7 +2,6 @@
  * This is a non-priority-driven scheduler. See scheduler.h for documentation.
  */
 
-
 #ifndef NUMBER_OF_WORKERS
 #define NUMBER_OF_WORKERS 1
 #endif // NUMBER_OF_WORKERS
@@ -33,7 +32,7 @@ extern lf_mutex_t mutex;
  * @param worker The number of the calling worker.
  */
 static void advance_level_and_unlock(size_t worker) {
-    if (try_increment_level()) {
+    if (try_advance_level()) {
         if (_lf_sched_advance_tag_locked()) {
             should_stop = true;
             worker_states_never_sleep_again();
@@ -75,7 +74,6 @@ reaction_t* lf_sched_get_ready_reaction(int worker_number) {
         // printf("%d failed to get.\n", worker_number);
         size_t level_counter_snapshot = level_counter;
         if (worker_assignments_finished_with_level_locked(worker_number)) {
-            // TODO: Advance level all the way to the next level with at least one reaction?
             advance_level_and_unlock(worker_number);
             // printf("%d !\n", worker_number);
         } else {
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index d45a5a7bd..bd3b5328d 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -160,19 +160,22 @@ static size_t get_num_workers_busy() {
 }
 
 /**
- * @brief Increment the level currently being processed by the workers.
+ * @brief Advance the level currently being processed by the workers.
  * 
  * @return true If the level was already at the maximum and was reset to zero.
  */
-static bool try_increment_level() {
+static bool try_advance_level() {
     assert(num_workers_busy == 0);
-    if (current_level + 1 == num_levels) {
-        data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
-        set_level(0);
-        return true;
+    size_t max_level = num_levels - 1;
+    while (current_level < max_level) {
+        set_level(current_level + 1);
+        for (size_t i = 0; i < num_workers; i++) {
+            if (num_reactions_by_worker[i]) return false;
+        }
     }
-    set_level(current_level + 1);
-    return false;
+    data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
+    set_level(0);
+    return true;
 }
 
 #endif

From 09b18cca924daf346508d1a2fed6e35b4cbad7bf Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sun, 20 Mar 2022 14:38:31 -0700
Subject: [PATCH 15/49] [scheduler] If everyone else is sleeping, do not
 acquire the mutex.

This optimization is lifted directly from the NP scheduler in response to visible differences in the number of cycles spent accessing mutexes.

Clear improvement is observed locally for the Counting benchmark.
---
 core/threaded/scheduler_NP2.c      |  6 ++--
 core/threaded/worker_assignments.h | 15 ++++++---
 core/threaded/worker_states.h      | 53 +++++++++++++++++++++++++++---
 3 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_NP2.c
index bf13fef9a..9dab1befc 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_NP2.c
@@ -21,8 +21,6 @@
 static bool init_called = false;
 static bool should_stop = false;
 
-extern lf_mutex_t mutex;
-
 ///////////////////////// Scheduler Private Functions ///////////////////////////
 
 /**
@@ -36,7 +34,7 @@ static void advance_level_and_unlock(size_t worker) {
         if (_lf_sched_advance_tag_locked()) {
             should_stop = true;
             worker_states_never_sleep_again();
-            lf_mutex_unlock(&mutex);
+            worker_states_unlock(worker);
             return;
         }
     }
@@ -45,7 +43,7 @@ static void advance_level_and_unlock(size_t worker) {
     if (num_workers_busy < worker && num_workers_busy) {  // FIXME: Is this branch still necessary?
         worker_states_sleep_and_unlock(worker, level_snapshot);
     } else {
-        lf_mutex_unlock(&mutex);
+        worker_states_unlock(worker);
     }
 }
 
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index bd3b5328d..c841673e7 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -16,21 +16,26 @@ static size_t* num_workers_by_level;
 static size_t num_levels;
 static size_t max_num_workers;
 
-// The following apply to the current level.
+/** The following values apply to the current level. */
 static size_t current_level;
+/** The number of workers who still have not finished their work. */
 static size_t num_workers_busy;
+/** The number of reactions each worker still has to execute, indexed by worker. */
 static size_t* num_reactions_by_worker;
+/** The reactions to be executed, indexed by assigned worker. */
 static reaction_t*** reactions_by_worker;
+/** The total number of workers active, including those who have finished their work. */
 static size_t num_workers;
 
 // A counter of the number of reactions triggered. No function should depend on the precise
 // correctness of this value. Race conditions when accessing this value are acceptable.
 static size_t reactions_triggered_counter = 0;
 
-extern lf_mutex_t mutex;
-
 #include "data_collection.h"
 
+static void worker_states_lock(size_t worker);
+static void worker_states_unlock(size_t worker);
+
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
  * impossible.
@@ -104,11 +109,11 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
         // printf("%ld <- %p @ %lld\n", worker, ret, LEVEL(ret->index));
         return ret;
     }
-    lf_mutex_lock(&mutex);
+    worker_states_lock(worker);
     if (!num_reactions_by_worker[worker]) {
         return NULL;
     }
-    lf_mutex_unlock(&mutex);
+    worker_states_unlock(worker);
     return reactions_by_worker[worker][--num_reactions_by_worker[worker]];
 }
 
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 41f7b5c8a..86b4420fe 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -20,12 +20,17 @@ extern size_t** num_reactions_by_worker_by_level;
 extern size_t max_num_workers;
 extern lf_mutex_t mutex;
 
+/** The number of non-waiting threads. */
+static volatile size_t num_loose_threads;
+/** Whether the mutex is held by each worker via this module's API. */
+static bool* mutex_held;
+
 /**
  * The level counter is a number that changes whenever the current level changes.
  *
- * This number must have a very long period in the sense that if it changes, the probability that it
- * is checked at a time in the future that is selected from some "reasonable" distribution, the
- * probability that it will have returned to the same value is vanishingly small.
+ * This number must have a very long period in the sense that if it changes and is checked at a time
+ * in the future that is selected from some "reasonable" distribution, the probability that it will
+ * have returned to the same value must be negligible.
  */
 static size_t level_counter = 0;
 
@@ -51,14 +56,17 @@ static void worker_states_init(size_t number_of_workers) {
     size_t greatest_worker_number = number_of_workers - 1;
     size_t num_conds = cond_of(greatest_worker_number) + 1;
     worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
+    mutex_held = (bool*) malloc(sizeof(bool) * number_of_workers);
     for (int i = 0; i < num_conds; i++) {
         lf_cond_init(worker_conds + i);
     }
+    num_loose_threads = number_of_workers;
 }
 
 static void worker_states_free() {
     // FIXME: Why do the condition variables and mutexes not need to be freed?
     free(worker_conds);
+    free(mutex_held);
 }
 
 /**
@@ -72,10 +80,11 @@ static size_t worker_states_awaken_locked(size_t num_to_awaken) {
     num_awakened = num_to_awaken;
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
     size_t max_cond = cond_of(greatest_worker_number_to_awaken);
+    size_t ret = ++level_counter;
+    // printf("Broadcasting to %ld workers.\n", num_to_awaken);
     for (int cond = 0; cond <= max_cond; cond++) {
         lf_cond_broadcast(worker_conds + cond);
     }
-    size_t ret = ++level_counter;
     return ret;
 }
 
@@ -91,6 +100,32 @@ static void worker_states_never_sleep_again() {
     lf_mutex_unlock(&mutex);
 }
 
+/** Lock the global mutex if needed. */
+static void worker_states_lock(size_t worker) {
+    assert(num_loose_threads > 0);
+    assert(num_loose_threads <= max_num_workers);
+    if (num_loose_threads > 1) {
+        // printf("%ld locking mutex.\n", worker);
+        lf_mutex_lock(&mutex);
+        // printf("%ld locked mutex.\n", worker);
+        assert(mutex_held[worker] == false);
+        mutex_held[worker] = true;
+    } else {
+        // printf("%ld not locking mutex.\n", worker);
+    }
+}
+
+/** Unlock the global mutex if needed. */
+static void worker_states_unlock(size_t worker) {
+    if (mutex_held[worker]) {
+        // printf("%ld unlocking mutex.\n", worker);
+        mutex_held[worker] = false;
+        lf_mutex_unlock(&mutex);
+    } else {
+        // printf("%ld not unlocking mutex.\n", worker);
+    }
+}
+
 /**
  * @brief Make the given worker go to sleep.
  *
@@ -103,12 +138,22 @@ static void worker_states_never_sleep_again() {
  */
 static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_snapshot) {
     assert(worker < max_num_workers);
+    assert(num_loose_threads <= max_num_workers);
+    if (!mutex_held[worker]) {
+        lf_mutex_lock(&mutex);
+    }
+    mutex_held[worker] = false;  // This will be true soon, upon call to lf_cond_wait.
+    // printf("%ld sleeping; nlt=%ld\n", worker, num_loose_threads);
     size_t cond = cond_of(worker);
     if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
         do {
+            num_loose_threads--;
             lf_cond_wait(worker_conds + cond, &mutex);
+            num_loose_threads++;
         } while (worker >= num_awakened);
     }
+    // printf("%ld awakening; nlt=%ld\n", worker, num_loose_threads);
+    assert(mutex_held[worker] == false);  // This thread holds the mutex, but it did not report that.
     lf_mutex_unlock(&mutex);
 }
 

From 34c1334cae1709e93647144e0572b9b46085a66d Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Fri, 25 Mar 2022 10:45:11 -0700
Subject: [PATCH 16/49] [scheduler] Fix bug caused by previous commit.

This is another step in the direction of the NP scheduler. The condition for level advancement should be that _everyone_ is sleeping; otherwise, we cannot optimize out the lock for level advancement without introducing a race condition.
---
 ...{scheduler_NP2.c => scheduler_heuristic.c} | 49 ++++++++++----
 core/threaded/worker_assignments.h            | 30 ---------
 core/threaded/worker_states.h                 | 66 +++++++++++++++----
 3 files changed, 90 insertions(+), 55 deletions(-)
 rename core/threaded/{scheduler_NP2.c => scheduler_heuristic.c} (60%)

diff --git a/core/threaded/scheduler_NP2.c b/core/threaded/scheduler_heuristic.c
similarity index 60%
rename from core/threaded/scheduler_NP2.c
rename to core/threaded/scheduler_heuristic.c
index 9dab1befc..b2b16dc79 100644
--- a/core/threaded/scheduler_NP2.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -1,3 +1,28 @@
+/*************
+Copyright (c) 2022, The University of California at Berkeley.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+***************/
+
 /**
  * This is a non-priority-driven scheduler. See scheduler.h for documentation.
  */
@@ -18,6 +43,8 @@
 #define MAX_REACTION_LEVEL INITIAL_REACT_QUEUE_SIZE
 #endif
 
+extern bool fast;
+
 static bool init_called = false;
 static bool should_stop = false;
 
@@ -31,20 +58,21 @@ static bool should_stop = false;
  */
 static void advance_level_and_unlock(size_t worker) {
     if (try_advance_level()) {
+        if (!fast && !mutex_held[worker]) {
+            mutex_held[worker] = true;
+            lf_mutex_lock(&mutex);
+        }
         if (_lf_sched_advance_tag_locked()) {
             should_stop = true;
-            worker_states_never_sleep_again();
+            worker_states_never_sleep_again(worker);
             worker_states_unlock(worker);
             return;
         }
     }
-    size_t num_workers_busy = get_num_workers_busy();
-    size_t level_snapshot = worker_states_awaken_locked(num_workers_busy);
-    if (num_workers_busy < worker && num_workers_busy) {  // FIXME: Is this branch still necessary?
-        worker_states_sleep_and_unlock(worker, level_snapshot);
-    } else {
-        worker_states_unlock(worker);
-    }
+    size_t num_workers_to_awaken = num_workers;
+    assert(num_workers_to_awaken > 0);
+    worker_states_awaken_locked(worker, num_workers_to_awaken);
+    worker_states_unlock(worker);
 }
 
 ///////////////////// Scheduler Init and Destroy API /////////////////////////
@@ -69,11 +97,10 @@ reaction_t* lf_sched_get_ready_reaction(int worker_number) {
     assert(worker_number >= 0);
     reaction_t* ret;
     while (!(ret = worker_assignments_get_or_lock(worker_number))) {
-        // printf("%d failed to get.\n", worker_number);
         size_t level_counter_snapshot = level_counter;
-        if (worker_assignments_finished_with_level_locked(worker_number)) {
-            advance_level_and_unlock(worker_number);
+        if (worker_states_finished_with_level_locked(worker_number)) {
             // printf("%d !\n", worker_number);
+            advance_level_and_unlock(worker_number);
         } else {
             worker_states_sleep_and_unlock(worker_number, level_counter_snapshot);
         }
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index c841673e7..9265b00a4 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -18,8 +18,6 @@ static size_t max_num_workers;
 
 /** The following values apply to the current level. */
 static size_t current_level;
-/** The number of workers who still have not finished their work. */
-static size_t num_workers_busy;
 /** The number of reactions each worker still has to execute, indexed by worker. */
 static size_t* num_reactions_by_worker;
 /** The reactions to be executed, indexed by assigned worker. */
@@ -47,7 +45,6 @@ static void set_level(size_t level) {
     assert(0 <= level);
     data_collection_end_level(current_level, num_workers);
     current_level = level;
-    num_workers_busy = num_workers_by_level[level];
     num_reactions_by_worker = num_reactions_by_worker_by_level[level];
     reactions_by_worker = reactions_by_worker_by_level[level];
     num_workers = num_workers_by_level[level];
@@ -117,23 +114,6 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
     return reactions_by_worker[worker][--num_reactions_by_worker[worker]];
 }
 
-/**
- * @brief Record that worker is finished working on the current level.
- * 
- * @param worker The number of a worker.
- * @return true If this is the last worker to finish working on the current level.
- * @return false If at least one other worker is still working on the current level.
- */
-static bool worker_assignments_finished_with_level_locked(size_t worker) {
-    assert(worker >= 0);
-    // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
-    assert(num_workers_busy > 0 || worker >= num_workers);
-    assert(num_reactions_by_worker[worker] != 1);
-    assert(num_reactions_by_worker[worker] == (((size_t) 0) - 1) || num_reactions_by_worker[worker] == 0);
-    num_workers_busy -= worker < num_workers;
-    return !num_workers_busy;
-}
-
 /**
  * @brief Trigger the given reaction.
  * 
@@ -155,22 +135,12 @@ static void worker_assignments_put(reaction_t* reaction) {
     reactions_by_worker_by_level[level][worker][num_preceding_reactions] = reaction;
 }
 
-/**
- * @brief Get the number of workers that should currently be working.
- * 
- * @return size_t The number of workers that should currently be working.
- */
-static size_t get_num_workers_busy() {
-    return num_workers_busy;
-}
-
 /**
  * @brief Advance the level currently being processed by the workers.
  * 
  * @return true If the level was already at the maximum and was reset to zero.
  */
 static bool try_advance_level() {
-    assert(num_workers_busy == 0);
     size_t max_level = num_levels - 1;
     while (current_level < max_level) {
         set_level(current_level + 1);
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 86b4420fe..4aa205f29 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -10,8 +10,8 @@
 #include "scheduler.h"
 #include "../platform.h"
 
-static size_t num_awakened = 0;
 static lf_cond_t* worker_conds;
+static size_t* cumsum_of_cond_of;
 
 static bool worker_states_sleep_forbidden = false;
 
@@ -22,6 +22,7 @@ extern lf_mutex_t mutex;
 
 /** The number of non-waiting threads. */
 static volatile size_t num_loose_threads;
+static volatile size_t num_awakened;
 /** Whether the mutex is held by each worker via this module's API. */
 static bool* mutex_held;
 
@@ -56,7 +57,14 @@ static void worker_states_init(size_t number_of_workers) {
     size_t greatest_worker_number = number_of_workers - 1;
     size_t num_conds = cond_of(greatest_worker_number) + 1;
     worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
+    cumsum_of_cond_of = (size_t*) calloc(num_conds, sizeof(size_t));
     mutex_held = (bool*) malloc(sizeof(bool) * number_of_workers);
+    for (int i = 0; i < number_of_workers; i++) {
+        cumsum_of_cond_of[cond_of(i)]++;
+    }
+    for (int i = 1; i < num_conds; i++) {
+        cumsum_of_cond_of[i] += cumsum_of_cond_of[i - 1];
+    }
     for (int i = 0; i < num_conds; i++) {
         lf_cond_init(worker_conds + i);
     }
@@ -72,20 +80,31 @@ static void worker_states_free() {
 /**
  * @brief Awaken the workers scheduled to work on the current level.
  * 
+ * @param worker The calling worker.
  * @param num_to_awaken The number of workers to awaken.
  * @return A snapshot of the level counter after awakening the workers.
  */
-static size_t worker_states_awaken_locked(size_t num_to_awaken) {
+static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     assert(num_to_awaken <= max_num_workers);
-    num_awakened = num_to_awaken;
+    if ((worker | num_to_awaken) == 0) {
+        num_loose_threads = 1;
+        level_counter++;
+        return;
+    }
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
     size_t max_cond = cond_of(greatest_worker_number_to_awaken);
-    size_t ret = ++level_counter;
-    // printf("Broadcasting to %ld workers.\n", num_to_awaken);
+    // printf("%ld +-> %ld\n", worker, num_to_awaken);
+    if (!mutex_held[worker]) {
+        mutex_held[worker] = true;
+        lf_mutex_lock(&mutex);
+    }
+    num_loose_threads = cumsum_of_cond_of[max_cond];
+    num_loose_threads += worker >= num_loose_threads;
+    num_awakened = num_loose_threads;
+    level_counter++;
     for (int cond = 0; cond <= max_cond; cond++) {
         lf_cond_broadcast(worker_conds + cond);
     }
-    return ret;
 }
 
 /**
@@ -93,10 +112,10 @@ static size_t worker_states_awaken_locked(size_t num_to_awaken) {
  * 
  * This is intended to coordinate shutdown (without leaving any dangling threads behind).
  */
-static void worker_states_never_sleep_again() {
+static void worker_states_never_sleep_again(size_t worker) {
     worker_states_sleep_forbidden = true;
     lf_mutex_lock(&mutex);
-    worker_states_awaken_locked(max_num_workers);
+    worker_states_awaken_locked(worker, max_num_workers);
     lf_mutex_unlock(&mutex);
 }
 
@@ -104,7 +123,9 @@ static void worker_states_never_sleep_again() {
 static void worker_states_lock(size_t worker) {
     assert(num_loose_threads > 0);
     assert(num_loose_threads <= max_num_workers);
-    if (num_loose_threads > 1) {
+    size_t lt = num_loose_threads;
+    // printf("%ld sees %ld loose threads\n", worker, lt);
+    if (lt > 1) {
         // printf("%ld locking mutex.\n", worker);
         lf_mutex_lock(&mutex);
         // printf("%ld locked mutex.\n", worker);
@@ -126,6 +147,25 @@ static void worker_states_unlock(size_t worker) {
     }
 }
 
+/**
+ * @brief Record that worker is finished working on the current level.
+ *
+ * @param worker The number of a worker.
+ * @return true If this is the last worker to finish working on the current level.
+ * @return false If at least one other worker is still working on the current level.
+ */
+static bool worker_states_finished_with_level_locked(size_t worker) {
+    assert(worker >= 0);
+    assert(num_loose_threads > 0);
+    assert(num_reactions_by_worker[worker] != 1);
+    assert(num_reactions_by_worker[worker] == (((size_t) 0) - 1) || num_reactions_by_worker[worker] == 0);
+    // Why use an atomic operation when we are supposed to be "as good as locked"? Because I took a
+    // shortcut, and it wasn't perfect.
+    size_t ret = lf_atomic_add_fetch(&num_loose_threads, -1);
+    // printf("worker=%ld, nlt=%ld\n", worker, ret);
+    return !ret;
+}
+
 /**
  * @brief Make the given worker go to sleep.
  *
@@ -143,16 +183,14 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
         lf_mutex_lock(&mutex);
     }
     mutex_held[worker] = false;  // This will be true soon, upon call to lf_cond_wait.
-    // printf("%ld sleeping; nlt=%ld\n", worker, num_loose_threads);
+    // printf("%ld sleep; nlt=%ld\n", worker, num_loose_threads);
     size_t cond = cond_of(worker);
     if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
         do {
-            num_loose_threads--;
             lf_cond_wait(worker_conds + cond, &mutex);
-            num_loose_threads++;
-        } while (worker >= num_awakened);
+        } while (level_counter_snapshot == level_counter || worker >= num_awakened);
     }
-    // printf("%ld awakening; nlt=%ld\n", worker, num_loose_threads);
+    // printf("%ld wake; nlt=%ld\n", worker, num_loose_threads);
     assert(mutex_held[worker] == false);  // This thread holds the mutex, but it did not report that.
     lf_mutex_unlock(&mutex);
 }

From c03077bfc345133c1a5ebce88ab7f1ec460d3de5 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Fri, 25 Mar 2022 16:13:59 -0700
Subject: [PATCH 17/49] [scheduler] Adjust data_collection.h.

---
 core/threaded/data_collection.h     | 83 ++++++++++++++++-------------
 core/threaded/scheduler_heuristic.c |  2 +
 core/threaded/worker_assignments.h  | 38 ++++++-------
 core/threaded/worker_states.h       |  2 +-
 4 files changed, 69 insertions(+), 56 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 3dad0bb85..2c2381ccd 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -15,14 +15,10 @@ static interval_t* execution_times_mins;
 static size_t* execution_times_argmins;
 static size_t data_collection_counter = 0;
 static bool collecting_data = false;
-static bool completing_experiment = false;
-static int experimental_jitter = 0;
 
 extern size_t num_levels;
 extern size_t max_num_workers;
 
-#define OPTIMAL_NANOSECONDS_WORK 32768
-#define STOP_USING_OPTIMAL_NANOSECONDS_WORK 5
 #define SLOW_EXPERIMENTS 256
 
 static void data_collection_init(sched_params_t* params) {
@@ -48,27 +44,6 @@ static void data_collection_start_level(size_t level) {
 static void data_collection_end_level(size_t level, size_t num_workers) {
     if (collecting_data && start_times_by_level[level]) {
         execution_times_by_level[level] = get_physical_time() - start_times_by_level[level];
-        interval_t score = execution_times_by_level[level] + execution_times_by_level[
-            (level + num_levels - 1) % num_levels
-        ];
-        if (!execution_times_mins[level] | (score < execution_times_mins[level]) | (num_workers == execution_times_argmins[level])) {
-            // printf(
-            //     "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
-            //     execution_times_argmins[level], execution_times_mins[level],
-            //     num_workers, score,
-            //     level
-            // );
-            execution_times_mins[level] = score;
-            execution_times_argmins[level] = num_workers;
-        }
-    }
-    if (level == 0) {
-        data_collection_counter++;
-        int shift = (data_collection_counter > SLOW_EXPERIMENTS) << 3;
-        size_t shifted = data_collection_counter >> shift;
-        completing_experiment = !completing_experiment & collecting_data;
-        collecting_data = completing_experiment | (data_collection_counter == (shifted << shift));
-        experimental_jitter = ((int) (shifted % 3)) - 1;
     }
 }
 
@@ -78,25 +53,61 @@ static size_t restrict_to_range(size_t start_inclusive, size_t end_inclusive, si
     return value;
 }
 
-static void data_collection_compute_number_of_workers(
+static void compute_number_of_workers(
     size_t* num_workers_by_level,
-    size_t* max_num_workers_by_level
+    size_t* max_num_workers_by_level,
+    int jitter
 ) {
-    if (!collecting_data) return;
     for (size_t level = 0; level < num_levels; level++) {
         size_t ideal_number_of_workers;
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
-        if (data_collection_counter < STOP_USING_OPTIMAL_NANOSECONDS_WORK) {
-            ideal_number_of_workers = execution_times_by_level[level] / OPTIMAL_NANOSECONDS_WORK;
-        } else {
-            ideal_number_of_workers = execution_times_argmins[level];
-            if (!completing_experiment) ideal_number_of_workers += experimental_jitter;
-            // printf("Assigning %ld @ %ld.\n", ideal_number_of_workers, level);
-        }
-        // printf("level=%ld, num_workers=%ld, jitter=%d\n", level, ideal_number_of_workers, experimental_jitter);
+        ideal_number_of_workers = execution_times_argmins[level] + jitter;
         num_workers_by_level[level] = restrict_to_range(
             1, max_reasonable_num_workers, ideal_number_of_workers
         );
+        // printf("level=%ld, jitter=%d, inow=%ld, mrnw=%ld, result=%ld\n",
+        // level, jitter, ideal_number_of_workers, max_reasonable_num_workers, num_workers_by_level[level]);
+    }
+}
+
+static void data_collection_end_tag(
+    size_t* num_workers_by_level,
+    size_t* max_num_workers_by_level
+) {
+    if (collecting_data && execution_times_by_level[0]) {
+        for (size_t level = 0; level < num_levels; level++) {
+            interval_t score = execution_times_by_level[level];
+            if (
+                !execution_times_mins[level]
+                | (score < execution_times_mins[level])
+                | (num_workers_by_level[level] == execution_times_argmins[level])
+            ) {
+                printf(
+                    "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
+                    execution_times_argmins[level], execution_times_mins[level],
+                    num_workers_by_level[level], score,
+                    level
+                );
+                execution_times_mins[level] = score;
+                execution_times_argmins[level] = num_workers_by_level[level];
+            }
+        }
+    }
+    data_collection_counter++;
+    size_t period = 2 + 128 * (data_collection_counter > SLOW_EXPERIMENTS);
+    size_t state = data_collection_counter % period;
+    if (state == 0) {
+        compute_number_of_workers(
+            num_workers_by_level,
+            max_num_workers_by_level,
+            ((int) (rand() % 3)) - 1
+        );
+        collecting_data = true;
+        // printf("collecting data");
+    } else if (state == 1) {
+        compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, 0);
+        collecting_data = false;
+        // printf("not collecting data");
     }
 }
 
diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index b2b16dc79..9c50eae57 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -57,12 +57,14 @@ static bool should_stop = false;
  * @param worker The number of the calling worker.
  */
 static void advance_level_and_unlock(size_t worker) {
+    // printf("%ld advance %ld\n", worker, current_level);
     if (try_advance_level()) {
         if (!fast && !mutex_held[worker]) {
             mutex_held[worker] = true;
             lf_mutex_lock(&mutex);
         }
         if (_lf_sched_advance_tag_locked()) {
+            printf("%ld end", worker);
             should_stop = true;
             worker_states_never_sleep_again(worker);
             worker_states_unlock(worker);
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 9265b00a4..bb9a39371 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -51,6 +51,24 @@ static void set_level(size_t level) {
     data_collection_start_level(current_level);
 }
 
+/**
+ * @brief Advance the level currently being processed by the workers.
+ * 
+ * @return true If the level was already at the maximum and was reset to zero.
+ */
+static bool try_advance_level() {
+    size_t max_level = num_levels - 1;
+    while (current_level < max_level) {
+        set_level(current_level + 1);
+        for (size_t i = 0; i < num_workers; i++) {
+            if (num_reactions_by_worker[i]) return false;
+        }
+    }
+    data_collection_end_tag(num_workers_by_level, max_num_workers_by_level);
+    set_level(0);
+    return true;
+}
+
 static void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {
     num_levels = params->num_reactions_per_level_size;
     max_num_workers = number_of_workers;
@@ -93,7 +111,7 @@ static void worker_assignments_free() {
 /**
  * @brief Get a reaction for the given worker to execute. If no such reaction exists, claim the
  * mutex.
- * 
+ *
  * @param worker A worker requesting work.
  * @return reaction_t* A reaction to execute, or NULL if no such reaction exists.
  */
@@ -135,22 +153,4 @@ static void worker_assignments_put(reaction_t* reaction) {
     reactions_by_worker_by_level[level][worker][num_preceding_reactions] = reaction;
 }
 
-/**
- * @brief Advance the level currently being processed by the workers.
- * 
- * @return true If the level was already at the maximum and was reset to zero.
- */
-static bool try_advance_level() {
-    size_t max_level = num_levels - 1;
-    while (current_level < max_level) {
-        set_level(current_level + 1);
-        for (size_t i = 0; i < num_workers; i++) {
-            if (num_reactions_by_worker[i]) return false;
-        }
-    }
-    data_collection_compute_number_of_workers(num_workers_by_level, max_num_workers_by_level);
-    set_level(0);
-    return true;
-}
-
 #endif
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 4aa205f29..4e5298a4f 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -86,7 +86,7 @@ static void worker_states_free() {
  */
 static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     assert(num_to_awaken <= max_num_workers);
-    if ((worker | num_to_awaken) == 0) {
+    if ((worker == 0) && (num_to_awaken <= 1)) {
         num_loose_threads = 1;
         level_counter++;
         return;

From e58497daf3426a069e71abb506e12da359aebfb7 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 30 Mar 2022 12:19:42 -0700
Subject: [PATCH 18/49] [scheduler] Adjust data_collection.h.

---
 core/threaded/data_collection.h     | 94 +++++++++++++++++++----------
 core/threaded/scheduler_heuristic.c |  2 +-
 2 files changed, 63 insertions(+), 33 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 2c2381ccd..4a5493dbc 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -10,7 +10,7 @@
 #include "scheduler.h"
 
 static interval_t* start_times_by_level;
-static interval_t* execution_times_by_level;
+static interval_t** execution_times_by_num_workers_by_level;
 static interval_t* execution_times_mins;
 static size_t* execution_times_argmins;
 static size_t data_collection_counter = 0;
@@ -20,21 +20,31 @@ extern size_t num_levels;
 extern size_t max_num_workers;
 
 #define SLOW_EXPERIMENTS 256
+#define PARALLELISM_COST 20000
+#define EXECUTION_TIME_MEMORY 15
 
 static void data_collection_init(sched_params_t* params) {
     size_t num_levels = params->num_reactions_per_level_size;
     start_times_by_level = (interval_t*) calloc(num_levels, sizeof(interval_t));
-    execution_times_by_level = (interval_t*) calloc(num_levels, sizeof(interval_t));
+    execution_times_by_num_workers_by_level = (interval_t**) calloc(
+        num_levels, sizeof(interval_t*)
+    );
     execution_times_mins = (interval_t*) calloc(num_levels, sizeof(interval_t));
     execution_times_argmins = (size_t*) malloc(num_levels * sizeof(size_t));
     for (size_t i = 0; i < num_levels; i++) {
         execution_times_argmins[i] = max_num_workers;
+        execution_times_by_num_workers_by_level[i] = (interval_t*) calloc(
+            max_num_workers, sizeof(interval_t)
+        ) - 1;
     }
 }
 
 static void data_collection_free() {
     free(start_times_by_level);
-    free(execution_times_by_level);
+    for (size_t i = 0; i < num_levels; i++) {
+        free(execution_times_by_num_workers_by_level[i] + 1);
+    }
+    free(execution_times_by_num_workers_by_level);
 }
 
 static void data_collection_start_level(size_t level) {
@@ -43,7 +53,17 @@ static void data_collection_start_level(size_t level) {
 
 static void data_collection_end_level(size_t level, size_t num_workers) {
     if (collecting_data && start_times_by_level[level]) {
-        execution_times_by_level[level] = get_physical_time() - start_times_by_level[level];
+        if (!execution_times_by_num_workers_by_level[level][num_workers]) {
+            execution_times_by_num_workers_by_level[level][num_workers]
+                = get_physical_time() - start_times_by_level[level];
+            printf("Initialize.\n");
+        } else {
+            interval_t prior_et = execution_times_by_num_workers_by_level[level][num_workers];
+            execution_times_by_num_workers_by_level[level][num_workers] = (
+                execution_times_by_num_workers_by_level[level][num_workers] * EXECUTION_TIME_MEMORY
+                + get_physical_time() - start_times_by_level[level]
+            ) / (EXECUTION_TIME_MEMORY + 1);
+        }
     }
 }
 
@@ -56,12 +76,21 @@ static size_t restrict_to_range(size_t start_inclusive, size_t end_inclusive, si
 static void compute_number_of_workers(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level,
-    int jitter
+    bool jitter
 ) {
     for (size_t level = 0; level < num_levels; level++) {
+        interval_t this_execution_time = execution_times_by_num_workers_by_level[level][
+            num_workers_by_level[level]
+        ];
+        if (0 < this_execution_time && this_execution_time < PARALLELISM_COST && (rand() & 1)) {
+            num_workers_by_level[level] = 1;
+            continue;
+        }
         size_t ideal_number_of_workers;
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
-        ideal_number_of_workers = execution_times_argmins[level] + jitter;
+        ideal_number_of_workers = execution_times_argmins[level];
+        int range = 1;
+        if (jitter) ideal_number_of_workers += ((int) (rand() % (2 * range + 1))) - range;
         num_workers_by_level[level] = restrict_to_range(
             1, max_reasonable_num_workers, ideal_number_of_workers
         );
@@ -70,44 +99,45 @@ static void compute_number_of_workers(
     }
 }
 
+static void compute_costs(size_t* num_workers_by_level) {
+    for (size_t level = 0; level < num_levels; level++) {
+        interval_t score = execution_times_by_num_workers_by_level[level][
+            num_workers_by_level[level]
+        ];
+        if (num_workers_by_level[level] > 1) score += PARALLELISM_COST;
+        if (
+            !execution_times_mins[level]
+            | (score < execution_times_mins[level])
+            | (num_workers_by_level[level] == execution_times_argmins[level])
+        ) {
+            if (num_workers_by_level[level] != execution_times_argmins[level]) printf(
+                "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
+                execution_times_argmins[level], execution_times_mins[level],
+                num_workers_by_level[level], score,
+                level
+            );
+            execution_times_mins[level] = score;
+            execution_times_argmins[level] = num_workers_by_level[level];
+        }
+    }
+}
+
 static void data_collection_end_tag(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level
 ) {
-    if (collecting_data && execution_times_by_level[0]) {
-        for (size_t level = 0; level < num_levels; level++) {
-            interval_t score = execution_times_by_level[level];
-            if (
-                !execution_times_mins[level]
-                | (score < execution_times_mins[level])
-                | (num_workers_by_level[level] == execution_times_argmins[level])
-            ) {
-                printf(
-                    "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
-                    execution_times_argmins[level], execution_times_mins[level],
-                    num_workers_by_level[level], score,
-                    level
-                );
-                execution_times_mins[level] = score;
-                execution_times_argmins[level] = num_workers_by_level[level];
-            }
-        }
+    if (collecting_data && start_times_by_level[0]) {
+        compute_costs(num_workers_by_level);
     }
     data_collection_counter++;
     size_t period = 2 + 128 * (data_collection_counter > SLOW_EXPERIMENTS);
     size_t state = data_collection_counter % period;
     if (state == 0) {
-        compute_number_of_workers(
-            num_workers_by_level,
-            max_num_workers_by_level,
-            ((int) (rand() % 3)) - 1
-        );
+        compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, true);
         collecting_data = true;
-        // printf("collecting data");
     } else if (state == 1) {
-        compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, 0);
+        compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, false);
         collecting_data = false;
-        // printf("not collecting data");
     }
 }
 
diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index 9c50eae57..f820343bc 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -64,7 +64,7 @@ static void advance_level_and_unlock(size_t worker) {
             lf_mutex_lock(&mutex);
         }
         if (_lf_sched_advance_tag_locked()) {
-            printf("%ld end", worker);
+            // printf("%ld end", worker);
             should_stop = true;
             worker_states_never_sleep_again(worker);
             worker_states_unlock(worker);

From 32946097d4c77298ee422f5462d41443a9c19fd8 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 30 Mar 2022 15:50:53 -0700
Subject: [PATCH 19/49] [scheduler] Add worker affinity.

Some of our benchmarks need either worker affinity OR use of just one worker, so that a particular reaction is always executed by the same worker. Perhaps worker affinity is easier to add. However, it must be combined with some sort of load-balancing strategy.
---
 core/threaded/worker_assignments.h | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index bb9a39371..a24c6da15 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -25,10 +25,6 @@ static reaction_t*** reactions_by_worker;
 /** The total number of workers active, including those who have finished their work. */
 static size_t num_workers;
 
-// A counter of the number of reactions triggered. No function should depend on the precise
-// correctness of this value. Race conditions when accessing this value are acceptable.
-static size_t reactions_triggered_counter = 0;
-
 #include "data_collection.h"
 
 static void worker_states_lock(size_t worker);
@@ -142,8 +138,8 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
-    // TODO: Hashing by a pointer to the reaction will let us cheaply simulate ``worker affinity''.
-    size_t worker = (reactions_triggered_counter++) % num_workers_by_level[level];
+    // TODO: Implement work stealing, for the following could lead to unfair work distribution.
+    size_t worker = ((size_t) reaction) % num_workers_by_level[level];
     assert(worker >= 0 && worker <= num_workers);
     size_t num_preceding_reactions = lf_atomic_fetch_add(
         &num_reactions_by_worker_by_level[level][worker],

From 4fdcc096407008ad3f1781720280b5617eb7647c Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 30 Mar 2022 16:10:20 -0700
Subject: [PATCH 20/49] [scheduler] Add work stealing.

This is a very crude implementation that should be improved upon.
---
 core/threaded/data_collection.h    |  1 -
 core/threaded/worker_assignments.h | 31 +++++++++++++++++++++---------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 4a5493dbc..b28b6b9fb 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -56,7 +56,6 @@ static void data_collection_end_level(size_t level, size_t num_workers) {
         if (!execution_times_by_num_workers_by_level[level][num_workers]) {
             execution_times_by_num_workers_by_level[level][num_workers]
                 = get_physical_time() - start_times_by_level[level];
-            printf("Initialize.\n");
         } else {
             interval_t prior_et = execution_times_by_num_workers_by_level[level][num_workers];
             execution_times_by_num_workers_by_level[level][num_workers] = (
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index a24c6da15..549f72bee 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -104,6 +104,15 @@ static void worker_assignments_free() {
     data_collection_free();
 }
 
+static reaction_t* get_reaction(size_t worker) {
+    int index = lf_atomic_add_fetch(num_reactions_by_worker + worker, -1);
+    if (index >= 0) {
+        return reactions_by_worker[worker][index];
+    }
+    num_reactions_by_worker[worker] = 0;
+    return NULL;
+}
+
 /**
  * @brief Get a reaction for the given worker to execute. If no such reaction exists, claim the
  * mutex.
@@ -115,17 +124,21 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
     assert(worker >= 0);
     // assert(worker < num_workers);  // There are edge cases where this doesn't hold.
     assert(num_reactions_by_worker[worker] >= 0);
-    if (num_reactions_by_worker[worker]) {
-        reaction_t* ret = reactions_by_worker[worker][--num_reactions_by_worker[worker]];
+    reaction_t* ret;
+    while (true) {
+        if ((ret = get_reaction(worker))) return ret;
+        if (worker < num_workers) {
+            for (size_t victim = (worker + 1) % num_workers; victim != worker; victim = (victim + 1) % num_workers) {
+                if ((ret = get_reaction(victim))) return ret;
+            }
+        }
         // printf("%ld <- %p @ %lld\n", worker, ret, LEVEL(ret->index));
-        return ret;
-    }
-    worker_states_lock(worker);
-    if (!num_reactions_by_worker[worker]) {
-        return NULL;
+        worker_states_lock(worker);
+        if (!num_reactions_by_worker[worker]) {
+            return NULL;
+        }
+        worker_states_unlock(worker);
     }
-    worker_states_unlock(worker);
-    return reactions_by_worker[worker][--num_reactions_by_worker[worker]];
 }
 
 /**

From 5b972f0f74b32ad6891b8a9a9a8e935f58539b69 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 30 Mar 2022 17:34:12 -0700
Subject: [PATCH 21/49] [scheduler] Fix the race condition.*

*Although this fixes a race condition, I have not tested if this fixes "the" race condition. The bug manifested itself so rarely that it was infeasible to reproduce.
---
 core/threaded/scheduler_heuristic.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index f820343bc..5bc772f59 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -98,8 +98,10 @@ void lf_sched_free() {
 reaction_t* lf_sched_get_ready_reaction(int worker_number) {
     assert(worker_number >= 0);
     reaction_t* ret;
-    while (!(ret = worker_assignments_get_or_lock(worker_number))) {
+    while (true) {
         size_t level_counter_snapshot = level_counter;
+        ret = worker_assignments_get_or_lock(worker_number);
+        if (ret) return ret;
         if (worker_states_finished_with_level_locked(worker_number)) {
             // printf("%d !\n", worker_number);
             advance_level_and_unlock(worker_number);

From 7e0e489bf4395268154463d8041a7ca8c0101414 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Thu, 31 Mar 2022 00:36:55 -0700
Subject: [PATCH 22/49] [scheduler] Address a rare concurrency bug in
 SleepingBarber.

Symptom: A thread goes to the event queue to wait for the next event and ends up waiting forever (while the other threads wait for work). This is possible only in SleepingBarber due to use of physical actions -- not in other benchmarks. This bug is not caught by our tests.
---
 core/threaded/scheduler_heuristic.c | 6 ------
 core/threaded/worker_assignments.h  | 1 -
 core/threaded/worker_states.h       | 7 +++++--
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index 5bc772f59..ff72f5154 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -43,8 +43,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define MAX_REACTION_LEVEL INITIAL_REACT_QUEUE_SIZE
 #endif
 
-extern bool fast;
-
 static bool init_called = false;
 static bool should_stop = false;
 
@@ -59,10 +57,6 @@ static bool should_stop = false;
 static void advance_level_and_unlock(size_t worker) {
     // printf("%ld advance %ld\n", worker, current_level);
     if (try_advance_level()) {
-        if (!fast && !mutex_held[worker]) {
-            mutex_held[worker] = true;
-            lf_mutex_lock(&mutex);
-        }
         if (_lf_sched_advance_tag_locked()) {
             // printf("%ld end", worker);
             should_stop = true;
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 549f72bee..1cad942a2 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -153,7 +153,6 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(level < num_levels);
     // TODO: Implement work stealing, for the following could lead to unfair work distribution.
     size_t worker = ((size_t) reaction) % num_workers_by_level[level];
-    assert(worker >= 0 && worker <= num_workers);
     size_t num_preceding_reactions = lf_atomic_fetch_add(
         &num_reactions_by_worker_by_level[level][worker],
         1
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 4e5298a4f..317b67a8e 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -26,6 +26,8 @@ static volatile size_t num_awakened;
 /** Whether the mutex is held by each worker via this module's API. */
 static bool* mutex_held;
 
+extern bool fast;
+
 /**
  * The level counter is a number that changes whenever the current level changes.
  *
@@ -125,7 +127,7 @@ static void worker_states_lock(size_t worker) {
     assert(num_loose_threads <= max_num_workers);
     size_t lt = num_loose_threads;
     // printf("%ld sees %ld loose threads\n", worker, lt);
-    if (lt > 1) {
+    if (lt > 1 || !fast) {  // FIXME: Lock should be partially optimized out even when !fast
         // printf("%ld locking mutex.\n", worker);
         lf_mutex_lock(&mutex);
         // printf("%ld locked mutex.\n", worker);
@@ -158,10 +160,11 @@ static bool worker_states_finished_with_level_locked(size_t worker) {
     assert(worker >= 0);
     assert(num_loose_threads > 0);
     assert(num_reactions_by_worker[worker] != 1);
-    assert(num_reactions_by_worker[worker] == (((size_t) 0) - 1) || num_reactions_by_worker[worker] == 0);
+    assert(((int64_t) num_reactions_by_worker[worker]) <= 0);
     // Why use an atomic operation when we are supposed to be "as good as locked"? Because I took a
     // shortcut, and it wasn't perfect.
     size_t ret = lf_atomic_add_fetch(&num_loose_threads, -1);
+    assert(ret >= 0);
     // printf("worker=%ld, nlt=%ld\n", worker, ret);
     return !ret;
 }

From 59afce7b511a1848b9e5406d653b2d209a820900 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Thu, 31 Mar 2022 16:16:57 -0700
Subject: [PATCH 23/49] [scheduler] Fix another race condition.

The level counter is part of the predicate associated with the condition variable. It is unsafe to change without acquiring a mutex.
---
 core/threaded/worker_states.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 317b67a8e..50b270ba6 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -90,7 +90,6 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     assert(num_to_awaken <= max_num_workers);
     if ((worker == 0) && (num_to_awaken <= 1)) {
         num_loose_threads = 1;
-        level_counter++;
         return;
     }
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;

From 48307021c15b9beb7062654beea8be72772a8836 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Thu, 31 Mar 2022 16:24:03 -0700
Subject: [PATCH 24/49] [scheduler] Adjust data_collection.h.

If I am right, expected trials required to go from an intermediate number of workers to an extreme (1 or the maximum) number of workers was previously quadtratic in # of workers (cuz random walk) -- no good. This should make it (log(n))^2.
---
 core/threaded/data_collection.h | 55 +++++++++++++++++++++++++++------
 1 file changed, 45 insertions(+), 10 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index b28b6b9fb..6816b0c42 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -15,6 +15,11 @@ static interval_t* execution_times_mins;
 static size_t* execution_times_argmins;
 static size_t data_collection_counter = 0;
 static bool collecting_data = false;
+/**
+ * A monotonically increasing sequence of numbers of workers, the first and last elements of which
+ * are too large or small to be realizable.
+ */
+static size_t* possible_nums_workers;
 
 extern size_t num_levels;
 extern size_t max_num_workers;
@@ -23,6 +28,33 @@ extern size_t max_num_workers;
 #define PARALLELISM_COST 20000
 #define EXECUTION_TIME_MEMORY 15
 
+static void possible_nums_workers_init() {
+    // Start with 0 and end with a number strictly greater than max_num_workers.
+    size_t pnw_length = 2;
+    size_t temp = max_num_workers;
+    while ((temp >>= 1)) pnw_length++;
+    possible_nums_workers = malloc(pnw_length * sizeof(size_t));
+    temp = 1;
+    possible_nums_workers[0] = 0;
+    for (int i = 1; i < pnw_length; i++) {
+        possible_nums_workers[i] = temp;
+        temp *= 2;
+    }
+    assert(temp > max_num_workers);
+}
+
+/** Get the result of a state transition. */
+static size_t get_nums_workers_neighboring_state(size_t current_state) {
+    // TODO: There is a more efficient way to do this. However, this operation should be uncommon
+    // asymptotically.
+    size_t range = 1;
+    size_t jitter = ((int) (rand() % (2 * range + 1))) - range;
+    if (!jitter) return current_state;
+    size_t i = 1;
+    while (possible_nums_workers[i] < current_state) i++;
+    return possible_nums_workers[i + jitter];
+}
+
 static void data_collection_init(sched_params_t* params) {
     size_t num_levels = params->num_reactions_per_level_size;
     start_times_by_level = (interval_t*) calloc(num_levels, sizeof(interval_t));
@@ -30,13 +62,14 @@ static void data_collection_init(sched_params_t* params) {
         num_levels, sizeof(interval_t*)
     );
     execution_times_mins = (interval_t*) calloc(num_levels, sizeof(interval_t));
-    execution_times_argmins = (size_t*) malloc(num_levels * sizeof(size_t));
+    execution_times_argmins = (size_t*) calloc(num_levels, sizeof(size_t));
     for (size_t i = 0; i < num_levels; i++) {
         execution_times_argmins[i] = max_num_workers;
         execution_times_by_num_workers_by_level[i] = (interval_t*) calloc(
             max_num_workers, sizeof(interval_t)
         ) - 1;
     }
+    possible_nums_workers_init();
 }
 
 static void data_collection_free() {
@@ -45,6 +78,7 @@ static void data_collection_free() {
         free(execution_times_by_num_workers_by_level[i] + 1);
     }
     free(execution_times_by_num_workers_by_level);
+    free(possible_nums_workers);
 }
 
 static void data_collection_start_level(size_t level) {
@@ -53,16 +87,15 @@ static void data_collection_start_level(size_t level) {
 
 static void data_collection_end_level(size_t level, size_t num_workers) {
     if (collecting_data && start_times_by_level[level]) {
+        interval_t dt = get_physical_time() - start_times_by_level[level];
         if (!execution_times_by_num_workers_by_level[level][num_workers]) {
-            execution_times_by_num_workers_by_level[level][num_workers]
-                = get_physical_time() - start_times_by_level[level];
-        } else {
-            interval_t prior_et = execution_times_by_num_workers_by_level[level][num_workers];
-            execution_times_by_num_workers_by_level[level][num_workers] = (
-                execution_times_by_num_workers_by_level[level][num_workers] * EXECUTION_TIME_MEMORY
-                + get_physical_time() - start_times_by_level[level]
-            ) / (EXECUTION_TIME_MEMORY + 1);
+            execution_times_by_num_workers_by_level[level][num_workers] = MAX(
+                dt,
+                2 * execution_times_by_num_workers_by_level[level][execution_times_argmins[level]]
+            );
         }
+        interval_t* prior_et = &execution_times_by_num_workers_by_level[level][num_workers];
+        *prior_et = (*prior_et * EXECUTION_TIME_MEMORY + dt) / (EXECUTION_TIME_MEMORY + 1);
     }
 }
 
@@ -89,7 +122,9 @@ static void compute_number_of_workers(
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
         ideal_number_of_workers = execution_times_argmins[level];
         int range = 1;
-        if (jitter) ideal_number_of_workers += ((int) (rand() % (2 * range + 1))) - range;
+        if (jitter) {
+            ideal_number_of_workers = get_nums_workers_neighboring_state(ideal_number_of_workers);
+        }
         num_workers_by_level[level] = restrict_to_range(
             1, max_reasonable_num_workers, ideal_number_of_workers
         );

From 17120adabbe424d8aa95998a2302d0f1de9734bc Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Thu, 31 Mar 2022 22:43:23 -0700
Subject: [PATCH 25/49] [scheduler] Bugfix.

This kind of bug reflects lack of care on my part in finding a clean implementation. Some cleanup will be necessary if this is ever to be merged.
---
 core/threaded/data_collection.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 6816b0c42..482eb6142 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -29,8 +29,8 @@ extern size_t max_num_workers;
 #define EXECUTION_TIME_MEMORY 15
 
 static void possible_nums_workers_init() {
-    // Start with 0 and end with a number strictly greater than max_num_workers.
-    size_t pnw_length = 2;
+    // Start with 0 and end with two numbers strictly greater than max_num_workers.
+    size_t pnw_length = 3;
     size_t temp = max_num_workers;
     while ((temp >>= 1)) pnw_length++;
     possible_nums_workers = malloc(pnw_length * sizeof(size_t));

From c6303d10b08b4e4cc632277700ebf69e76d7dae5 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 2 Apr 2022 11:57:42 -0700
Subject: [PATCH 26/49] [scheduler] Fix another race condition.

I did not observe a race condition, but in principle I think there was one. For this reason, this change could not be justified by empirical results.
---
 core/threaded/worker_states.h | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 50b270ba6..f355f1aef 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -94,11 +94,13 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     }
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
     size_t max_cond = cond_of(greatest_worker_number_to_awaken);
-    // printf("%ld +-> %ld\n", worker, num_to_awaken);
+    // printf("%ld +-> %ld @ %ld\n", worker, num_to_awaken, current_level);
     if (!mutex_held[worker]) {
         mutex_held[worker] = true;
         lf_mutex_lock(&mutex);
     }
+    // The predicate of the condition variable depends on num_awakened and level_counter, so
+    // this is a critical section.
     num_loose_threads = cumsum_of_cond_of[max_cond];
     num_loose_threads += worker >= num_loose_threads;
     num_awakened = num_loose_threads;
@@ -187,7 +189,10 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
     mutex_held[worker] = false;  // This will be true soon, upon call to lf_cond_wait.
     // printf("%ld sleep; nlt=%ld\n", worker, num_loose_threads);
     size_t cond = cond_of(worker);
-    if ((level_counter_snapshot == level_counter) & !worker_states_sleep_forbidden) {
+    if (
+        ((level_counter_snapshot == level_counter) || worker >= num_awakened)
+        && !worker_states_sleep_forbidden
+    ) {
         do {
             lf_cond_wait(worker_conds + cond, &mutex);
         } while (level_counter_snapshot == level_counter || worker >= num_awakened);

From 0625c7c92d38c47ecd4331eecd4b1233d6465a4e Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 2 Apr 2022 12:55:40 -0700
Subject: [PATCH 27/49] [scheduler] Minor cleanup.

---
 core/threaded/data_collection.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 482eb6142..6e0f1c192 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -66,8 +66,9 @@ static void data_collection_init(sched_params_t* params) {
     for (size_t i = 0; i < num_levels; i++) {
         execution_times_argmins[i] = max_num_workers;
         execution_times_by_num_workers_by_level[i] = (interval_t*) calloc(
-            max_num_workers, sizeof(interval_t)
-        ) - 1;
+            max_num_workers + 1,  // Add 1 for 1-based indexing
+            sizeof(interval_t)
+        );
     }
     possible_nums_workers_init();
 }
@@ -75,7 +76,7 @@ static void data_collection_init(sched_params_t* params) {
 static void data_collection_free() {
     free(start_times_by_level);
     for (size_t i = 0; i < num_levels; i++) {
-        free(execution_times_by_num_workers_by_level[i] + 1);
+        free(execution_times_by_num_workers_by_level[i]);
     }
     free(execution_times_by_num_workers_by_level);
     free(possible_nums_workers);

From 3be18ed1fa9d1b0625cfb991fc48048e88d502bb Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 2 Apr 2022 12:57:15 -0700
Subject: [PATCH 28/49] [scheduler] Check number of workers required more
 dynamically.

Again, this is another optimization borrowed from the NP scheduler. The difference is that the NP scheduler is designed such that this optimization needs no explicit code to handle it; there, it just works.
---
 core/threaded/scheduler_heuristic.c | 29 ++++++++++++++++++++---------
 core/threaded/worker_assignments.h  | 23 +++++++----------------
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index ff72f5154..2d1524a65 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -56,19 +56,30 @@ static bool should_stop = false;
  */
 static void advance_level_and_unlock(size_t worker) {
     // printf("%ld advance %ld\n", worker, current_level);
-    if (try_advance_level()) {
-        if (_lf_sched_advance_tag_locked()) {
-            // printf("%ld end", worker);
-            should_stop = true;
-            worker_states_never_sleep_again(worker);
+    size_t max_level = num_levels - 1;
+    while (true) {
+        if (current_level == max_level) {
+            data_collection_end_tag(num_workers_by_level, max_num_workers_by_level);
+            set_level(0);
+            if (_lf_sched_advance_tag_locked()) {
+                // printf("%ld end", worker);
+                should_stop = true;
+                worker_states_never_sleep_again(worker);
+                worker_states_unlock(worker);
+                return;
+            }
+        } else {
+            set_level(current_level + 1);
+        }
+        size_t total_num_reactions = get_num_reactions();
+        if (total_num_reactions) {
+            size_t num_workers_to_awaken = MIN(total_num_reactions, num_workers);
+            assert(num_workers_to_awaken > 0);
+            worker_states_awaken_locked(worker, num_workers_to_awaken);
             worker_states_unlock(worker);
             return;
         }
     }
-    size_t num_workers_to_awaken = num_workers;
-    assert(num_workers_to_awaken > 0);
-    worker_states_awaken_locked(worker, num_workers_to_awaken);
-    worker_states_unlock(worker);
 }
 
 ///////////////////// Scheduler Init and Destroy API /////////////////////////
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 1cad942a2..07d998ba0 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -46,23 +46,14 @@ static void set_level(size_t level) {
     num_workers = num_workers_by_level[level];
     data_collection_start_level(current_level);
 }
-
-/**
- * @brief Advance the level currently being processed by the workers.
- * 
- * @return true If the level was already at the maximum and was reset to zero.
- */
-static bool try_advance_level() {
-    size_t max_level = num_levels - 1;
-    while (current_level < max_level) {
-        set_level(current_level + 1);
-        for (size_t i = 0; i < num_workers; i++) {
-            if (num_reactions_by_worker[i]) return false;
-        }
+/** Return the total number of reactions enqueued on the current level. */
+static size_t get_num_reactions() {
+    size_t total_num_reactions = 0;
+    for (size_t i = 0; i < num_workers; i++) {
+        total_num_reactions += num_reactions_by_worker[i];
     }
-    data_collection_end_tag(num_workers_by_level, max_num_workers_by_level);
-    set_level(0);
-    return true;
+    // TODO: if num_workers was > total_num_reactions, report this to data_collection
+    return total_num_reactions;
 }
 
 static void worker_assignments_init(size_t number_of_workers, sched_params_t* params) {

From e0adf738c1320c99d9a7ce02dfdb8903a3b5b897 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 2 Apr 2022 14:12:03 -0700
Subject: [PATCH 29/49] [scheduler] Let there be no hypergalactic state
 transitions.

---
 core/threaded/data_collection.h | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 6e0f1c192..f285f1b3d 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -25,7 +25,6 @@ extern size_t num_levels;
 extern size_t max_num_workers;
 
 #define SLOW_EXPERIMENTS 256
-#define PARALLELISM_COST 20000
 #define EXECUTION_TIME_MEMORY 15
 
 static void possible_nums_workers_init() {
@@ -43,15 +42,27 @@ static void possible_nums_workers_init() {
     assert(temp > max_num_workers);
 }
 
+static int get_jitter(size_t current_state, interval_t execution_time) {
+    static const size_t parallelism_cost_max = 131072;
+    // The following handles the case where the current level really is just fluff:
+    // No parallelism needed, no work to be done.
+    if (execution_time < 16384 && current_state == 1) return 0;
+    int left_score = 65536;
+    int middle_score = 65536;
+    int right_score = 65536;
+    if (execution_time < parallelism_cost_max) left_score += parallelism_cost_max - execution_time;
+    int result = rand() % (left_score + middle_score + right_score);
+    if (result < left_score) return -1;
+    if (result < left_score + middle_score) return 0;
+    return 1;
+}
+
 /** Get the result of a state transition. */
-static size_t get_nums_workers_neighboring_state(size_t current_state) {
-    // TODO: There is a more efficient way to do this. However, this operation should be uncommon
-    // asymptotically.
-    size_t range = 1;
-    size_t jitter = ((int) (rand() % (2 * range + 1))) - range;
+static size_t get_nums_workers_neighboring_state(size_t current_state, interval_t execution_time) {
+    size_t jitter = get_jitter(current_state, execution_time);
     if (!jitter) return current_state;
     size_t i = 1;
-    while (possible_nums_workers[i] < current_state) i++;
+    while (possible_nums_workers[i] < current_state) i++; // TODO: There are more efficient ways to do this.
     return possible_nums_workers[i + jitter];
 }
 
@@ -115,16 +126,13 @@ static void compute_number_of_workers(
         interval_t this_execution_time = execution_times_by_num_workers_by_level[level][
             num_workers_by_level[level]
         ];
-        if (0 < this_execution_time && this_execution_time < PARALLELISM_COST && (rand() & 1)) {
-            num_workers_by_level[level] = 1;
-            continue;
-        }
         size_t ideal_number_of_workers;
         size_t max_reasonable_num_workers = max_num_workers_by_level[level];
         ideal_number_of_workers = execution_times_argmins[level];
         int range = 1;
         if (jitter) {
-            ideal_number_of_workers = get_nums_workers_neighboring_state(ideal_number_of_workers);
+            ideal_number_of_workers = get_nums_workers_neighboring_state(ideal_number_of_workers, this_execution_time);
+            // printf("%ld -> %ld @ %ld\n", execution_times_argmins[level], ideal_number_of_workers, level);
         }
         num_workers_by_level[level] = restrict_to_range(
             1, max_reasonable_num_workers, ideal_number_of_workers
@@ -139,7 +147,6 @@ static void compute_costs(size_t* num_workers_by_level) {
         interval_t score = execution_times_by_num_workers_by_level[level][
             num_workers_by_level[level]
         ];
-        if (num_workers_by_level[level] > 1) score += PARALLELISM_COST;
         if (
             !execution_times_mins[level]
             | (score < execution_times_mins[level])

From 0aa11048ed7df50fd1e86fcf21f42ee1215b901a Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 2 Apr 2022 18:52:26 -0700
Subject: [PATCH 30/49] [scheduler] More relatively minor tweaks.

I run the risk of overfitting by doing this. However, it makes sense not to do a state change before collecting any information about the initial state -- the initial state was chosen for a reason. The decision to make left/right transition probabilities more even was not really based on empirical results as much as a vague sort of common sense.
---
 core/threaded/data_collection.h | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index f285f1b3d..e69b15949 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -24,6 +24,7 @@ static size_t* possible_nums_workers;
 extern size_t num_levels;
 extern size_t max_num_workers;
 
+#define START_EXPERIMENTS 8
 #define SLOW_EXPERIMENTS 256
 #define EXECUTION_TIME_MEMORY 15
 
@@ -32,7 +33,7 @@ static void possible_nums_workers_init() {
     size_t pnw_length = 3;
     size_t temp = max_num_workers;
     while ((temp >>= 1)) pnw_length++;
-    possible_nums_workers = malloc(pnw_length * sizeof(size_t));
+    possible_nums_workers = (size_t*) malloc(pnw_length * sizeof(size_t));
     temp = 1;
     possible_nums_workers[0] = 0;
     for (int i = 1; i < pnw_length; i++) {
@@ -43,11 +44,11 @@ static void possible_nums_workers_init() {
 }
 
 static int get_jitter(size_t current_state, interval_t execution_time) {
-    static const size_t parallelism_cost_max = 131072;
+    static const size_t parallelism_cost_max = 114688;
     // The following handles the case where the current level really is just fluff:
     // No parallelism needed, no work to be done.
     if (execution_time < 16384 && current_state == 1) return 0;
-    int left_score = 65536;
+    int left_score = 16384;  // Want: For execution time = 65536, p(try left) = p(try right)
     int middle_score = 65536;
     int right_score = 65536;
     if (execution_time < parallelism_cost_max) left_score += parallelism_cost_max - execution_time;
@@ -175,7 +176,11 @@ static void data_collection_end_tag(
     size_t period = 2 + 128 * (data_collection_counter > SLOW_EXPERIMENTS);
     size_t state = data_collection_counter % period;
     if (state == 0) {
-        compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, true);
+        compute_number_of_workers(
+            num_workers_by_level,
+            max_num_workers_by_level,
+            data_collection_counter > START_EXPERIMENTS
+        );
         collecting_data = true;
     } else if (state == 1) {
         compute_number_of_workers(num_workers_by_level, max_num_workers_by_level, false);

From c8a342550bcfcd3b1bcdae4e7435b944cba905ef Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 6 Apr 2022 11:36:53 -0700
Subject: [PATCH 31/49] [scheduler] Bugfix.

---
 core/threaded/data_collection.h    | 2 +-
 core/threaded/worker_assignments.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index e69b15949..ee0e70bb6 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -30,7 +30,7 @@ extern size_t max_num_workers;
 
 static void possible_nums_workers_init() {
     // Start with 0 and end with two numbers strictly greater than max_num_workers.
-    size_t pnw_length = 3;
+    size_t pnw_length = 4;
     size_t temp = max_num_workers;
     while ((temp >>= 1)) pnw_length++;
     possible_nums_workers = (size_t*) malloc(pnw_length * sizeof(size_t));
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 07d998ba0..a4ae37474 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -142,7 +142,7 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
-    // TODO: Implement work stealing, for the following could lead to unfair work distribution.
+    // TODO: Find some robust way to compute a hash from the reaction. This method here is wrong.
     size_t worker = ((size_t) reaction) % num_workers_by_level[level];
     size_t num_preceding_reactions = lf_atomic_fetch_add(
         &num_reactions_by_worker_by_level[level][worker],

From 606cab93506a2161765017fc9a16490f7184fa3a Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 13 Apr 2022 10:43:41 -0700
Subject: [PATCH 32/49] Compute a hash to assign a reaction to a worker.

---
 core/threaded/worker_assignments.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index a4ae37474..54de92f94 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -142,8 +142,12 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(reaction != NULL);
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
-    // TODO: Find some robust way to compute a hash from the reaction. This method here is wrong.
-    size_t worker = ((size_t) reaction) % num_workers_by_level[level];
+    // Source: https://xorshift.di.unimi.it/splitmix64.c
+    uint64_t hash = (uint64_t) reaction;
+    hash = (hash ^ (hash >> 30)) * 0xbf58476d1ce4e5b9;
+    hash = (hash ^ (hash >> 27)) * 0x94d049bb133111eb;
+    hash = hash ^ (hash >> 31);
+    size_t worker = hash % num_workers_by_level[level];
     size_t num_preceding_reactions = lf_atomic_fetch_add(
         &num_reactions_by_worker_by_level[level][worker],
         1

From a5b8780da6df6e4905c7ed5dc2e18e39c8ecd9da Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 09:19:55 -0700
Subject: [PATCH 33/49] [scheduler] Update CI.

---
 .github/workflows/ci.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 871dc1b9f..5f3f9578d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,7 +23,7 @@ jobs:
     uses: lf-lang/lingua-franca/.github/workflows/extract-ref.yml@master
     with:
       file: 'lingua-franca-ref.txt'
-  
+
   lf-default:
     needs: fetch-lf
     uses: lf-lang/lingua-franca/.github/workflows/c-tests.yml@master
@@ -38,7 +38,7 @@ jobs:
       runtime-ref: ${{ github.ref }}
       compiler-ref: ${{ needs.fetch-lf.outputs.ref }}
       scheduler: GEDF_NP
-      
+
   lf-gedf-np-ci:
     needs: fetch-lf
     uses: lf-lang/lingua-franca/.github/workflows/c-tests.yml@master
@@ -46,4 +46,11 @@ jobs:
       runtime-ref: ${{ github.ref }}
       compiler-ref: ${{ needs.fetch-lf.outputs.ref }}
       scheduler: GEDF_NP_CI
-  
+
+  lf-heuristic:
+    needs: fetch-lf
+    uses: lf-lang/lingua-franca/.github/workflows/c-tests.yml@master
+    with:
+      runtime-ref: ${{ github.ref }}
+      compiler-ref: ${{ needs.fetch-lf.outputs.ref }}
+      scheduler: heuristic

From ae3444c1c631735c4266769332c172b9584a7bac Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 10:07:13 -0700
Subject: [PATCH 34/49] [scheduler] Update lingua-franca-ref.txt.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 88ab221f4..5550ed4b0 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-b50058ca475f1cd24b064ae2c9a278c9a958df51
+11c7278bc21e9411203f57e5ed7c4f7e48ec8b00

From 80a90f0323b739cc58bd542a36eb49f57c6ebf37 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 14:57:16 -0700
Subject: [PATCH 35/49] [scheduler] Comments and superficial style changes.

---
 core/threaded/data_collection.h     | 85 ++++++++++++++++++++++++-----
 core/threaded/scheduler_heuristic.c |  7 +--
 core/threaded/worker_assignments.h  | 50 ++++++++++++++---
 core/threaded/worker_states.h       | 50 +++++++++++------
 4 files changed, 146 insertions(+), 46 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index ee0e70bb6..e617f62bd 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -1,3 +1,31 @@
+/*************
+Copyright (c) 2022, The University of California at Berkeley.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+***************/
+
+/**
+ * Scheduling-related data collection and analysis that is performed at run-time.
+ * @author{Peter Donovan <peterdonovan@berkeley.edu>}
+ */
 
 #ifndef DATA_COLLECTION
 #define DATA_COLLECTION
@@ -15,21 +43,26 @@ static interval_t* execution_times_mins;
 static size_t* execution_times_argmins;
 static size_t data_collection_counter = 0;
 static bool collecting_data = false;
+
 /**
  * A monotonically increasing sequence of numbers of workers, the first and last elements of which
- * are too large or small to be realizable.
+ * are too large or small to represent valid states of the system (i.e., state transitions to them
+ * are instantaneously reflected).
  */
 static size_t* possible_nums_workers;
 
 extern size_t num_levels;
 extern size_t max_num_workers;
+instant_t lf_time_physical(void);
 
 #define START_EXPERIMENTS 8
 #define SLOW_EXPERIMENTS 256
 #define EXECUTION_TIME_MEMORY 15
 
+/** @brief Initialize the possible_nums_workers array. */
 static void possible_nums_workers_init() {
-    // Start with 0 and end with two numbers strictly greater than max_num_workers.
+    // Start with 0 and end with two numbers strictly greater than max_num_workers. This must start
+    // at 4 because the first two and last two entries are not counted.
     size_t pnw_length = 4;
     size_t temp = max_num_workers;
     while ((temp >>= 1)) pnw_length++;
@@ -43,6 +76,14 @@ static void possible_nums_workers_init() {
     assert(temp > max_num_workers);
 }
 
+/**
+ * @brief Return a random integer in the interval [-1, +1] representing whether the number of
+ * workers used on a certain level should increase, decrease, or remain the same, with a probability
+ * distribution possibly dependent on the parameters.
+ * @param current_state The index currently used by this level in the possible_nums_workers array.
+ * @param execution_time An estimate of the execution time of the level in the case for which we
+ * would like to optimize.
+ */
 static int get_jitter(size_t current_state, interval_t execution_time) {
     static const size_t parallelism_cost_max = 114688;
     // The following handles the case where the current level really is just fluff:
@@ -58,7 +99,7 @@ static int get_jitter(size_t current_state, interval_t execution_time) {
     return 1;
 }
 
-/** Get the result of a state transition. */
+/** @brief Get the number of workers resulting from a random state transition. */
 static size_t get_nums_workers_neighboring_state(size_t current_state, interval_t execution_time) {
     size_t jitter = get_jitter(current_state, execution_time);
     if (!jitter) return current_state;
@@ -94,13 +135,15 @@ static void data_collection_free() {
     free(possible_nums_workers);
 }
 
+/** @brief Record that the execution of the given level is beginning. */
 static void data_collection_start_level(size_t level) {
-    if (collecting_data) start_times_by_level[level] = get_physical_time();
+    if (collecting_data) start_times_by_level[level] = lf_time_physical();
 }
 
+/** @brief Record that the execution of the given level has completed. */
 static void data_collection_end_level(size_t level, size_t num_workers) {
     if (collecting_data && start_times_by_level[level]) {
-        interval_t dt = get_physical_time() - start_times_by_level[level];
+        interval_t dt = lf_time_physical() - start_times_by_level[level];
         if (!execution_times_by_num_workers_by_level[level][num_workers]) {
             execution_times_by_num_workers_by_level[level][num_workers] = MAX(
                 dt,
@@ -118,6 +161,14 @@ static size_t restrict_to_range(size_t start_inclusive, size_t end_inclusive, si
     return value;
 }
 
+/**
+ * @brief Update num_workers_by_level in-place.
+ * @param num_workers_by_level The number of workers that should be used to execute each level.
+ * @param max_num_workers_by_level The maximum possible number of workers that could reasonably be
+ * assigned to each level.
+ * @param jitter Whether the possibility of state transitions to numbers of workers that are not
+ * (yet) empirically optimal is desired.
+ */
 static void compute_number_of_workers(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level,
@@ -132,17 +183,21 @@ static void compute_number_of_workers(
         ideal_number_of_workers = execution_times_argmins[level];
         int range = 1;
         if (jitter) {
-            ideal_number_of_workers = get_nums_workers_neighboring_state(ideal_number_of_workers, this_execution_time);
-            // printf("%ld -> %ld @ %ld\n", execution_times_argmins[level], ideal_number_of_workers, level);
+            ideal_number_of_workers = get_nums_workers_neighboring_state(
+                ideal_number_of_workers, this_execution_time
+            );
         }
         num_workers_by_level[level] = restrict_to_range(
             1, max_reasonable_num_workers, ideal_number_of_workers
         );
-        // printf("level=%ld, jitter=%d, inow=%ld, mrnw=%ld, result=%ld\n",
-        // level, jitter, ideal_number_of_workers, max_reasonable_num_workers, num_workers_by_level[level]);
     }
 }
 
+/**
+ * @brief Update minimum and argmin (wrt number of workers used) execution times according the most
+ * recent execution times recorded.
+ * @param num_workers_by_level The number of workers most recently used to execute each level.
+ */
 static void compute_costs(size_t* num_workers_by_level) {
     for (size_t level = 0; level < num_levels; level++) {
         interval_t score = execution_times_by_num_workers_by_level[level][
@@ -153,18 +208,18 @@ static void compute_costs(size_t* num_workers_by_level) {
             | (score < execution_times_mins[level])
             | (num_workers_by_level[level] == execution_times_argmins[level])
         ) {
-            if (num_workers_by_level[level] != execution_times_argmins[level]) printf(
-                "Argmin update: %ld(%ld) -> %ld(%ld) @ %ld\n",
-                execution_times_argmins[level], execution_times_mins[level],
-                num_workers_by_level[level], score,
-                level
-            );
             execution_times_mins[level] = score;
             execution_times_argmins[level] = num_workers_by_level[level];
         }
     }
 }
 
+/**
+ * @brief Record that the execution of a tag has completed.
+ * @param num_workers_by_level The number of workers used to execute each level of the tag.
+ * @param max_num_workers_by_level The maximum number of workers that could reasonably be used to
+ * execute each level, for any tag.
+ */
 static void data_collection_end_tag(
     size_t* num_workers_by_level,
     size_t* max_num_workers_by_level
diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_heuristic.c
index 2d1524a65..dd998e538 100644
--- a/core/threaded/scheduler_heuristic.c
+++ b/core/threaded/scheduler_heuristic.c
@@ -49,20 +49,16 @@ static bool should_stop = false;
 ///////////////////////// Scheduler Private Functions ///////////////////////////
 
 /**
- * @brief Increment the level currently being executed, and the tag if need be.
- * 
- * Sleep thereafter if that is what the current worker ought to do.
+ * @brief Increment the level currently being executed, and the tag if necessary.
  * @param worker The number of the calling worker.
  */
 static void advance_level_and_unlock(size_t worker) {
-    // printf("%ld advance %ld\n", worker, current_level);
     size_t max_level = num_levels - 1;
     while (true) {
         if (current_level == max_level) {
             data_collection_end_tag(num_workers_by_level, max_num_workers_by_level);
             set_level(0);
             if (_lf_sched_advance_tag_locked()) {
-                // printf("%ld end", worker);
                 should_stop = true;
                 worker_states_never_sleep_again(worker);
                 worker_states_unlock(worker);
@@ -108,7 +104,6 @@ reaction_t* lf_sched_get_ready_reaction(int worker_number) {
         ret = worker_assignments_get_or_lock(worker_number);
         if (ret) return ret;
         if (worker_states_finished_with_level_locked(worker_number)) {
-            // printf("%d !\n", worker_number);
             advance_level_and_unlock(worker_number);
         } else {
             worker_states_sleep_and_unlock(worker_number, level_counter_snapshot);
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 54de92f94..22903a97f 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -1,3 +1,31 @@
+/*************
+Copyright (c) 2022, The University of California at Berkeley.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+***************/
+
+/**
+ * Assign reactions to workers.
+ * @author{Peter Donovan <peterdonovan@berkeley.edu>}
+ */
 
 #ifndef WORKER_ASSIGNMENTS
 #define WORKER_ASSIGNMENTS
@@ -33,7 +61,6 @@ static void worker_states_unlock(size_t worker);
 /**
  * @brief Set the level to be executed now. This function assumes that concurrent calls to it are
  * impossible.
- * 
  * @param level The new current level.
  */
 static void set_level(size_t level) {
@@ -44,15 +71,20 @@ static void set_level(size_t level) {
     num_reactions_by_worker = num_reactions_by_worker_by_level[level];
     reactions_by_worker = reactions_by_worker_by_level[level];
     num_workers = num_workers_by_level[level];
+    // TODO: Experiment with not recording that the level is starting in the case that there is
+    // nothing to execute. We need not optimize for the case when there is nothing to execute
+    // because that case is not merely optimized, but is optimized out (we do not bother with
+    // executing nothing).
     data_collection_start_level(current_level);
 }
-/** Return the total number of reactions enqueued on the current level. */
+
+/** @brief Return the total number of reactions enqueued on the current level. */
 static size_t get_num_reactions() {
     size_t total_num_reactions = 0;
     for (size_t i = 0; i < num_workers; i++) {
         total_num_reactions += num_reactions_by_worker[i];
     }
-    // TODO: if num_workers was > total_num_reactions, report this to data_collection
+    // TODO: if num_workers was > total_num_reactions, report this to data_collection?
     return total_num_reactions;
 }
 
@@ -95,6 +127,11 @@ static void worker_assignments_free() {
     data_collection_free();
 }
 
+/**
+ * @brief Return a reaction that has been assigned to the given worker, or NULL if no such reaction
+ * exists.
+ * @param worker The number of a worker needing work.
+ */
 static reaction_t* get_reaction(size_t worker) {
     int index = lf_atomic_add_fetch(num_reactions_by_worker + worker, -1);
     if (index >= 0) {
@@ -107,7 +144,6 @@ static reaction_t* get_reaction(size_t worker) {
 /**
  * @brief Get a reaction for the given worker to execute. If no such reaction exists, claim the
  * mutex.
- *
  * @param worker A worker requesting work.
  * @return reaction_t* A reaction to execute, or NULL if no such reaction exists.
  */
@@ -123,7 +159,6 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
                 if ((ret = get_reaction(victim))) return ret;
             }
         }
-        // printf("%ld <- %p @ %lld\n", worker, ret, LEVEL(ret->index));
         worker_states_lock(worker);
         if (!num_reactions_by_worker[worker]) {
             return NULL;
@@ -134,7 +169,6 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
 
 /**
  * @brief Trigger the given reaction.
- * 
  * @param reaction A reaction to be executed in the current tag.
  */
 static void worker_assignments_put(reaction_t* reaction) {
@@ -143,6 +177,9 @@ static void worker_assignments_put(reaction_t* reaction) {
     assert(level > current_level || current_level == 0);
     assert(level < num_levels);
     // Source: https://xorshift.di.unimi.it/splitmix64.c
+    // FIXME: This is probably not the most efficient way to get the randomness that we need because
+    // it is designed to give an entire word of randomness, whereas we only need
+    // ~log2(num_workers_by_level[level]) bits of randomness.
     uint64_t hash = (uint64_t) reaction;
     hash = (hash ^ (hash >> 30)) * 0xbf58476d1ce4e5b9;
     hash = (hash ^ (hash >> 27)) * 0x94d049bb133111eb;
@@ -152,7 +189,6 @@ static void worker_assignments_put(reaction_t* reaction) {
         &num_reactions_by_worker_by_level[level][worker],
         1
     );
-    // printf("%p -> %ld @ %ld[%ld]\n", reaction, worker, level, num_preceding_reactions);
     reactions_by_worker_by_level[level][worker][num_preceding_reactions] = reaction;
 }
 
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index f355f1aef..b7a66e325 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -1,3 +1,31 @@
+/*************
+Copyright (c) 2022, The University of California at Berkeley.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+***************/
+
+/**
+ * Management of worker wakefulness and locking.
+ * @author{Peter Donovan <peterdonovan@berkeley.edu>}
+ */
 
 #ifndef WORKER_STATES
 #define WORKER_STATES
@@ -94,7 +122,6 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     }
     size_t greatest_worker_number_to_awaken = num_to_awaken - 1;
     size_t max_cond = cond_of(greatest_worker_number_to_awaken);
-    // printf("%ld +-> %ld @ %ld\n", worker, num_to_awaken, current_level);
     if (!mutex_held[worker]) {
         mutex_held[worker] = true;
         lf_mutex_lock(&mutex);
@@ -112,7 +139,6 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
 
 /**
  * @brief Wake up all workers and forbid them from ever sleeping again.
- * 
  * This is intended to coordinate shutdown (without leaving any dangling threads behind).
  */
 static void worker_states_never_sleep_again(size_t worker) {
@@ -127,27 +153,18 @@ static void worker_states_lock(size_t worker) {
     assert(num_loose_threads > 0);
     assert(num_loose_threads <= max_num_workers);
     size_t lt = num_loose_threads;
-    // printf("%ld sees %ld loose threads\n", worker, lt);
     if (lt > 1 || !fast) {  // FIXME: Lock should be partially optimized out even when !fast
-        // printf("%ld locking mutex.\n", worker);
         lf_mutex_lock(&mutex);
-        // printf("%ld locked mutex.\n", worker);
         assert(mutex_held[worker] == false);
         mutex_held[worker] = true;
-    } else {
-        // printf("%ld not locking mutex.\n", worker);
     }
 }
 
 /** Unlock the global mutex if needed. */
 static void worker_states_unlock(size_t worker) {
-    if (mutex_held[worker]) {
-        // printf("%ld unlocking mutex.\n", worker);
-        mutex_held[worker] = false;
-        lf_mutex_unlock(&mutex);
-    } else {
-        // printf("%ld not unlocking mutex.\n", worker);
-    }
+    if (!mutex_held[worker]) return;
+    mutex_held[worker] = false;
+    lf_mutex_unlock(&mutex);
 }
 
 /**
@@ -163,10 +180,9 @@ static bool worker_states_finished_with_level_locked(size_t worker) {
     assert(num_reactions_by_worker[worker] != 1);
     assert(((int64_t) num_reactions_by_worker[worker]) <= 0);
     // Why use an atomic operation when we are supposed to be "as good as locked"? Because I took a
-    // shortcut, and it wasn't perfect.
+    // shortcut, and the shortcut was imperfect.
     size_t ret = lf_atomic_add_fetch(&num_loose_threads, -1);
     assert(ret >= 0);
-    // printf("worker=%ld, nlt=%ld\n", worker, ret);
     return !ret;
 }
 
@@ -187,7 +203,6 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
         lf_mutex_lock(&mutex);
     }
     mutex_held[worker] = false;  // This will be true soon, upon call to lf_cond_wait.
-    // printf("%ld sleep; nlt=%ld\n", worker, num_loose_threads);
     size_t cond = cond_of(worker);
     if (
         ((level_counter_snapshot == level_counter) || worker >= num_awakened)
@@ -197,7 +212,6 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
             lf_cond_wait(worker_conds + cond, &mutex);
         } while (level_counter_snapshot == level_counter || worker >= num_awakened);
     }
-    // printf("%ld wake; nlt=%ld\n", worker, num_loose_threads);
     assert(mutex_held[worker] == false);  // This thread holds the mutex, but it did not report that.
     lf_mutex_unlock(&mutex);
 }

From c020c6e604c597fda7d70f6070d7a836ce25ac7e Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 17:32:59 -0700
Subject: [PATCH 36/49] Update lingua-franca-ref.txt.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 5550ed4b0..f0a7efbcb 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-11c7278bc21e9411203f57e5ed7c4f7e48ec8b00
+8575800cd7dc07a9c0a26da8bcdf1041b45daf53

From 85e45fbd3b967e7fe8e15b86823f906ae54e9c45 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 22:02:53 -0700
Subject: [PATCH 37/49] Update lingua-franca-ref.txt.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index f0a7efbcb..9a51102c7 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-8575800cd7dc07a9c0a26da8bcdf1041b45daf53
+45b22645e3b57978ca57cd2e138d96b801c0b11f

From c8d697a96471d2f3f1ed292febc617f653e3c54c Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 22:10:41 -0700
Subject: [PATCH 38/49] [scheduler] Rename "heuristic" -> "adaptive"

---
 .github/workflows/ci.yml                                      | 4 ++--
 core/threaded/{scheduler_heuristic.c => scheduler_adaptive.c} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename core/threaded/{scheduler_heuristic.c => scheduler_adaptive.c} (100%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5f3f9578d..88659485a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,10 +47,10 @@ jobs:
       compiler-ref: ${{ needs.fetch-lf.outputs.ref }}
       scheduler: GEDF_NP_CI
 
-  lf-heuristic:
+  lf-adaptive:
     needs: fetch-lf
     uses: lf-lang/lingua-franca/.github/workflows/c-tests.yml@master
     with:
       runtime-ref: ${{ github.ref }}
       compiler-ref: ${{ needs.fetch-lf.outputs.ref }}
-      scheduler: heuristic
+      scheduler: adaptive
diff --git a/core/threaded/scheduler_heuristic.c b/core/threaded/scheduler_adaptive.c
similarity index 100%
rename from core/threaded/scheduler_heuristic.c
rename to core/threaded/scheduler_adaptive.c

From 701cfad1123cb694c2d0c83df0a20a9452a0de4e Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Tue, 31 May 2022 22:14:55 -0700
Subject: [PATCH 39/49] Update lingua-franca-ref.txt.

---
 lingua-franca-ref.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 9a51102c7..68523dc4b 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-45b22645e3b57978ca57cd2e138d96b801c0b11f
+b6c0971bd810b48f1a3e816beaf9ce5d86eb21ea

From 29cf529b99dfaa51d367f25cf7f9574a4f691c76 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 1 Jun 2022 14:22:26 -0700
Subject: [PATCH 40/49] [scheduler] Remove a special case.

---
 core/threaded/scheduler_adaptive.c |  2 +-
 core/threaded/worker_states.h      | 14 --------------
 2 files changed, 1 insertion(+), 15 deletions(-)

diff --git a/core/threaded/scheduler_adaptive.c b/core/threaded/scheduler_adaptive.c
index dd998e538..fdf069861 100644
--- a/core/threaded/scheduler_adaptive.c
+++ b/core/threaded/scheduler_adaptive.c
@@ -60,7 +60,7 @@ static void advance_level_and_unlock(size_t worker) {
             set_level(0);
             if (_lf_sched_advance_tag_locked()) {
                 should_stop = true;
-                worker_states_never_sleep_again(worker);
+                worker_states_awaken_locked(worker, max_num_workers);
                 worker_states_unlock(worker);
                 return;
             }
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index b7a66e325..5a4955175 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -41,8 +41,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 static lf_cond_t* worker_conds;
 static size_t* cumsum_of_cond_of;
 
-static bool worker_states_sleep_forbidden = false;
-
 extern size_t current_level;
 extern size_t** num_reactions_by_worker_by_level;
 extern size_t max_num_workers;
@@ -137,17 +135,6 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     }
 }
 
-/**
- * @brief Wake up all workers and forbid them from ever sleeping again.
- * This is intended to coordinate shutdown (without leaving any dangling threads behind).
- */
-static void worker_states_never_sleep_again(size_t worker) {
-    worker_states_sleep_forbidden = true;
-    lf_mutex_lock(&mutex);
-    worker_states_awaken_locked(worker, max_num_workers);
-    lf_mutex_unlock(&mutex);
-}
-
 /** Lock the global mutex if needed. */
 static void worker_states_lock(size_t worker) {
     assert(num_loose_threads > 0);
@@ -206,7 +193,6 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
     size_t cond = cond_of(worker);
     if (
         ((level_counter_snapshot == level_counter) || worker >= num_awakened)
-        && !worker_states_sleep_forbidden
     ) {
         do {
             lf_cond_wait(worker_conds + cond, &mutex);

From eb202c9ac6d307dfffcd24baf34e29931c143bb3 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 1 Jun 2022 14:51:09 -0700
Subject: [PATCH 41/49] [scheduler] Superficial commenting/renaming.

---
 core/threaded/worker_assignments.h |  6 ++++++
 core/threaded/worker_states.h      | 28 +++++++++++++++++-----------
 lingua-franca-ref.txt              |  2 +-
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 22903a97f..a27d6fc67 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -37,11 +37,17 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <assert.h>
 #include "scheduler.h"
 
+/** The queued reactions. */
 static reaction_t**** reactions_by_worker_by_level;
+/** The number of queued reactions currently assigned to each worker at each level. */
 static size_t** num_reactions_by_worker_by_level;
+/** The maximum number of workers that could possibly be kept simultaneously busy at each level. */
 static size_t* max_num_workers_by_level;
+/** The number of workers that will be used to execute each level. */
 static size_t* num_workers_by_level;
+/** The number of levels. */
 static size_t num_levels;
+/** The maximum number of workers that can be used to execute any level. */
 static size_t max_num_workers;
 
 /** The following values apply to the current level. */
diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 5a4955175..cafd27878 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -38,20 +38,26 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "scheduler.h"
 #include "../platform.h"
 
+/** An array of condition variables, each corresponding to a group of workers. */
 static lf_cond_t* worker_conds;
-static size_t* cumsum_of_cond_of;
-
-extern size_t current_level;
-extern size_t** num_reactions_by_worker_by_level;
-extern size_t max_num_workers;
-extern lf_mutex_t mutex;
-
+/** The cumsum of the sizes of the groups of workers corresponding to each successive cond. */
+static size_t* cumsum_of_worker_group_sizes;
 /** The number of non-waiting threads. */
 static volatile size_t num_loose_threads;
+/** The number of threads that were awakened for the purpose of executing the current level. */
 static volatile size_t num_awakened;
 /** Whether the mutex is held by each worker via this module's API. */
 static bool* mutex_held;
 
+/** See worker_assignments.h for documentation. */
+extern size_t current_level;
+extern size_t** num_reactions_by_worker_by_level;
+extern size_t max_num_workers;
+
+/** See reactor_threaded.c for documentation. */
+extern lf_mutex_t mutex;
+
+/** See reactor_common.c for documentation. */
 extern bool fast;
 
 /**
@@ -85,13 +91,13 @@ static void worker_states_init(size_t number_of_workers) {
     size_t greatest_worker_number = number_of_workers - 1;
     size_t num_conds = cond_of(greatest_worker_number) + 1;
     worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
-    cumsum_of_cond_of = (size_t*) calloc(num_conds, sizeof(size_t));
+    cumsum_of_worker_group_sizes = (size_t*) calloc(num_conds, sizeof(size_t));
     mutex_held = (bool*) malloc(sizeof(bool) * number_of_workers);
     for (int i = 0; i < number_of_workers; i++) {
-        cumsum_of_cond_of[cond_of(i)]++;
+        cumsum_of_worker_group_sizes[cond_of(i)]++;
     }
     for (int i = 1; i < num_conds; i++) {
-        cumsum_of_cond_of[i] += cumsum_of_cond_of[i - 1];
+        cumsum_of_worker_group_sizes[i] += cumsum_of_worker_group_sizes[i - 1];
     }
     for (int i = 0; i < num_conds; i++) {
         lf_cond_init(worker_conds + i);
@@ -126,7 +132,7 @@ static void worker_states_awaken_locked(size_t worker, size_t num_to_awaken) {
     }
     // The predicate of the condition variable depends on num_awakened and level_counter, so
     // this is a critical section.
-    num_loose_threads = cumsum_of_cond_of[max_cond];
+    num_loose_threads = cumsum_of_worker_group_sizes[max_cond];
     num_loose_threads += worker >= num_loose_threads;
     num_awakened = num_loose_threads;
     level_counter++;
diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index 68523dc4b..f909e9ee8 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-b6c0971bd810b48f1a3e816beaf9ce5d86eb21ea
+2aaf0ea831e11e4e78af6ce30b6982c44548c210

From e09ad62bcce5bf926d4f4f3cd02850ddf9af0686 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Wed, 1 Jun 2022 15:44:35 -0700
Subject: [PATCH 42/49] [scheduler] Adjust assert.

---
 core/threaded/worker_states.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index cafd27878..083068079 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -175,7 +175,7 @@ static bool worker_states_finished_with_level_locked(size_t worker) {
     // Why use an atomic operation when we are supposed to be "as good as locked"? Because I took a
     // shortcut, and the shortcut was imperfect.
     size_t ret = lf_atomic_add_fetch(&num_loose_threads, -1);
-    assert(ret >= 0);
+    assert(ret <= max_num_workers);  // Check for underflow
     return !ret;
 }
 

From d682724d22a5b349915e12d7a3b887393241a170 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Fri, 3 Jun 2022 14:40:38 -0700
Subject: [PATCH 43/49] [scheduler] Account for insertion into the current
 level.

---
 core/threaded/worker_assignments.h | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index a27d6fc67..4681ee66f 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -139,12 +139,29 @@ static void worker_assignments_free() {
  * @param worker The number of a worker needing work.
  */
 static reaction_t* get_reaction(size_t worker) {
+#ifndef FEDERATED
     int index = lf_atomic_add_fetch(num_reactions_by_worker + worker, -1);
     if (index >= 0) {
         return reactions_by_worker[worker][index];
     }
     num_reactions_by_worker[worker] = 0;
     return NULL;
+#else
+    int old_num_reactions;
+    int current_num_reactions = num_reactions_by_worker[worker];
+    int index;
+    do {
+        old_num_reactions = current_num_reactions;
+        if (old_num_reactions <= 0) return NULL;
+    } while (
+        (current_num_reactions = __sync_val_compare_and_swap(
+            num_reactions_by_worker + worker,
+            old_num_reactions,
+            (index = old_num_reactions - 1)
+        )) != old_num_reactions
+    );
+    return reactions_by_worker[worker][index];
+#endif
 }
 
 /**

From 510cd4c41a915f13348185bfd6233d9d9f9ac2f0 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Fri, 3 Jun 2022 18:47:09 -0700
Subject: [PATCH 44/49] [scheduler] Unrelated bugfix.

By "unrelated" I mean, "unrelated to the hard-to-reproduce race condition that is manifesting itself in LoopDistributedCentralized in CI."
---
 core/threaded/worker_states.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/threaded/worker_states.h b/core/threaded/worker_states.h
index 083068079..6029a3321 100644
--- a/core/threaded/worker_states.h
+++ b/core/threaded/worker_states.h
@@ -92,7 +92,7 @@ static void worker_states_init(size_t number_of_workers) {
     size_t num_conds = cond_of(greatest_worker_number) + 1;
     worker_conds = (lf_cond_t*) malloc(sizeof(lf_cond_t) * num_conds);
     cumsum_of_worker_group_sizes = (size_t*) calloc(num_conds, sizeof(size_t));
-    mutex_held = (bool*) malloc(sizeof(bool) * number_of_workers);
+    mutex_held = (bool*) calloc(number_of_workers, sizeof(bool));
     for (int i = 0; i < number_of_workers; i++) {
         cumsum_of_worker_group_sizes[cond_of(i)]++;
     }
@@ -148,7 +148,7 @@ static void worker_states_lock(size_t worker) {
     size_t lt = num_loose_threads;
     if (lt > 1 || !fast) {  // FIXME: Lock should be partially optimized out even when !fast
         lf_mutex_lock(&mutex);
-        assert(mutex_held[worker] == false);
+        assert(!mutex_held[worker]);
         mutex_held[worker] = true;
     }
 }
@@ -204,7 +204,7 @@ static void worker_states_sleep_and_unlock(size_t worker, size_t level_counter_s
             lf_cond_wait(worker_conds + cond, &mutex);
         } while (level_counter_snapshot == level_counter || worker >= num_awakened);
     }
-    assert(mutex_held[worker] == false);  // This thread holds the mutex, but it did not report that.
+    assert(!mutex_held[worker]);  // This thread holds the mutex, but it did not report that.
     lf_mutex_unlock(&mutex);
 }
 

From ea3011bad0ca500b8a628d0f6999dd116b91098e Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Sat, 4 Jun 2022 21:50:20 -0700
Subject: [PATCH 45/49] [scheduler] Update assertion.

---
 core/threaded/worker_assignments.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 4681ee66f..e0785ef3e 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -197,7 +197,9 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
 static void worker_assignments_put(reaction_t* reaction) {
     size_t level = LEVEL(reaction->index);
     assert(reaction != NULL);
+#ifndef FEDERATED
     assert(level > current_level || current_level == 0);
+#endif
     assert(level < num_levels);
     // Source: https://xorshift.di.unimi.it/splitmix64.c
     // FIXME: This is probably not the most efficient way to get the randomness that we need because

From c19bb3d1895ad15c9eec27292f0731d16801c62d Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Mon, 6 Jun 2022 20:59:18 -0700
Subject: [PATCH 46/49] [scheduler] Fix deadlock.

---
 core/threaded/data_collection.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index e617f62bd..a95d8d3c6 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -187,8 +187,12 @@ static void compute_number_of_workers(
                 ideal_number_of_workers, this_execution_time
             );
         }
+        int minimum_workers = 1;
+#ifdef FEDERATED
+        minimum_workers = WORKERS_NEEDED_FOR_FEDERATE;
+#endif
         num_workers_by_level[level] = restrict_to_range(
-            1, max_reasonable_num_workers, ideal_number_of_workers
+            minimum_workers, max_reasonable_num_workers, ideal_number_of_workers
         );
     }
 }

From f1e20d54ac75103781ac2e8d4c132424d8072730 Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Mon, 6 Jun 2022 22:02:59 -0700
Subject: [PATCH 47/49] [scheduler] Clean up after previous commit.

---
 core/threaded/data_collection.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index a95d8d3c6..112e8ca2c 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -188,8 +188,10 @@ static void compute_number_of_workers(
             );
         }
         int minimum_workers = 1;
-#ifdef FEDERATED
-        minimum_workers = WORKERS_NEEDED_FOR_FEDERATE;
+#ifdef WORKERS_NEEDED_FOR_FEDERATE
+        // TODO: only apply this constraint on levels containing control reactions
+        minimum_workers = WORKERS_NEEDED_FOR_FEDERATE > max_reasonable_num_workers ?
+            max_reasonable_num_workers : WORKERS_NEEDED_FOR_FEDERATE;
 #endif
         num_workers_by_level[level] = restrict_to_range(
             minimum_workers, max_reasonable_num_workers, ideal_number_of_workers

From d23d17fd66517dcc907fcdebd1edc808ad991d3c Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Mon, 6 Jun 2022 22:06:19 -0700
Subject: [PATCH 48/49] [scheduler] Do not directly use compiler builtins.

---
 core/platform.h                    | 20 ++++++++++++++++++--
 core/threaded/worker_assignments.h |  2 +-
 lingua-franca-ref.txt              |  2 +-
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/core/platform.h b/core/platform.h
index 941cce803..3b00f440f 100644
--- a/core/platform.h
+++ b/core/platform.h
@@ -210,10 +210,10 @@ extern int lf_cond_timedwait(lf_cond_t* cond, lf_mutex_t* mutex, instant_t absol
  * @param ptr A pointer to a variable.
  * @param oldval The value to compare against.
  * @param newval The value to assign to *ptr if comparison is successful.
- * @return The true if comparison was successful. False otherwise.
+ * @return True if comparison was successful. False otherwise.
  */
 #if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
-// Assume that an integer is 32 bits.
+// Assume that a boolean is represented with a 32-bit integer.
 #define lf_bool_compare_and_swap(ptr, oldval, newval) (InterlockedCompareExchange(ptr, newval, oldval) == oldval)
 #elif defined(__GNUC__) || defined(__clang__)
 #define lf_bool_compare_and_swap(ptr, oldval, newval) __sync_bool_compare_and_swap(ptr, oldval, newval)
@@ -221,6 +221,22 @@ extern int lf_cond_timedwait(lf_cond_t* cond, lf_mutex_t* mutex, instant_t absol
 #error "Compiler not supported"
 #endif
 
+/*
+ * Atomically compare the 32-bit value that ptr points to against oldval. If the
+ * current value is oldval, then write newval into *ptr.
+ * @param ptr A pointer to a variable.
+ * @param oldval The value to compare against.
+ * @param newval The value to assign to *ptr if comparison is successful.
+ * @return The initial value of *ptr.
+ */
+#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__NT__)
+#define lf_val_compare_and_swap(ptr, oldval, newval) InterlockedCompareExchange(ptr, newval, oldval)
+#elif defined(__GNUC__) || defined(__clang__)
+#define lf_val_compare_and_swap(ptr, oldval, newval) __sync_val_compare_and_swap(ptr, oldval, newval)
+#else
+#error "Compiler not supported"
+#endif
+
 #endif
 
 /**
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index e0785ef3e..2c753d80c 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -154,7 +154,7 @@ static reaction_t* get_reaction(size_t worker) {
         old_num_reactions = current_num_reactions;
         if (old_num_reactions <= 0) return NULL;
     } while (
-        (current_num_reactions = __sync_val_compare_and_swap(
+        (current_num_reactions = lf_val_compare_and_swap(
             num_reactions_by_worker + worker,
             old_num_reactions,
             (index = old_num_reactions - 1)
diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt
index f909e9ee8..1a62c2c9f 100644
--- a/lingua-franca-ref.txt
+++ b/lingua-franca-ref.txt
@@ -1 +1 @@
-2aaf0ea831e11e4e78af6ce30b6982c44548c210
+36c2e76fdb75717a825ab4217fe80b74076d0cd0

From 3e1ab3e048c9ee4755ff0a3bc7c8ba1f465c8ade Mon Sep 17 00:00:00 2001
From: Peter Donovan <peterdonovan@berkeley.edu>
Date: Thu, 9 Jun 2022 10:25:59 -0700
Subject: [PATCH 49/49] [scheduler] Minor cleanups.

---
 core/threaded/data_collection.h    |  3 ++-
 core/threaded/scheduler_adaptive.c |  2 +-
 core/threaded/worker_assignments.h | 10 ++++++++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/core/threaded/data_collection.h b/core/threaded/data_collection.h
index 112e8ca2c..036c78467 100644
--- a/core/threaded/data_collection.h
+++ b/core/threaded/data_collection.h
@@ -104,7 +104,8 @@ static size_t get_nums_workers_neighboring_state(size_t current_state, interval_
     size_t jitter = get_jitter(current_state, execution_time);
     if (!jitter) return current_state;
     size_t i = 1;
-    while (possible_nums_workers[i] < current_state) i++; // TODO: There are more efficient ways to do this.
+    // TODO: There are more efficient ways to do this.
+    while (possible_nums_workers[i] < current_state) i++;
     return possible_nums_workers[i + jitter];
 }
 
diff --git a/core/threaded/scheduler_adaptive.c b/core/threaded/scheduler_adaptive.c
index fdf069861..8b951fe44 100644
--- a/core/threaded/scheduler_adaptive.c
+++ b/core/threaded/scheduler_adaptive.c
@@ -25,6 +25,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 /**
  * This is a non-priority-driven scheduler. See scheduler.h for documentation.
+ * @author{Peter Donovan <peterdonovan@berkeley.edu>}
  */
 
 #ifndef NUMBER_OF_WORKERS
@@ -123,7 +124,6 @@ void lf_sched_done_with_reaction(size_t worker_number, reaction_t* done_reaction
 
 void lf_sched_trigger_reaction(reaction_t* reaction, int worker_number) {
     assert(worker_number >= -1);
-    if (reaction == NULL) return; // FIXME: When does this happen again? In federated execution?
     if (!lf_bool_compare_and_swap(&reaction->status, inactive, queued)) return;
     worker_assignments_put(reaction);
 }
diff --git a/core/threaded/worker_assignments.h b/core/threaded/worker_assignments.h
index 2c753d80c..b934c30b5 100644
--- a/core/threaded/worker_assignments.h
+++ b/core/threaded/worker_assignments.h
@@ -147,6 +147,8 @@ static reaction_t* get_reaction(size_t worker) {
     num_reactions_by_worker[worker] = 0;
     return NULL;
 #else
+    // This is necessary for federated programs because reactions may be inserted into the current
+    // level.
     int old_num_reactions;
     int current_num_reactions = num_reactions_by_worker[worker];
     int index;
@@ -178,7 +180,11 @@ static reaction_t* worker_assignments_get_or_lock(size_t worker) {
     while (true) {
         if ((ret = get_reaction(worker))) return ret;
         if (worker < num_workers) {
-            for (size_t victim = (worker + 1) % num_workers; victim != worker; victim = (victim + 1) % num_workers) {
+            for (
+                size_t victim = (worker + 1) % num_workers;
+                victim != worker;
+                victim = (victim + 1) % num_workers
+            ) {
                 if ((ret = get_reaction(victim))) return ret;
             }
         }
@@ -202,7 +208,7 @@ static void worker_assignments_put(reaction_t* reaction) {
 #endif
     assert(level < num_levels);
     // Source: https://xorshift.di.unimi.it/splitmix64.c
-    // FIXME: This is probably not the most efficient way to get the randomness that we need because
+    // TODO: This is probably not the most efficient way to get the randomness that we need because
     // it is designed to give an entire word of randomness, whereas we only need
     // ~log2(num_workers_by_level[level]) bits of randomness.
     uint64_t hash = (uint64_t) reaction;