Merge pull request #365 from NVlabs/UserStatusSplit

User session status split

.gitignore

@@ -2,22 +2,22 @@
 .vscode
 .vs
 # Don't check in any compiled results
-vs/Build
-vs/packages
+/vs/Build
+/vs/packages
 # Avoid .pyc files (everywhere)
 *.pyc
 # Don't check-in any of these data-files
-data-files/debugging_db.db
-data-files/g3d-license.txt
-data-files/log.txt
-data-files/*.db
-data-files/*.csv
-data-files/*.pdn
-data-files/*.Any
-data-files/*.jpg
+/data-files/debugging_db.db
+/data-files/g3d-license.txt
+/data-files/log.txt
+/data-files/*.db
+/data-files/*.csv
+/data-files/*.pdn
+/data-files/*.Any
+/data-files/*.jpg
 *.mp4
 # Test generated files
-data-files/test/userconfig.Any
+/data-files/test/userconfig.Any
 # Don't check in any results
 "scripts/event logger/software/Logs"
 results/
@@ -35,6 +35,9 @@ results/
 !/data-files/scene/Test_Cornell_Box.Scene.Any
 # FPSci provided audio
 !/data-files/sound/fpsci_*
+# Ignore sample and test status
+/data-files/samples/*.csv
+/data-files/test/*.csv
 # This stuff ignores any packaged outputs
 *.zip
 *.tar

data-files/samples/sa2019_1hit.Status.Any

@@ -4,7 +4,6 @@
     settingsVersion = 1;
     users = (
         { 
-            completedSessions = ( ); 
             id = "sa2019_sampleuser"; 
             sessions = ("60hz_0delay", "60hz_2delay", "120hz_0delay", "120hz_4delay", "240hz_0delay", "240hz_8delay");
         },

data-files/samples/sa2019_track.Status.Any

@@ -4,7 +4,6 @@
     settingsVersion = 1;
     users = (
         { 
-            completedSessions = ( ); 
             id = "sa2019_sampleuser"; 
             sessions = ("60hz_0delay", "60hz_2delay", "120hz_0delay", "120hz_4delay", "240hz_0delay", "240hz_8delay");
         },

data-files/samples/spheres.Status.Any

@@ -4,7 +4,6 @@
     settingsVersion = 1; 
     users = ( 
         { 
-            completedSessions = ( ); 
             id = "Sample User"; 
             sessions = ( "mixedtypes", "icosohedron", "lowpoly", "midpoly", "highpoly"); 
         } ); 

data-files/samples/targets.Status.Any

@@ -4,7 +4,6 @@
     settingsVersion = 1; 
     users = ( 
         { 
-            completedSessions = ( ); 
             id = "Sample User"; 
             sessions = ( "1target", "2targets", "5targets", "moving", "jumping", "mixedtypes"); 
         } ); 

data-files/samples/weapons.Status.Any

@@ -4,7 +4,6 @@
     settingsVersion = 1; 
     users = ( 
         { 
-            completedSessions = ( ); 
             id = "Sample User"; 
             sessions = ( "1shot", "2shots", "3shots", "autofire", "fastfire", "showweapon"); 
         } ); 

data-files/test/userstatus.Any

@@ -4,12 +4,10 @@
     settingsVersion = 1; 
     users = ( 
         { 
-            completedSessions = ( ); 
             id = "TestUser"; 
             sessions = ( "main", "sizes"); 
         }, 
         { 
-            completedSessions = ( ); 
             id = "anon"; 
             sessions = ( "main", "sizes", "defaultCamera", "customCamera", "60HzContinuous", "30HzContinuous"); 
         } ); 

docs/userStatusReadme.md

@@ -1,7 +1,9 @@
 # Introduction
 The user status file tracks all user infomration specific to a given experiment (unlike the [userconfig.Any](./userConfigReadme.md) file which tracks experiment-indepdent user settings). 
 
-The user status is the primary mechanism by which sessions are ordered and progress is tracked in `FirstPersonScience`. Similarly to the [`userconfig.Any`](./userConfigReadme.md) this file controls per-user actions using a user table and also records completed sessions so that the application can track user progress over multiple runtimes.
+The user status is the primary mechanism by which sessions are ordered and progress is tracked in `FirstPersonScience`. Similarly to the [`userconfig.Any`](./userConfigReadme.md) this file controls per-user session ordering using a user table.
+
+A second file records completed sessions to track user progress over multiple restarts. This second file name can be specified inside the user status, otherwise it'll use the same basename (excluding the last extension - e.g. `.Any`) as the user status and append `.sessions.csv`. To restart an experiment, or reset progress, delete, move, or rename that file.
 
 ## File Location
 The `userstatus.Any` file is located in the [`data-files` directory](../data-files) at the root of the project. If no `userstatus.Any` file is present at startup the application writes a set of default values to `userstatus.Any`. The default user name in this file is `anon` and the sessions assigned to this user refer to the default values for `experimentconfig.Any` to make the solution work as-is without any config files present.
@@ -12,12 +14,12 @@ The top-level user status table contains the following fields:
 * `allowRepeat` determines whether or not to (strictly) sequence the trials, allowing for repeats
 * `randomizeSessionOrder` determines whether or not individual users are assigned randomized orderings from the (default) `sessions` array (defined below)
 * `sessions` is the default sessions list for any user in the file that does not have a sessions list specified
+* `completedLogFilename` is the (optional) filename to use to store the completed session log (formatted as a CSV with `user id, session id` format). If unspecified the completed log filename will be `[user status filename].sessions.csv`.
 
 Each entry in the user table contains the following fields:
 
 * `id` a quick ID used to identify the user
 * `sessions` a list of sessions to be completed, in order
-* `completedSessions` a list of sessions completed by any given user
 
 The `sessions` list above can be used to control session ordering (this is an ordered list). If random ordering is desired a quick script can be written to read all users from the `userconfig.Any` file and write a new sequence of sessions for each user present. Alternatively the top-level `sessions` list can be used to specify a single ordering for all participants:
 
@@ -30,6 +32,8 @@ users = (
 );
 ```
 
-Once all the items from the `sessions` list are present in the `completedSessions` list the experiment is considered "done" for this user. At this point, if a user wants to re-run the experiment they need to open the `userstatus.Any` file and delete all items from their `completedSessions` list. Alternatively if new trials will be run for all users a new copy of the file (w/ empty `completedSessions` lists for all users) can be copy-pasted over the full one.
+Once all the items from the `sessions` list are present in the completed sessions list (which is saved to the `completedLogFilename` file) the experiment is considered "done" for this user. At this point, if a user wants to re-run the experiment they need to reset by deleting the file whose name matches the `completedLogFilename` or the user status basename plus `.sessions.csv`. Note that by deleting the `.sessions.csv` file, all users' progress will be reset. Alternatively a single user can be reset by removing all lines with that user's name in the `.sessions.csv` or file using the `completedLogFilename`.
+
+If the `allowRepeat` flag is set to `true` then repeated sessions may appear in the `sessions` array as these will be strictly ordered. In addition when `allowRepeat = true` the completed sessions array is expected to match the `sessions` array item-for-item (as ordered). If this is not the case the application may not behave as intended. For this reason it is a good idea to make sure remove all entries for a given user following any change to the `sessions` arrays within `userstatus.Any`. Additionally, you shouldn't allow users to select sessions out of order when `allowRepeat = true` as it will allow sessions to be completed out of order.
 
-If the `sequence` flag is set to `true` then repeated sessions may appear in the `sessions` array as these will be strictly ordered. In addition when `sequence = true` the `completedSession` array is expected to match the `sessions` array item-for-item (as ordered). If this is not the case the application may not behave as intended. For this reason it is a good idea to make sure the `completeSessions` arrays are emptied following any change to the `sessions` arrays within `userstatus.Any`.
+We recommend only using either `allowRepeat` or `randomizeSessionOrder` exclusively as `randomizeSessionOrder` expects a single copy of each session in the list.

scripts/package/fpsci_packager.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Autogenerated 2022 04 11 17:27:44
+# Autogenerated 2022 05 27 11:59:57
 mkdir -p dist/sound/
 mkdir -p dist/shader/UniversalSurface/
 mkdir -p dist/shader/UniversalMaterial/

source/FPSciApp.cpp

@@ -508,7 +508,6 @@ void FPSciApp::markSessComplete(String sessId) {
 	}
 	// Add the session id to completed session array and save the user status table
 	userStatusTable.addCompletedSession(userStatusTable.currentUser, sessId);
-	saveUserStatus();
 	logPrintf("Marked session: %s complete for user %s.\n", sessId, userStatusTable.currentUser);
 
 	// Update the session drop-down to remove this session

source/UserStatus.cpp

@@ -2,70 +2,80 @@
 #include "FPSciAnyTableReader.h"
 
 UserSessionStatus::UserSessionStatus(const Any& any) {
-	int settingsVersion = 1; // used to allow different version numbers to be loaded differently
 	FPSciAnyTableReader reader(any);
-	reader.getIfPresent("settingsVersion", settingsVersion);
-
-	switch (settingsVersion) {
-	case 1:
-		// Require a user ID
-		reader.get("id", id, "All user status fields must include the user ID!");
-		// Setup default session order, then overwrite if specified	
-		sessionOrder = defaultSessionOrder;
-		if (randomizeDefaults) sessionOrder.randomize();
-		reader.getIfPresent("sessions", sessionOrder);			// Override the default session order if one is provided for this user
-		if (sessionOrder.length() == 0) {						// Check for sessions in list
-			throw format("Must provide \"sessions\" array (or default) for User ID:\"%s\" in user status!", id);
-		}
-		// Get the completed sessions array
-		reader.getIfPresent("completedSessions", completedSessions);
-		break;
-	default:
-		debugPrintf("Settings version '%d' not recognized in UserSessionStatus.\n", settingsVersion);
-		break;
+
+	// Require a user ID
+	reader.get("id", id, "All user status fields must include the user ID!");
+	// Setup default session order, then overwrite if specified	
+	sessionOrder = defaultSessionOrder;
+	if (randomizeDefaults) sessionOrder.randomize();
+	reader.getIfPresent("sessions", sessionOrder);			// Override the default session order if one is provided for this user
+	if (sessionOrder.length() == 0) {						// Check for sessions in list
+		throw format("Must provide \"sessions\" array (or default) for User ID:\"%s\" in user status!", id);
 	}
 }
 
 Any UserSessionStatus::toAny(const bool forceAll) const {
 	Any a(Any::TABLE);
 	a["id"] = id;									// populate id
 	a["sessions"] = sessionOrder;					// populate session order
-	a["completedSessions"] = completedSessions; 	// Include updated subject table
 	return a;
 }
 
 UserStatusTable::UserStatusTable(const Any& any) {
-	int settingsVersion = 1; // used to allow different version numbers to be loaded differently
 	FPSciAnyTableReader reader(any);
-	reader.getIfPresent("settingsVersion", settingsVersion);
-
-	switch (settingsVersion) {
-	case 1:
-		reader.getIfPresent("currentUser", currentUser);
-		reader.getIfPresent("allowRepeat", allowRepeat);
-		reader.getIfPresent("sessions", defaultSessionOrder);
-		UserSessionStatus::defaultSessionOrder = defaultSessionOrder;				// Set the default order here
-		reader.getIfPresent("randomizeSessionOrder", randomizeDefaults);
-		UserSessionStatus::randomizeDefaults = randomizeDefaults;					// Set whether default session order is randomized
-		reader.get("users", userInfo, "Issue in the (required) \"users\" array from the user status file!");
-		break;
-	default:
-		debugPrintf("Settings version '%d' not recognized in UserStatus.\n", settingsVersion);
-		break;
-	}
+
+	reader.getIfPresent("currentUser", currentUser);
+	reader.getIfPresent("allowRepeat", allowRepeat);
+	reader.getIfPresent("sessions", defaultSessionOrder);
+	UserSessionStatus::defaultSessionOrder = defaultSessionOrder;				// Set the default order here
+	reader.getIfPresent("randomizeSessionOrder", randomizeDefaults);
+	UserSessionStatus::randomizeDefaults = randomizeDefaults;					// Set whether default session order is randomized
+	reader.get("users", userInfo, "Issue in the (required) \"users\" array from the user status file!");
+	reader.getIfPresent("completedLogFilename", completedLogFilename);
 }
 
 UserStatusTable UserStatusTable::load(const String& filename, bool saveJSON) {
+	UserStatusTable status;
 	if (!FileSystem::exists(filename)) {						// if file not found, create a default
-		UserStatusTable defaultStatus = UserStatusTable();		// Create empty status
 		UserSessionStatus user;
 		user.sessionOrder = Array<String>({ "60Hz", "30Hz" });	// Add "default" sessions we add to
-		defaultStatus.userInfo.append(user);					// Add single "default" user
-		defaultStatus.currentUser = user.id;					// Set "default" user as current user
-		defaultStatus.save(filename, saveJSON);					// Save .any file
-		return defaultStatus;
+		status.userInfo.append(user);							// Add single "default" user
+		status.currentUser = user.id;							// Set "default" user as current user
+		status.save(filename, saveJSON);						// Save .any file
+	}
+	else status = Any::fromFile(filename);
+
+	// Populate completed session log name if missing (use user status filename as base)
+	if (status.completedLogFilename.empty()) {
+		status.completedLogFilename = filename.substr(0, filename.find_last_of('.')) + ".sessions.csv";
+	}
+
+	// Load into completedSessions
+	if (FileSystem::exists(status.completedLogFilename)) {
+		std::ifstream log;
+		log.open(status.completedLogFilename.c_str());
+		std::string line;
+		while (std::getline(log, line)) {
+			size_t commaIdx = line.find(',');
+			String userID = String(line).substr(0, commaIdx);
+			String sessID = String(line).substr(commaIdx + 1, line.length() - commaIdx - 1);
+			for (UserSessionStatus& info : status.userInfo) {
+				if (!info.id.compare(userID)) { 
+					info.completedSessions.append(sessID); 
+				}
+			}
+		}
+		log.close();
 	}
-	return Any::fromFile(System::findDataFile(filename));
+
+	// Open for writing
+	status.completedLog.open(status.completedLogFilename.c_str(), std::ios_base::app);
+	if (!status.completedLog.is_open()) {
+		logPrintf("Failed to open user completed session log!");
+	}
+
+	return status;
 }
 
 Any UserStatusTable::toAny(const bool forceAll) const {
@@ -81,7 +91,7 @@ Any UserStatusTable::toAny(const bool forceAll) const {
 }
 
 shared_ptr<UserSessionStatus> UserStatusTable::getUserStatus(const String& id) {
-	for (UserSessionStatus user : userInfo) {
+	for (UserSessionStatus& user : userInfo) {
 		if (!user.id.compare(id)) return std::make_shared<UserSessionStatus>(user);
 	}
 	return nullptr;
@@ -112,11 +122,16 @@ String UserStatusTable::getNextSession(String userId) {
 }
 
 void UserStatusTable::addCompletedSession(const String& userId, const String& sessId) {
+	// Update the user info
 	for (int i = 0; i < userInfo.length(); i++) {
 		if (!userInfo[i].id.compare(userId)) {
 			userInfo[i].completedSessions.append(sessId);
 		}
 	}
+
+	// Log the completed session to the session log
+	completedLog << userId.c_str() << "," << sessId.c_str() << "\n";
+	completedLog.flush();
 }
 
 void UserStatusTable::validate(const Array<String>& sessions, const Array<String>& users) {

source/UserStatus.h

@@ -1,12 +1,13 @@
 #pragma once
 #include <G3D/G3D.h>
+#include <fstream>
 
 /** Class for handling user status */
 class UserSessionStatus {
 public:
 	String					id = "anon";					///< User ID
 	Array<String>			sessionOrder;					///< Array containing session ordering
-	Array<String>			completedSessions;				///< Array containing all completed session ids for this user
+	Array<String>			completedSessions;				///< Array containing all completed session ids for this user (loaded from log)
 	static Array<String>	defaultSessionOrder;			///< Default session order
 	static bool				randomizeDefaults;				///< Randomize default session order when applying to individual?
 
@@ -24,14 +25,20 @@ class UserStatusTable {
 	String currentUser;									///< Currently selected user
 	Array<String> defaultSessionOrder = {};				///< Default session ordering (for all unspecified users)
 	Array<UserSessionStatus> userInfo = {};				///< Array of user status
+	Table<String, Array<String>> completed;
+
+	String completedLogFilename;						///< File containing completed sessions
+	std::ofstream completedLog;
 
 	UserStatusTable() {}
 	UserStatusTable(const Any& any);
 
 	static UserStatusTable load(const String& filename, bool saveJSON);
 	Any toAny(const bool forceAll = false) const;
 
-	inline void save(const String& filename, bool json) { toAny().save(filename, json); }
+	inline void save(const String& filename, bool json) {
+		toAny().save(filename, json);
+	};
 
 	shared_ptr<UserSessionStatus> getUserStatus(const String& id);			// Get a given user's status from the table by ID
 	String getNextSession(String userId = "");								// Get the next session ID for a given user (by ID)

tests/FPSciTests.cpp

@@ -868,6 +868,8 @@ TEST_F(FPSciTests, TestFreshStart) {
 	EXPECT_TRUE(failDelete) << "User Config present from previous run!";
 	failDelete = remove("test/emptystatus.Any");
 	EXPECT_TRUE(failDelete) << "User Status present from previous run!";
+	failDelete = remove("test/emptystatus.sessions.csv");
+	EXPECT_TRUE(failDelete) << "User Status sessions present from previous run!";
 	failDelete = remove("test/emptykeymap.Any");
 	EXPECT_TRUE(failDelete) << "Keymap present from previous run!";
 	failDelete = remove("test/emptysystem.Any");
@@ -892,4 +894,19 @@ TEST_F(FPSciTests, TestFreshStart) {
 	EXPECT_FALSE(failDelete) << "Keymap not generated!";
 	failDelete = remove("test/emptysystem.Any");
 	EXPECT_FALSE(failDelete) << "System Config not generated!";
+
+	// The sessions csv doesn't generate until some progress happens. 
+	// We switch experiments here to force that to happen.
+	// Load `testExperiment` (at index 0)
+	s_app->experimentIdx = 0;
+	EXPECT_NO_THROW(
+		s_app->initExperiment();
+	) << "Failed to initialize with test config";
+	// Run one frame (to make sure there's no crash)
+	EXPECT_NO_FATAL_FAILURE(
+		s_app->oneFrame();
+	) << "Failed to run one frame with test config";
+	// Delete the sessions csv
+	failDelete = remove("test/emptystatus.sessions.csv");
+	EXPECT_FALSE(failDelete) << "User Status sessions csv not generated!";
 }