main.py evaluate.py ADVERSARIAL PRESET PUSH GRASP EVAL WORKS

evaluate.py

@@ -20,7 +20,22 @@
 num_obj_complete = args.num_obj_complete
 # There is a preset directory with this number of objects each time
 num_obj_presets = [4, 5, 3, 5, 5, 6, 3, 6, 6, 5, 4]
-preset_trials = args.preset_num_trials
+# num_obj_presets = [3, 5, 6, 3, 4, 6, 5, 5, 5, 4, 6]
+# number of objects per preset case file:
+# file: num_obj
+# 00: 4
+# 01: 5
+# 02: 3
+# 03: 5
+# 04: 5
+# 05: 6
+# 06: 3
+# 07: 6
+# 08: 6
+# 09: 5
+# 10: 4
+
+preset_trials_per_case = args.preset_num_trials
 
 # Parse data from session (action executed, reward values)
 # NOTE: reward_value_log just stores some value which is indicative of successful grasping, which could be a class ID (reactive) or actual reward value (from MDP, reinforcement)
@@ -29,8 +44,11 @@
 max_iteration = executed_action_log.shape[0]
 executed_action_log = executed_action_log[0:max_iteration,:]
 reward_value_log = np.loadtxt(os.path.join(transitions_directory, 'reward-value.log.txt'), delimiter=' ')
+grasp_success_log = np.loadtxt(os.path.join(transitions_directory, 'grasp-success.log.txt'), delimiter=' ')
 reward_value_log = reward_value_log[0:max_iteration]
 clearance_log = np.loadtxt(os.path.join(transitions_directory, 'clearance.log.txt'), delimiter=' ')
+# work around a bug where the clearance steps were written twice per clearance
+clearance_log = np.unique(clearance_log)
 max_trials = len(clearance_log)
 clearance_log = np.concatenate((np.asarray([0]), clearance_log), axis=0).astype(int)
 print('number of clearances: ' + str(len(clearance_log)-1))
@@ -44,31 +62,40 @@
 for trial_idx in range(1, len(clearance_log)):
     if args.preset:
         # TODO(ahundt) If a bug is fixed in the logging code, there is one too many trials here.
-        preset_situation_num = int((trial_idx-2)/preset_trials)
+        num_preset_files = 11
+        preset_situation_num = min(num_preset_files-1, int(float(trial_idx-1)/float(preset_trials_per_case)))
+        # preset_situation_num = (trial_idx-1) % num_preset_files
+        # preset_situation_num = int((trial_idx-2)/preset_trials_per_case)
         num_obj_complete = num_obj_presets[preset_situation_num]
     # Get actions and reward values for current trial
-    tmp_executed_action_log = executed_action_log[clearance_log[trial_idx-1]:clearance_log[trial_idx],0]
-    tmp_reward_value_log = reward_value_log[clearance_log[trial_idx-1]:clearance_log[trial_idx]]
+    start_idx = clearance_log[trial_idx-1]
+    end_idx = clearance_log[trial_idx]
+    # print('trial: ' + str(trial_idx) + ' start: ' + str(start_idx) + ' end: ' + str(end_idx))
+    tmp_executed_action_log = executed_action_log[start_idx:end_idx,0]
+    tmp_reward_value_log = reward_value_log[start_idx:end_idx]
+    tmp_grasp_success_log = grasp_success_log[start_idx:end_idx]
 
     # Get indices of pushing and grasping actions for current trial
     tmp_grasp_attempt_ind = np.argwhere(tmp_executed_action_log == 1)
     tmp_push_attempt_ind = np.argwhere(tmp_executed_action_log == 0)
 
+    # print('debug len(tmp_executed_action_log)' + str(len(tmp_executed_action_log)))
     grasp_to_push_ratio[trial_idx-1] = float(len(tmp_grasp_attempt_ind))/float(len(tmp_executed_action_log))
 
     # Count number of times grasp attempts were successful
     if method == 'reactive':
         tmp_num_grasp_success = np.sum(tmp_reward_value_log[tmp_grasp_attempt_ind] == 0) # Class ID for successful grasping is 0 (reactive policy)
     elif method == 'reinforcement':
         tmp_num_grasp_success = np.sum(tmp_reward_value_log[tmp_grasp_attempt_ind] >= 0.5) # Reward value for successful grasping is anything larger than 0.5 (reinforcement policy)
-
-    grasp_num_success[trial_idx-1] = tmp_num_grasp_success
+        # print('trial: ' + str(trial_idx) + ' num grasp success:' + str(tmp_num_grasp_success))
+    grasp_num_success[trial_idx-1] = np.sum(tmp_grasp_success_log)
     grasp_success_rate[trial_idx-1] = float(tmp_num_grasp_success)/float(len(tmp_grasp_attempt_ind))
     # Did the trial reach task completion?
     cleared = tmp_num_grasp_success >= num_obj_complete
     if args.preset:
         print('trial: ' + str(trial_idx) + ' preset_situation_num: ' + str(preset_situation_num) + ' num_obj: ' + str(num_obj_complete) +
-              ' num_grasps: ' + str(tmp_num_grasp_success) + ' cleared: ' + str(cleared))
+              ' num_grasp_success: ' + str(tmp_num_grasp_success) + ' cleared: ' + str(cleared) + 
+              ' start: ' + str(start_idx) + ' end: ' + str(end_idx) + ' actions: ' + str(len(tmp_executed_action_log)))
     valid_clearance.append(cleared)
 
 # Display results

main.py

@@ -144,6 +144,7 @@ def main(args):
             # load a directory of files
             preset_files = os.listdir(args.test_preset_dir)
             preset_files = [os.path.abspath(os.path.join(args.test_preset_dir, filename)) for filename in preset_files]
+            preset_files = sorted(preset_files)
         trials_per_case = max_test_trials
         # run each preset file max_test_trials times.
         max_test_trials *= len(preset_files)
@@ -520,7 +521,10 @@ def process_actions():
     # -------------------------------------------------------------
     prev_primitive_action = None
     prev_reward_value = None
-
+    if test_preset_cases:
+        # save out the order we will visit the preset files for a sanity check
+        print('preset files order: ' + str(preset_files))
+        np.savetxt(os.path.join(logger.transitions_directory, 'preset-case-files.log.txt'), preset_files, delimiter=' ', fmt='%s')
     if show_preset_cases_then_exit and test_preset_cases:
         # Just a quick temporary mode for viewing the saved preset test cases
         for case_file in preset_files:
@@ -588,7 +592,11 @@ def process_actions():
             # we're still not totally done, we still need to finilaize the log for the trial
             nonlocal_variables['finalize_prev_trial_log'] = True
             if is_testing and test_preset_cases:
-                case_file = preset_files[min(len(preset_files)-1, int(num_trials+1/trials_per_case))]
+                # min(num_preset_files-1, int(float(trial_idx-1)/float(preset_trials_per_case)))
+                # TODO(ahundt) we shouldn't really be setting nonlocal_variables['trial'] here, but it is a workaround so the trials log file lines up
+                nonlocal_variables['trial'] = num_trials
+                case_file = preset_files[min(len(preset_files)-1, int(float(num_trials+1)/float(trials_per_case)))]
+                # case_file = preset_files[min(len(preset_files)-1, int(float(num_trials-1)/float(trials_per_case)))]
                 # load the current preset case, incrementing as trials are cleared
                 print('loading case file: ' + str(case_file))
                 robot.load_preset_case(case_file)