Support hinting in tests + WWPDs

.github/workflows/main.yml

@@ -103,6 +103,7 @@ jobs:
           *       ) scripts="bin";;
         esac
         . "${VENV_DIR}/${scripts}/activate"
+        python -m pip install --upgrade pip
         python -m pip install --default-timeout=1000 -r requirements.txt
     - name: Run tests
       shell: bash

client/protocols/analytics.py

@@ -147,9 +147,7 @@ def is_correct(grading_results):
     provides the count of tests passed, failed or locked for a single
     question. Return True if all tests have passed.
     """
-    if grading_results['locked'] > 0:
-        return False
-    return sum(grading_results.values()) == grading_results['passed']
+    return grading_results['failed'] == 0 and grading_results['locked'] == 0
 
 def first_failed_test(tests, scores):
     test_names = [t.name for t in tests]

client/protocols/hinting.py

@@ -4,21 +4,18 @@
 obtains them from the hint generation server. Free response questions
 may be posed before and after hints are provided.
 """
-from client.sources.common import core
-from client.sources.common import models as sources_models
-from client.protocols.common import models as protocol_models
-from client.utils import auth
-from client.utils import format
-from client.utils import prompt
-
 import logging
 import random
+
 import requests
 
+from client.protocols.common import models as protocol_models
+from client.utils import format, prompt
 from client.utils.printer import print_error
 
 log = logging.getLogger(__name__)
 
+
 #####################
 # Hinting Mechanism #
 #####################
@@ -31,11 +28,11 @@ class HintingProtocol(protocol_models.Protocol):
     HINT_ENDPOINT = 'api/hints'
     SMALL_EFFORT = 2
     WAIT_ATTEMPTS = 5
-    SUPPORTED_ASSIGNMENTS = ['cal/cs61a/fa17/hw09', 'cal/cs61a/fa17/hw10',
-            'cal/cs61a/fa17/lab10']
+    SUPPORTED_ASSIGNMENTS = ['ok/test/su16/ex', 'cal/cs61a/fa17/hw10',
+                             'cal/cs61a/fa17/lab10']
 
     def run(self, messages):
-        """Determine if a student is elgible to recieve a hint. Based on their
+        """Determine if a student is eligible to recieve a hint. Based on their
         state, poses reflection questions.
 
         After more attempts, ask if students would like hints. If so, query
@@ -74,38 +71,31 @@ def run(self, messages):
                 continue
             stats = questions[question]
             is_solved = stats['solved'] == True
-            messages['hinting'][question] = {'prompts': {}, 'reflection': {}}
-            hint_info = messages['hinting'][question]
+            hint_info = messages['hinting']["question"] = {'name': question, 'prompts': {}, 'reflection': {}}
 
-            # Determine a users elgibility for a prompt
+            # Determine a users eligibility for a prompt
 
             # If the user just solved this question, provide a reflection prompt
             if is_solved:
-                hint_info['elgible'] = False
+                hint_info['eligible'] = False
                 hint_info['disabled'] = 'solved'
                 if self.args.hint:
                     print("This question has already been solved.")
                 continue
             elif stats['attempts'] < self.SMALL_EFFORT:
-                log.info("Question %s is not elgible: Attempts: %s, Solved: %s",
+                log.info("Question %s is not eligible: Attempts: %s, Solved: %s",
                          question, stats['attempts'], is_solved)
-                hint_info['elgible'] = False
+                hint_info['eligible'] = False
                 if self.args.hint:
                     hint_info['disabled'] = 'attempt-count'
                     print("You need to make a few more attempts before the hint system is enabled")
                     continue
             else:
                 # Only prompt every WAIT_ATTEMPTS attempts to avoid annoying user
-                if stats['attempts'] % self.WAIT_ATTEMPTS != 0:
-                    hint_info['disabled'] = 'timer'
-                    hint_info['elgible'] = False
-                    log.info('Waiting for %d more attempts before prompting',
-                             stats['attempts'] % self.WAIT_ATTEMPTS)
-                else:
-                    hint_info['elgible'] = not is_solved
+                hint_info['eligible'] = not is_solved
 
             if not self.args.hint:
-                if hint_info['elgible']:
+                if hint_info['eligible']:
                     with format.block("-"):
                         print("To get hints, try using python3 ok --hint -q {}".format(question))
                     hint_info['suggested'] = True
@@ -118,7 +108,7 @@ def run(self, messages):
                        "... (This could take up to 30 seconds)"))
                 pre_hint = random.choice(PRE_HINT_MESSAGES)
                 print("In the meantime, consider: \n{}".format(pre_hint))
-                hint_info['pre-prompt'] = pre_hint
+                hint_info['pre_hint'] = pre_hint
 
                 log.info('Prompting for hint on %s', question)
                 try:
@@ -127,29 +117,25 @@ def run(self, messages):
                     log.debug("Network error while fetching hint", exc_info=True)
                     hint_info['fetch_error'] = True
                     print_error("\r\nNetwork Error while generating hint. Try again later")
-                    response = None
                     continue
 
-                if response:
-                    hint_info['response'] = response
-
-                    hint = response.get('message')
-                    pre_prompt = response.get('pre-prompt')
-                    post_prompt = response.get('post-prompt')
-                    system_error = response.get('system-error')
-                    log.info("Hint server response: {}".format(response))
-                    if not hint:
-                        if system_error:
-                            print("{}".format(system_error))
-                        else:
-                            print("Sorry. No hints found for the current code. Try again making after some changes")
-                        continue
+                hint_info['response'] = response
+
+                hint = response.get('message')
+                post_prompt = response.get('post_prompt')
+                system_error = response.get('system_error')
+                log.info("Hint server response: {}".format(response))
+                if not hint:
+                    if system_error:
+                        print("{}".format(system_error))
+                    else:
+                        print("Sorry. No hints found for the current code. Try again making after some changes")
+                    continue
 
-                    # Provide padding for the the hint
-                    print("\n{}".format(hint.rstrip()))
+                # Provide padding for the the hint
+                print("\n{}".format(hint.rstrip()))
 
-                    if post_prompt:
-                        results['prompts'][query] = prompt.explanation_msg(post_prompt)
+                prompt.explanation_msg(post_prompt)
 
     def query_server(self, messages, test):
         user = self.assignment.get_identifier()
@@ -167,6 +153,7 @@ def query_server(self, messages, test):
         response.raise_for_status()
         return response.json()
 
+
 SOLVE_SUCCESS_MSG = [
     "If another student had the same error on this question, what advice would you give them?",
     "What did you learn from writing this program about things that you'll continue to do in the future?",
@@ -175,24 +162,24 @@ def query_server(self, messages, test):
 ]
 
 ps_strategies_messages = ("Which of the following problem solving strategies will you attempt next?\n"
-"- Manually running the code against the test cases\n"
-"- Drawing out the environment diagram\n"
-"- Try to solve the problem in another programming language and then translating\n"
-"- Ensuring that all of the types in the input/output of the function match the specification\n"
-"- Solve a few of the test cases manually and then trying to find a pattern\n"
-"- Using print statements/inspecting the value of variables to debug")
+                          "- Manually running the code against the test cases\n"
+                          "- Drawing out the environment diagram\n"
+                          "- Try to solve the problem in another programming language and then translating\n"
+                          "- Ensuring that all of the types in the input/output of the function match the specification\n"
+                          "- Solve a few of the test cases manually and then trying to find a pattern\n"
+                          "- Using print statements/inspecting the value of variables to debug")
 
 PRE_HINT_MESSAGES = [
     'Could you describe what the function you are working is supposed to do at a high level?',
-    'It would be helpful if you could explain the error to the computer:', # Rubber duck
+    'It would be helpful if you could explain the error to the computer:',  # Rubber duck
     'Try to create a hypothesis for how that output was produced. This output is produced because ...',
     'What is the simplest test that exposes this error?',
     ps_strategies_messages,
     'What type of value does the code output (a list, a number etc). ',
     'What type of value (a string, a number etc) does the test indicate is outputted?',
     'Are you convinced that the test case provided is correct?',
     'Describe how exactly the program behaves incorrectly?',
-    'In two sentences or less, explain how the error/output is produced by the code in the function', # Rubber Duck
+    'In two sentences or less, explain how the error/output is produced by the code in the function',  # Rubber Duck
     'Are there lines that you suspect could be causing the program? Why those lines?',
     'Have you tried to use print statements? On what line of the program would a print statement be useful?',
     'Where is the last place you are sure your program was correct? How do you know?',
@@ -202,5 +189,4 @@ def query_server(self, messages, test):
     ps_strategies_messages,
 ]
 
-
 protocol = HintingProtocol

client/protocols/unlock.py

@@ -41,7 +41,7 @@ def __init__(self, cmd_args, assignment):
         super().__init__(cmd_args, assignment)
         self.hash_key = assignment.name
         self.analytics = []
-        self.guidance_util = guidance.Guidance("", assignment=assignment, suppress_warning_message=True)
+        self.guidance_util = guidance.Guidance()
 
     def run(self, messages):
         """Responsible for unlocking each test.
@@ -69,9 +69,6 @@ def run(self, messages):
             log.info('Unlocking test {}'.format(test.name))
             self.current_test = test.name
 
-            # Reset guidance explanation probability for every question
-            self.guidance_util.prompt_probability = guidance.DEFAULT_PROMPT_PROBABILITY
-
             try:
                 test.unlock(self.interact)
             except (KeyboardInterrupt, EOFError):
@@ -154,16 +151,11 @@ def interact(self, unique_id, case_id, question_prompt, answer, choices=None, ra
                     break
             else:
                 correct = True
-            tg_id = -1
-            misU_count_dict = {}
             rationale = "Unknown - Default Value"
 
             if not correct:
-                guidance_data = self.guidance_util.show_guidance_msg(unique_id, input_lines,
-                                                                     self.hash_key)
-                misU_count_dict, tg_id, printed_msg, rationale = guidance_data
+                printed_msg = self.guidance_util.show_guidance_msg(unique_id, input_lines)
             else:
-                rationale = self.guidance_util.prompt_with_prob()
                 print("-- OK! --")
                 printed_msg = ["-- OK! --"]
 
@@ -175,10 +167,8 @@ def interact(self, unique_id, case_id, question_prompt, answer, choices=None, ra
                 'prompt': question_prompt,
                 'answer': input_lines,
                 'correct': correct,
-                'treatment group id': tg_id,
                 'rationale': rationale,
-                'misU count': misU_count_dict,
-                'printed msg': printed_msg
+                'printed msg': printed_msg,
             })
             print()
         return input_lines

client/sources/doctest/models.py

@@ -101,7 +101,7 @@ def run(self, env):
         if success:
             return {'passed': 1, 'failed': 0, 'locked': 0}
         else:
-            return {'passed': 0, 'failed': 1, 'locked': 0}
+            return {'passed': 0, 'failed': 1, 'locked': 0, 'failed_outputs': [''.join(output_log)]}
 
     def score(self):
         format.print_line('-')
@@ -133,4 +133,4 @@ def get_code(self):
             'teardown': '',
             }
         }
-        return data
+        return data

client/sources/ok_test/concept.py

@@ -37,8 +37,8 @@ def run(self, test_name, suite_number, env=None):
                 results['locked'] += 1
                 continue
 
-            success = self._run_case(test_name, suite_number,
-                                     case, i + 1)
+            success, output = self._run_case(test_name, suite_number,
+                                             case, i + 1)
             assert success, 'Concept case should never fail while grading'
             results['passed'] += 1
         return results

client/sources/ok_test/doctest.py

@@ -48,6 +48,7 @@ def run(self, test_name, suite_number, env=None):
         results = {
             'passed': 0,
             'failed': 0,
+            'failed_outputs': [],
             'locked': 0,
         }
 
@@ -66,7 +67,7 @@ def run(self, test_name, suite_number, env=None):
                 results['locked'] += 1
                 continue
 
-            success = self._run_case(test_name, suite_number,
+            success, output = self._run_case(test_name, suite_number,
                                      case, i + 1)
             if not success and self.interactive:
                 self.console.interact()
@@ -75,6 +76,7 @@ def run(self, test_name, suite_number, env=None):
                 results['passed'] += 1
             else:
                 results['failed'] += 1
+                results['failed_outputs'].append(output)
 
             if not success and not self.verbose:
                 # Stop at the first failed test

client/sources/ok_test/models.py

@@ -60,7 +60,7 @@ def run(self, env):
             'locked': int,
         }
         """
-        passed, failed, locked = 0, 0, 0
+        passed, failed, locked, failed_outputs = 0, 0, 0, []
         for i, suite in enumerate(self.suites):
             if self.run_only and self.run_only != i + 1:
                 continue
@@ -72,6 +72,7 @@ def run(self, env):
             passed += results['passed']
             failed += results['failed']
             locked += results['locked']
+            failed_outputs += results.get('failed_outputs', [])
 
             if not self.verbose and (failed > 0 or locked > 0):
                 # Stop at the first failed test
@@ -90,6 +91,7 @@ def run(self, env):
             'passed': passed,
             'failed': failed,
             'locked': locked,
+            'failed_outputs': failed_outputs,
         }
 
     def score(self, env=None):
@@ -308,12 +310,14 @@ def _run_case(self, test_name, suite_number, case, case_number):
         output_log = output.get_log(log_id)
         output.remove_log(log_id)
 
+        output_str = ''.join(output_log)
+
         if not success or self.verbose:
-            print(''.join(output_log))
+            print(output_str)
         if not success:
             short_name = self.test.get_short_name()
             # TODO: Change when in notebook mode
             print('Run only this test case with '
                 '"python3 ok -q {} --suite {} --case {}"'.format(
                     short_name, suite_number, case_number))
-        return success
+        return success, output_str

client/sources/ok_test/wwpp.py

@@ -38,7 +38,7 @@ def run(self, test_name, suite_number, env=None):
                 results['locked'] += 1
                 continue
 
-            success = self._run_case(test_name, suite_number,
+            success, output = self._run_case(test_name, suite_number,
                                      case, i + 1)
             assert success, 'Wwpp case should never fail while grading'
             results['passed'] += 1