test: remove flaky test functionality

Reverts nodejs/node-v0.x-archive#8689

PR-URL: #812
Reviewed-By: Jeremiah Senkpiel <fishrock123@rocketmail.com>
Reviewed-By: Johan Bergström <bugs@bergstroem.nu>
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>

tools/test.py

@@ -57,9 +57,8 @@
 
 class ProgressIndicator(object):
 
-  def __init__(self, cases, flaky_tests_mode):
+  def __init__(self, cases):
     self.cases = cases
-    self.flaky_tests_mode = flaky_tests_mode
     self.parallel_queue = Queue(len(cases))
     self.sequential_queue = Queue(len(cases))
     for case in cases:
@@ -248,19 +247,13 @@ def HasRun(self, output):
     self._done += 1
     command = basename(output.command[-1])
     if output.UnexpectedOutput():
-      status_line = 'not ok %i - %s' % (self._done, command)
-      if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare":
-        status_line = status_line + " # TODO : Fix flaky test"
-      print status_line
+      print 'not ok %i - %s' % (self._done, command)
       for l in output.output.stderr.splitlines():
         print '#' + l
       for l in output.output.stdout.splitlines():
         print '#' + l
     else:
-      status_line = 'ok %i - %s' % (self._done, command)
-      if FLAKY in output.test.outcomes:
-        status_line = status_line + " # TODO : Fix flaky test"
-      print status_line
+      print 'ok %i - %s' % (self._done, command)
 
     duration = output.test.duration
 
@@ -278,8 +271,8 @@ def Done(self):
 
 class CompactProgressIndicator(ProgressIndicator):
 
-  def __init__(self, cases, flaky_tests_mode, templates):
-    super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
+  def __init__(self, cases, templates):
+    super(CompactProgressIndicator, self).__init__(cases)
     self.templates = templates
     self.last_status_length = 0
     self.start_time = time.time()
@@ -334,29 +327,29 @@ def PrintProgress(self, name):
 
 class ColorProgressIndicator(CompactProgressIndicator):
 
-  def __init__(self, cases, flaky_tests_mode):
+  def __init__(self, cases):
     templates = {
       'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
       'stdout': "\033[1m%s\033[0m",
       'stderr': "\033[31m%s\033[0m",
     }
-    super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
+    super(ColorProgressIndicator, self).__init__(cases, templates)
 
   def ClearLine(self, last_line_length):
     print "\033[1K\r",
 
 
 class MonochromeProgressIndicator(CompactProgressIndicator):
 
-  def __init__(self, cases, flaky_tests_mode):
+  def __init__(self, cases):
     templates = {
       'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
       'stdout': '%s',
       'stderr': '%s',
       'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"),
       'max_length': 78
     }
-    super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
+    super(MonochromeProgressIndicator, self).__init__(cases, templates)
 
   def ClearLine(self, last_line_length):
     print ("\r" + (" " * last_line_length) + "\r"),
@@ -776,8 +769,8 @@ def GetVmFlags(self, testcase, mode):
   def GetTimeout(self, mode):
     return self.timeout * TIMEOUT_SCALEFACTOR[mode]
 
-def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
-  progress = PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
+def RunTestCases(cases_to_run, progress, tasks):
+  progress = PROGRESS_INDICATORS[progress](cases_to_run)
   return progress.Run(tasks)
 
 
@@ -801,7 +794,6 @@ def BuildRequirements(context, requirements, mode, scons_flags):
 TIMEOUT = 'timeout'
 CRASH = 'crash'
 SLOW = 'slow'
-FLAKY = 'flaky'
 
 
 class Expression(object):
@@ -1248,9 +1240,6 @@ def BuildOptions():
       default=False, action="store_true")
   result.add_option("--cat", help="Print the source of the tests",
       default=False, action="store_true")
-  result.add_option("--flaky-tests",
-      help="Regard tests marked as flaky (run|skip|dontcare)",
-      default="run")
   result.add_option("--warn-unused", help="Report unused rules",
       default=False, action="store_true")
   result.add_option("-j", help="The number of parallel tasks to run",
@@ -1280,35 +1269,24 @@ def ProcessOptions(options):
   options.mode = options.mode.split(',')
   if options.J:
     options.j = multiprocessing.cpu_count()
-  def CheckTestMode(name, option):
-    if not option in ["run", "skip", "dontcare"]:
-      print "Unknown %s mode %s" % (name, option)
-      return False
-    return True
-  if not CheckTestMode("--flaky-tests", options.flaky_tests):
-    return False
   return True
 
 
 REPORT_TEMPLATE = """\
 Total: %(total)i tests
  * %(skipped)4d tests will be skipped
- * %(nocrash)4d tests are expected to be flaky but not crash
  * %(pass)4d tests are expected to pass
  * %(fail_ok)4d tests are expected to fail that we won't fix
  * %(fail)4d tests are expected to fail that we should fix\
 """
 
 def PrintReport(cases):
-  def IsFlaky(o):
-    return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o)
   def IsFailOk(o):
     return (len(o) == 2) and (FAIL in o) and (OKAY in o)
   unskipped = [c for c in cases if not SKIP in c.outcomes]
   print REPORT_TEMPLATE % {
     'total': len(cases),
     'skipped': len(cases) - len(unskipped),
-    'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]),
     'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]),
     'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]),
     'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]])
@@ -1486,15 +1464,15 @@ def Main():
 
   result = None
   def DoSkip(case):
-    return SKIP in case.outcomes or SLOW in case.outcomes or (FLAKY in case.outcomes and options.flaky_tests == "skip")
+    return SKIP in case.outcomes or SLOW in case.outcomes
   cases_to_run = [ c for c in all_cases if not DoSkip(c) ]
   if len(cases_to_run) == 0:
     print "No tests to run."
     return 1
   else:
     try:
       start = time.time()
-      if RunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
+      if RunTestCases(cases_to_run, options.progress, options.j):
         result = 0
       else:
         result = 1