Merge branch 'testerror06'

lib/sass_spec/cli.rb

@@ -69,7 +69,7 @@ def self.parse
       end
 
       opts.on("-s", "--skip", "Skip tests that fail to exit successfully") do
-        options[:skip] = true
+        # Note: --skip is no longer necessary as this is now the default behavior, since we test for errors
       end
 
       opts.on("--nuke", "Write a new expected_output for every test from whichever engine we are using") do

lib/sass_spec/engine_adapter.rb

@@ -68,10 +68,11 @@ def compile(sass_filename, style)
     require 'sass'
     begin
       captured_stderr = StringIO.new
+      # Does not work as expected when tests are run in parallel
       real_stderr, $stderr = $stderr, captured_stderr
       begin
         css_output = Sass.compile_file(sass_filename.to_s, :style => style.to_sym)
-        [css_output, captured_stderr.to_s, 0]
+        [css_output, captured_stderr.string, 0]
       rescue Sass::SyntaxError => e
         ["", "Error: " + e.message.to_s, 1]
       rescue => e

lib/sass_spec/runner.rb

@@ -39,14 +39,22 @@ def _get_cases
       glob = File.join(@options[:spec_directory], "**", input_file)
       Dir.glob(glob) do |filename|
         input = Pathname.new(filename)
+        folder = File.dirname(filename)
+        expected_stderr_file_path = File.join(folder, "error")
+        expected_status_file_path = File.join(folder, "status")
         @options[:output_styles].each do |output_style|
-          folder = File.dirname(filename)
           output_file_name = @options["#{output_style}_output_file".to_sym]
-          expected_file_path = File.join(folder, output_file_name + ".css")
+          expected_stdout_file_path = File.join(folder, output_file_name + ".css")
           clean_file_name = File.join(folder, output_file_name + ".clean")
-          if File.file?(expected_file_path) && !File.file?(expected_file_path.sub(/\.css$/, ".skip")) && filename.include?(@options[:filter])
+          if ( File.file?(expected_stdout_file_path) ) &&
+             !File.file?(expected_stdout_file_path.sub(/\.css$/, ".skip")) &&
+             filename.include?(@options[:filter]) 
             clean = File.file?(clean_file_name)
-            cases.push SassSpec::TestCase.new(input.realpath(), expected_file_path, output_style, clean, @options)
+            cases.push SassSpec::TestCase.new(input.realpath(),
+              expected_stdout_file_path,
+              expected_stderr_file_path,
+              expected_status_file_path,
+              output_style, clean, @options)
           end
         end
       end

lib/sass_spec/test.rb

@@ -6,44 +6,64 @@ def run_spec_test(test_case, options = {})
   end
 
   assert File.exists?(test_case.input_path), "Input #{test_case.input_path} file does not exist"
-  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
 
   output, clean_output, error, status = test_case.output
 
-  if status != 0 && !options[:unexpected_pass]
-    msg = "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}"
-
-    if options[:skip]
-      raise msg
+  if options[:nuke]
+    if status != 0
+      File.open(test_case.status_path, "w+") do |f|
+        f.write(status)
+        f.close
+      end
     end
 
-    puts msg
-    exit 4
-  end
-
-
-  if options[:unexpected_pass] && test_case.todo? && (test_case.expected == clean_output)
-    raise "#{test_case.input_path} passed a test we expected it to fail"
-  end
+    if error.length > 0
+      File.open(test_case.error_path, "w+") do |f|
+        f.write(error)
+        f.close
+      end
+    end
 
-  if options[:nuke]
     File.open(test_case.expected_path, "w+") do |f|
       f.write(output)
       f.close
     end
   end
 
-  if test_case.todo? && options[:unexpected_pass]
-    assert test_case.expected != clean_output, "Marked as todo and passed"
-  elsif !test_case.todo? || !options[:skip_todo]
+  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
+
+  begin
+    if test_case.should_fail?
+       # XXX Ruby returns 65 etc. SassC returns 1
+       refute_equal status, 0, "Test case should fail, but it did not"
+    else
+       assert_equal status, 0, "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}"
+    end
     assert_equal test_case.expected, clean_output, "Expected did not match output"
+    if test_case.verify_stderr?
+      # Compare only first line of error output (we can't compare stacktraces etc.)
+      error_msg = error.each_line.next.rstrip
+      expected_error_msg = test_case.expected_error.each_line.next.rstrip
+      assert_equal expected_error_msg, error_msg, "Expected did not match error"
+    end
+  rescue Minitest::Assertion
+    if test_case.todo? && options[:unexpected_pass]
+      pass
+    else
+      raise
+    end
+  else
+    if test_case.todo? && options[:unexpected_pass]
+      raise "#{test_case.input_path} passed a test we expected it to fail"
+    else
+      pass
+    end
   end
 end
 
 
 # Holder to put and run test cases
 class SassSpec::Test < Minitest::Test
-  parallelize_me!
   def self.create_tests(test_cases, options = {})
     test_cases[0..options[:limit]].each do |test_case|
       define_method('test__' << test_case.output_style + "_" + test_case.name) do

lib/sass_spec/test_case.rb

@@ -1,11 +1,17 @@
 # This represents a specific test case.
 class SassSpec::TestCase
-  def initialize(input_scss, expected_css, style, clean, options = {})
+  def initialize(input_scss, expected_css, error_file, status_file, style, clean, options = {})
     @input_path = input_scss
     @expected_path = expected_css
+    @error_path = error_file
+    @status_path = status_file
     @output_style = style
     @clean_test = clean
     @options = options
+
+    # Probe filesystem once and cache the results
+    @should_fail = File.file?(@status_path)
+    @verify_stderr = File.file?(@error_path)
   end
 
   def name
@@ -28,6 +34,22 @@ def expected_path
     @expected_path
   end
 
+  def error_path
+    @error_path
+  end
+
+  def verify_stderr?
+    @verify_stderr
+  end
+
+  def status_path
+    @status_path
+  end
+
+  def should_fail?
+    @should_fail
+  end
+
   def todo?
     @input_path.to_s.include? "todo"
   end
@@ -36,7 +58,9 @@ def output
     if @output
       return @output
     end
+
     stdout, stderr, status = engine.compile(@input_path, @output_style)
+
     if @clean_test
       cleaned = _clean_output(stdout)
     else
@@ -54,6 +78,18 @@ def expected
     end
   end
 
+  def expected_error
+    @expected_error = File.read(@error_path, :encoding => "utf-8")
+  end
+
+  def expected_status
+    if should_fail?
+      @expected_status = File.read(@status_path, :encoding => "utf-8").to_i
+    else
+      @expected_status = 0
+    end
+  end
+
   def engine
     @options[:engine_adapter]
   end