Merge branch 'testerror05' into errortestmerge

lib/sass_spec/cli.rb

@@ -69,7 +69,7 @@ def self.parse
       end
 
       opts.on("-s", "--skip", "Skip tests that fail to exit successfully") do
-        options[:skip] = true
+        # Note: --skip is no longer necessary as this is now the default behavior, since we test for errors
       end
 
       opts.on("--nuke", "Write a new expected_output for every test from whichever engine we are using") do

lib/sass_spec/engine_adapter.rb

@@ -68,10 +68,11 @@ def compile(sass_filename, style)
     require 'sass'
     begin
       captured_stderr = StringIO.new
+      # Does not work as expected when tests are run in parallel
       real_stderr, $stderr = $stderr, captured_stderr
       begin
         css_output = Sass.compile_file(sass_filename.to_s, :style => style.to_sym)
-        [css_output, captured_stderr.to_s, 0]
+        [css_output, captured_stderr.string, 0]
       rescue Sass::SyntaxError => e
         ["", "Error: " + e.message.to_s, 1]
       rescue => e

lib/sass_spec/runner.rb

@@ -39,14 +39,26 @@ def _get_cases
       glob = File.join(@options[:spec_directory], "**", input_file)
       Dir.glob(glob) do |filename|
         input = Pathname.new(filename)
+        folder = File.dirname(filename)
+        expected_stderr_file_path = File.join(folder, "error")
+        expected_status_file_path = File.join(folder, "status")
         @options[:output_styles].each do |output_style|
-          folder = File.dirname(filename)
           output_file_name = @options["#{output_style}_output_file".to_sym]
-          expected_file_path = File.join(folder, output_file_name + ".css")
+          expected_stdout_file_path = File.join(folder, output_file_name + ".css")
           clean_file_name = File.join(folder, output_file_name + ".clean")
-          if File.file?(expected_file_path) && !File.file?(expected_file_path.sub(/\.css$/, ".skip")) && filename.include?(@options[:filter])
+          if ( File.file?(expected_stdout_file_path) || 
+               File.file?(expected_stderr_file_path) ||
+               File.file?(expected_status_file_path) ||
+               @options[:nuke]
+             ) &&
+             !File.file?(expected_stdout_file_path.sub(/\.css$/, ".skip")) &&
+             filename.include?(@options[:filter]) 
             clean = File.file?(clean_file_name)
-            cases.push SassSpec::TestCase.new(input.realpath(), expected_file_path, output_style, clean, @options)
+            cases.push SassSpec::TestCase.new(input.realpath(),
+              expected_stdout_file_path,
+              expected_stderr_file_path,
+              expected_status_file_path,
+              output_style, clean, @options)
           end
         end
       end

lib/sass_spec/test.rb

@@ -6,44 +6,63 @@ def run_spec_test(test_case, options = {})
   end
 
   assert File.exists?(test_case.input_path), "Input #{test_case.input_path} file does not exist"
-  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
 
   output, clean_output, error, status = test_case.output
 
-  if status != 0 && !options[:unexpected_pass]
-    msg = "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}"
+  if options[:nuke]
+    if status != 0
+      File.open(test_case.status_path, "w+") do |f|
+        f.write(status)
+        f.close
+      end
+    end
 
-    if options[:skip]
-      raise msg
+    if error.length > 0
+      File.open(test_case.error_path, "w+") do |f|
+        f.write(error)
+        f.close
+      end
     end
 
-    puts msg
-    exit 4
+    File.open(test_case.expected_path, "w+") do |f|
+      f.write(output)
+      f.close
+    end
   end
 
+  assert File.exists?(test_case.expected_path), "Expected #{test_case.expected_path} file does not exist"
 
-  if options[:unexpected_pass] && test_case.todo? && (test_case.expected == clean_output)
-    raise "#{test_case.input_path} passed a test we expected it to fail"
+  if status != 0 && !options[:unexpected_pass] && (options[:nuke] || !test_case.should_fail?)
+    msg = "Command `#{options[:engine_adapter]}` did not complete:\n\n#{error}"
+
+    raise msg
   end
 
-  if options[:nuke]
-    File.open(test_case.expected_path, "w+") do |f|
-      f.write(output)
-      f.close
-    end
+
+  if options[:unexpected_pass] && test_case.todo? && (test_case.expected == clean_output)
+    raise "#{test_case.input_path} passed a test we expected it to fail"
   end
 
   if test_case.todo? && options[:unexpected_pass]
     assert test_case.expected != clean_output, "Marked as todo and passed"
   elsif !test_case.todo? || !options[:skip_todo]
+    if test_case.should_fail?
+       # XXX Ruby returns 65 etc. SassC returns 1
+       assert status > 0, "Expected did not match status"
+    end
     assert_equal test_case.expected, clean_output, "Expected did not match output"
+    if test_case.verify_stderr?
+      # Compare only first line of error output (we can't compare stacktraces etc.)
+      error_msg = error.each_line.next.rstrip
+      expected_error_msg = test_case.expected_error.each_line.next.rstrip
+      assert_equal expected_error_msg, error_msg, "Expected did not match error"
+    end
   end
 end
 
 
 # Holder to put and run test cases
 class SassSpec::Test < Minitest::Test
-  parallelize_me!
   def self.create_tests(test_cases, options = {})
     test_cases[0..options[:limit]].each do |test_case|
       define_method('test__' << test_case.output_style + "_" + test_case.name) do

lib/sass_spec/test_case.rb

@@ -1,8 +1,10 @@
 # This represents a specific test case.
 class SassSpec::TestCase
-  def initialize(input_scss, expected_css, style, clean, options = {})
+  def initialize(input_scss, expected_css, error_file, status_file, style, clean, options = {})
     @input_path = input_scss
     @expected_path = expected_css
+    @error_path = error_file
+    @status_path = status_file
     @output_style = style
     @clean_test = clean
     @options = options
@@ -28,6 +30,22 @@ def expected_path
     @expected_path
   end
 
+  def error_path
+    @error_path
+  end
+
+  def verify_stderr?
+    File.file?(@error_path)
+  end
+
+  def status_path
+    @status_path
+  end
+
+  def should_fail?
+    File.file?(@status_path)
+  end
+
   def todo?
     @input_path.to_s.include? "todo"
   end
@@ -36,7 +54,9 @@ def output
     if @output
       return @output
     end
+
     stdout, stderr, status = engine.compile(@input_path, @output_style)
+
     if @clean_test
       cleaned = _clean_output(stdout)
     else
@@ -54,6 +74,18 @@ def expected
     end
   end
 
+  def expected_error
+    @expected_error = File.read(@error_path, :encoding => "utf-8")
+  end
+
+  def expected_status
+    if should_fail?
+      @expected_status = File.read(@status_path, :encoding => "utf-8").to_i
+    else
+      @expected_status = 0
+    end
+  end
+
   def engine
     @options[:engine_adapter]
   end

spec/maps/duplicate-keys-todo/error

@@ -0,0 +1,3 @@
+Error: Duplicate key "eta" in map (eta: 5, eta: 6).
+        on line 5 of /home/saper/sw/libsass/sass-spec/spec/maps/duplicate-keys/input.scss
+  Use --trace for backtrace.

spec/maps/duplicate-keys-todo/input.scss

@@ -0,0 +1,9 @@
+$map: (
+  alpha: 1,
+  beta: 2,
+  gamma: 3,
+  delta: (
+    eta: 5,
+    eta: 6,
+  ),
+);

spec/maps/duplicate-keys-todo/status

@@ -0,0 +1 @@
+65

spec/misc/error-directive/error

@@ -0,0 +1,3 @@
+Error: Buckle your seatbelt Dorothy, 'cause Kansas is going bye-bye
+        on line 1 of /home/saper/sw/libsass/sass-spec/spec/misc/error-directive/input.scss
+  Use --trace for backtrace.

spec/misc/error-directive/input.scss

@@ -0,0 +1 @@
+@error "Buckle your seatbelt Dorothy, 'cause Kansas is going bye-bye"

spec/misc/error-directive/status

@@ -0,0 +1 @@
+65

spec/misc/warn-directive/error

@@ -0,0 +1,3 @@
+WARNING: Don't crash the ambulance, whatever you do
+         on line 2 of /home/saper/sw/libsass/sass-spec/spec/misc/warn-directive/input.scss
+

spec/misc/warn-directive/expected.compact.css

@@ -0,0 +1 @@
+h1 { color: blue; }

spec/misc/warn-directive/expected.compressed.css

@@ -0,0 +1 @@
+h1{color:blue}

spec/misc/warn-directive/expected.expanded.css

@@ -0,0 +1,3 @@
+h1 {
+  color: blue;
+}

spec/misc/warn-directive/expected_output.css

@@ -0,0 +1,2 @@
+h1 {
+  color: blue; }

spec/misc/warn-directive/input.scss

@@ -0,0 +1,2 @@
+h1 { color: blue; } 
+@warn "Don't crash the ambulance, whatever you do"