require 'minitest'
require 'thread'
require 'fileutils'
require 'yaml'
require_relative "interactor"
require_relative "util"
# Holder to put and run test cases
class SassSpec::Test < Minitest::Test
def self.create_tests(test_cases, options = {})
options[:limit] ||= test_cases.length + 1
test_cases[0...options[:limit]].each do |test_case|
define_method("test__#{test_case.name}") do
runner = SassSpecRunner.new(test_case, options)
test_case.finalize(runner.run)
self.assertions += runner.assertions
end
end
end
end
class SassSpecRunner
include MiniTest::Assertions
attr_accessor :assertions
@@interaction_memory = {}
def initialize(test_case, options = {})
@assertions = 0
@test_case = test_case
@options = options
end
def run
if @test_case.todo? && !@options[:run_todo]
skip "Skipped #{@test_case.name}"
end
unless @test_case.dir.hrx?
@test_case.dir.glob("*").each {|p| assert_filename_length!(File.join(@test_case.dir.path, p))}
end
@output, @error, @status = @test_case.run(@options[:engine_adapter])
@output = @output.gsub(/\r\n/, "\n")
@normalized_output = SassSpec::Util.normalize_output(@output)
@error = SassSpec::Util.normalize_error(@error)
if @options[:generate]
overwrite_test!
return true
end
# Allow checks to throw `:done` to indicate that no more checks need to be
# performed. We throw rather than returning a boolean so that we can do
# checks in nested functions without worrying about piping return values.
catch :done do
check_annotations!
handle_conflicting_files!
handle_missing_output!
handle_unexpected_error!
handle_unexpected_pass!
handle_output_difference!
# Run these checks last because `handle_stderr_difference!` can skip the
# test if `@test_case.warning_todo?` is set, and we only want to check for
# an unnecessary TODO if the test isn't skipped because of a TODO.
handle_stderr_difference!
handle_unnecessary_todo!
end
return true
end
private
## Failure handlers
def check_annotations!
return unless @options[:check_annotations]
ignored_warning_impls = @test_case.metadata.warnings_ignored_for
if ignored_warning_impls.any? && @error.empty?
message = "No warning issued, but warnings are ignored for #{ignored_warning_impls.join(', ')}"
choice = interact(:ignore_warning_nonexistant, :fail) do |i|
i.prompt message
i.choice('R', "Remove ignored status for #{ignored_warning_impls.join(', ')}") do
change_options(remove_ignore_warning_for: ignored_warning_impls)
end
fail_or_exit_choice(i)
end
assert choice != :fail, message
end
todo_warning_impls = @test_case.metadata.all_warning_todos
if todo_warning_impls.any? && @error.length == 0
message = "No warning issued, but warnings are pending for #{todo_warning_impls.join(', ')}"
choice = interact(:todo_warning_nonexistant, :fail) do |i|
i.prompt message
i.choice('R', "Remove TODO status for #{todo_warning_impls.join(', ')}") do
change_options(remove_warning_todo: todo_warning_impls)
end
fail_or_exit_choice(i)
end
assert choice != :fail, message
end
end
def handle_conflicting_files!
if @test_case.file?("error", impl: true)
impl = true
elsif @test_case.file?("error")
impl = false
else
return
end
output_file_exists = @test_case.file?("output.css", impl: impl)
warning_file_exists = @test_case.file?("warning", impl: impl)
return unless output_file_exists || warning_file_exists
choice = interact(:conflicting_files, :fail) do |i|
i.prompt "Test has both error and success outputs."
show_test_case_choice(i)
show_output_choice(i)
delete_choice(i)
i.choice('S', 'Keep the success output.') do
@test_case.delete("error", impl: :auto)
throw :done
end
i.choice('E', 'Keep the error output.') do
@test_case.delete("output.css") if output_file_exists
@test_case.delete("warning") if warning_file_exists
throw :done
end
migrate_warning_choice(i) unless warning_file_exists
update_output_choice(i)
fail_or_exit_choice(i)
end
assert choice != :fail, "Expected #{@test_case.expected_path} file does not exist"
end
def handle_missing_output!
return if @test_case.should_fail? || @test_case.expected
skip_test_case!("TODO test is failing") if probe_todo?
choice = interact(:missing_output, :fail) do |i|
i.prompt "Expected output file does not exist."
show_test_case_choice(i)
show_output_choice(i)
delete_choice(i)
update_output_choice(i)
fail_or_exit_choice(i)
end
assert choice != :fail, "Expected output.css file does not exist"
end
def handle_unexpected_error!
return if @status == 0 || @test_case.should_fail?
if !@options[:interactive] && @options[:migrate_impl]
migrate_impl!
throw :done
end
skip_test_case!("TODO test is failing") if @test_case.todo?
choice = interact(:unexpected_error, :fail) do |i|
i.prompt "An unexpected compiler error was encountered."
show_test_case_choice(i)
i.choice('e', "Show me the error.") do
display_text_block(@error)
i.restart!
end
update_output_choice(i)
migrate_impl_choice(i)
todo_choice(i)
ignore_choice(i)
fail_or_exit_choice(i)
end
throw :done unless choice == :fail
assert_equal 0, @status,
"Command `#{@options[:engine_adapter]}` did not complete:\n\n#{@error}"
end
def handle_unexpected_pass!
return unless @status == 0 && @test_case.should_fail?
if !@options[:interactive] && @options[:migrate_impl]
migrate_impl!
throw :done
end
skip_test_case!("TODO test is failing") if probe_todo?
choice = interact(:unexpected_pass, :fail) do |i|
i.prompt "A failure was expected but it compiled instead."
show_test_case_choice(i)
i.choice('o', "Show me the output.") do
display_text_block(@output)
i.restart!
end
migrate_warning_choice(i)
update_output_choice(i)
migrate_impl_choice(i)
todo_choice(i)
fail_or_exit_choice(i)
end
throw :done unless choice == :fail
refute_equal @status, 0, "Test case should fail, but it did not"
end
def handle_output_difference!
return if @test_case.should_fail? || @normalized_output == @test_case.expected
if !@options[:interactive] && @options[:migrate_impl]
migrate_impl!
throw :done
end
skip_test_case!("TODO test is failing") if probe_todo?
interact(:output_difference, :fail) do |i|
i.prompt "Output does not match expectation."
show_test_case_choice(i)
i.choice('d', "show diff.") do
require 'diffy'
display_text_block(
Diffy::Diff.new("Expected\n" + @test_case.expected,
"Actual\n" + @normalized_output).to_s(:color))
i.restart!
end
update_output_choice(i)
migrate_impl_choice(i)
todo_choice(i)
ignore_choice(i)
fail_or_exit_choice(i)
end
assert_equal @test_case.expected, @normalized_output, "expected did not match output"
end
def handle_unnecessary_todo!
return if probe_todo? && !@options[:interactive]
return unless @test_case.todo? || @test_case.warning_todo?
interact(:unnecessary_todo, :fail) do |i|
i.prompt "Test is passing but marked as TODO."
show_test_case_choice(i)
unless @output.empty?
i.choice('o', "Show me the output.") do
display_text_block(@output)
i.restart!
end
end
i.choice('R', "Remove TODO status for #{@test_case.impl}.") do
change_options(remove_todo: [@test_case.impl], remove_warning_todo: [@test_case.impl])
throw :done
end
i.choice('f', "Mark as skipped.") do
skip_test_case!("TODO test is passing")
end
i.choice('X', "Exit testing.") do
raise Interrupt
end
end
assert_equal @test_case.expected, @normalized_output, "expected did not match output"
end
def handle_stderr_difference!
unless @test_case.should_fail?
if @test_case.ignore_warning?
return
elsif @test_case.warning_todo? && !@options[:run_todo]
skip_test_case! "Skipped warning check for #{@test_case.name}"
end
end
error_msg = extract_error_message(@error)
expected_error_msg = extract_error_message(
@test_case.expected_error || @test_case.expected_warning)
return if expected_error_msg == error_msg
skip_test_case!("TODO test is failing") if probe_todo?
message =
if error_msg.empty?
if @test_case.should_fail?
"An error message was expected but wasn't produced."
else
"A warning was expected but wasn't produced."
end
elsif expected_error_msg.empty?
"An unexpected warning was produced."
else
if @test_case.should_fail?
"Error message doesn't match the expected error."
else
"Warning doesn't match the expected warning."
end
end
type = @test_case.should_fail? ? :expected_error_different : :expected_warning_different
interact(type, :fail) do |i|
i.prompt(message)
show_test_case_choice(i)
unless error_msg.empty?
i.choice('e', "Show #{@test_case.should_fail? ? 'error' : 'warning'}.") do
display_text_block(error_msg)
i.restart!
end
end
i.choice('d', "Show diff.") do
require 'diffy'
display_text_block(
Diffy::Diff.new("Expected\n#{expected_error_msg}",
"Actual\n#{error_msg}").to_s(:color))
i.restart!
end
update_output_choice(i)
migrate_impl_choice(i)
todo_warning_choice(i)
ignore_warning_choice(i)
fail_or_exit_choice(i)
end
assert_equal expected_error_msg, error_msg, message
end
## Interaction utilities
# If the runner is running in interactive mode, runs the interaction defined
# in the block and returns the result. Otherwise, just returns the default
# value.
def interact(prompt_id, default, &block)
if @options[:interactive]
print "\nIn test case: #{@test_case.name}"
return SassSpec::Interactor.interact_with_memory(@@interaction_memory, prompt_id, &block)
else
return default
end
end
def show_test_case_choice(i)
i.choice('t', "Show me the test case.") do
display_text_block(@test_case.dir.to_hrx)
i.restart!
end
end
def show_output_choice(i)
if @status == 0
i.choice('o', "Show me the output generated.") do
display_text_block(@output)
i.restart!
end
if @error.length > 0
i.choice('e', "Show me the warning generated.") do
display_text_block(@error)
i.restart!
end
end
else
i.choice('e', "Show me the error generated.") do
display_text_block(@error)
i.restart!
end
end
end
def migrate_warning_choice(i)
i.choice('W', "Migrate the error to a warning.") do
@test_case.rename("error", "warning")
throw :done
end
end
def update_output_choice(i)
i.choice('O', "Update expected output and pass test.") do
overwrite_test!
throw :done
end
end
def migrate_impl_choice(i)
i.choice('I', "Migrate copy of test to pass on #{@test_case.impl}.") do
migrate_impl! || i.restart!
throw :done
end
end
def todo_choice(i)
return if @test_case.todo?
i.choice('T', "Mark spec as todo for #{@test_case.impl}.") do
change_options(add_todo: [@test_case.impl])
throw :done
end
end
def ignore_choice(i)
i.choice('G', "Ignore test for #{@test_case.impl} FOREVER.") do
change_options(
add_ignore_for: [@test_case.impl],
remove_warning_todo: [@test_case.impl],
remove_todo: [@test_case.impl])
throw :done
end
end
def delete_choice(i)
i.choice('D', "Delete test.") do
if delete_test!
throw :done
else
i.restart!
end
end
end
def todo_warning_choice(i)
return if @test_case.warning_todo?
i.choice('T', "Mark warning as todo for #{@test_case.impl}.") do
change_options(add_warning_todo: [@test_case.impl])
throw :done
end
end
def ignore_warning_choice(i)
i.choice('G', "Ignore warning for #{@test_case.impl}.") do
change_options(add_ignore_warning_for: [@test_case.impl])
throw :done
end
end
def fail_or_exit_choice(i)
i.choice('f', "Mark as failed.")
i.choice('X', "Exit testing.") do
raise Interrupt
end
end
# Deletes the current test case.
#
# In interactive mode, prompts the user and returns `false` if they decide not
# to delete the test.
def delete_test!
result = interact(:delete_test, :proceed) do |i|
files = @test_case.dir.glob("**/*")
i.prompt("The following files will be removed:\n * " + files.join("\n * "))
i.choice('D', "Delete them.") do
@test_case.dir.delete_dir!
end
i.choice('x', "I changed my mind.") {}
end
result == :proceed || result == "D"
end
# Adds separate outputs for the test that are compatible with the current
# implementation.
def migrate_impl!
if @status == 0
if @test_case.expected != @normalized_output || @test_case.should_fail?
@test_case.write("output.css", @output, impl: true)
end
if extract_error_message(@test_case.expected_warning) != extract_error_message(@error)
@test_case.write("warning", @error, impl: true)
end
else
actual_error = @test_case.expected_error && extract_error_message(@test_case.expected_error)
if actual_error != extract_error_message(@error)
@test_case.write("error", @error, impl: true)
end
end
change_options(remove_warning_todo: [@test_case.impl], remove_todo: [@test_case.impl])
end
## Other utilities
# Returns whether the current test case is marked as TODO, but is still being
# run because --probe-todo was passed. These specs shouldn't produce errors
# when they fail.
def probe_todo?
@options[:probe_todo] && (@test_case.todo? || @test_case.warning_todo?)
end
def skip_test_case!(reason = nil)
msg = "Skipped #{@test_case.name}"
if reason
msg << ": #{reason}"
else
msg << "."
end
skip msg
end
ANSI_ESCAPES = /[\u001b\u009b][\[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/
def display_text_block(text)
if text.empty?
puts "*" * 20 + " (empty) " + "*" * 20
return
end
delim = "*" * text.gsub(ANSI_ESCAPES, '').lines.map{|l| l.rstrip.size}.max
puts delim
puts text
puts delim
end
def overwrite_test!
if @status == 0
@test_case.write("output.css", @output, impl: :auto)
@test_case.delete("error", if_exists: true, impl: :auto)
if @error.empty?
@test_case.delete("warning", if_exists: true, impl: :auto)
else
@test_case.write("warning", @error, impl: :auto)
end
else
@test_case.write("error", @error, impl: :auto)
@test_case.delete("output.css", if_exists: true, impl: :auto)
@test_case.delete("warning", if_exists: true, impl: :auto)
end
change_options(remove_warning_todo: [@test_case.impl], remove_todo: [@test_case.impl])
end
def change_options(new_options)
existing_options = if @test_case.file?("options.yml")
YAML.load(@test_case.read("options.yml"))
else
{}
end
existing_options = SassSpec::TestCaseMetadata.merge_options(existing_options, new_options)
if existing_options.any?
@test_case.write("options.yml", existing_options.to_yaml)
else
@test_case.delete("options.yml", if_exists: true)
end
end
GEMFILE_PREFIX_LENGTH = 68
# When running sass-spec as a gem from github very long filenames can cause
# installation issues. This checks that the paths in use will work.
def assert_filename_length!(filename)
name = relative_name = filename.to_s.sub(SassSpec::SPEC_DIR, "")
assert false, "Filename #{name} must no more than #{256 - GEMFILE_PREFIX_LENGTH} characters long" if name.size > (256 - GEMFILE_PREFIX_LENGTH)
if name.size <= 100 then
prefix = ""
else
parts = name.split(/\//)
newname = parts.pop
nxt = ""
loop do
nxt = parts.pop
break if newname.size + 1 + nxt.size > 100
newname = nxt + "/" + newname
end
prefix = (parts + [nxt]).join "/"
name = newname
assert false, "base name (#{name}) of #{relative_name} must no more than 100 characters long" if name.size > 100
assert false, "prefix (#{prefix}) of #{relative_name} must no more than #{155 - GEMFILE_PREFIX_LENGTH} characters long" if prefix.size > (155 - GEMFILE_PREFIX_LENGTH)
end
return nil
end
def extract_error_message(error)
# We want to make sure dart-sass continues to generate correct stack traces
# and text highlights, so we check its full error messages.
if @options[:engine_adapter].describe == 'dart-sass'
return clean_debug_path(error.rstrip)
end
error_message = ""
consume_next_line = false
error.each_line do |line|
if consume_next_line
next if line.strip == ""
error_message += line
break
end
if (line =~ /(DEPRECATION )?WARNING/)
if line.rstrip.end_with?(":")
error_message = line.rstrip + "\n"
consume_next_line = true
next
else
error_message = line
break
end
end
if (line =~ /Error:/)
error_message = line
break
end
end
clean_debug_path(error_message.rstrip)
end
def clean_debug_path(error)
error.gsub(/^.*?(input.scss:\d+ DEBUG:)/, '\1')
.gsub(/\/+/, "/")
.gsub(/^#{Regexp.quote(SassSpec::SPEC_DIR.gsub(/\\/, '\/'))}\//, "/sass/sass-spec/")
.gsub(/^#{Regexp.quote(SassSpec::SPEC_DIR)}\//, "/sass/sass-spec/")
.gsub(/(?:\/todo_|_todo\/)/, "/")
.gsub(/\/libsass\-[a-z]+\-tests\//, "/")
.gsub(/\/libsass\-[a-z]+\-issues/, "/libsass-issues")
.strip
end
end