Update to latest HTML5lib

Fix that Tokenizer bug for real this time.

vendor/plugins/HTML5lib/tests/preamble.rb

@@ -68,3 +68,21 @@ module HTML5
     end
   end
 end
+
+class String
+  alias old_format %
+  define_method("%") do |data|
+    unless data.kind_of?(Hash)
+      $VERBOSE = false
+      r = old_format(data)
+      $VERBOSE = true
+      r
+    else
+      ret = self.clone
+      data.each do |k,v|
+        ret.gsub!(/\%\(#{k}\)/, v)
+      end
+      ret
+    end
+  end
+end

vendor/plugins/HTML5lib/tests/test_encoding.rb

@@ -10,7 +10,7 @@ class Html5EncodingTestCase < Test::Unit::TestCase
     require 'rubygems'
     require 'UniversalDetector'
 
-    def test_chardet
+    def test_chardet #TODO: can we get rid of this?
       file = File.open(File.join(TESTDATA_DIR, 'encoding', 'chardet', 'test_big5.txt'), 'r')
       stream = HTML5::HTMLInputStream.new(file, :chardet => true)
       assert_equal 'big5', stream.char_encoding.downcase

vendor/plugins/HTML5lib/tests/test_parser.rb

@@ -50,8 +50,8 @@ class Html5ParserTestCase < Test::Unit::TestCase
             '', 'Recieved:', actual_output
           ].join("\n")
 
-          actual_errors = parser.errors.map do |(line, col), message|
-            'Line: %i Col: %i %s' % [line, col, message]
+          actual_errors = parser.errors.map do |(line, col), message, datavars|
+            'Line: %i Col: %i %s' % [line, col, E[message] % datavars]
           end
           assert_equal errors.length, parser.errors.length, [
             '', 'Input', input,

vendor/plugins/HTML5lib/tests/test_validator.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby -wKU
+
+require File.join(File.dirname(__FILE__), 'preamble')
+
+require 'html5'
+require 'html5/filters/validator'
+
+class TestValidator < Test::Unit::TestCase
+  def run_validator_test(test)
+    p = HTML5::HTMLParser.new(:tokenizer => HTMLConformanceChecker)
+    p.parse(test['input'])
+    errorCodes = p.errors.collect{|e| e[1]}
+    if test.has_key?('fail-if')
+      assert !errorCodes.include?(test['fail-if'])
+    end
+    if test.has_key?('fail-unless')
+      assert errorCodes.include?(test['fail-unless'])
+    end
+  end
+
+  for filename in html5_test_files('validator')
+    tests    = JSON.load(open(filename))
+    testName = File.basename(filename).sub(".test", "")
+    tests['tests'].each_with_index do |test, index|
+      define_method "test_#{testName}_#{index}" do
+        run_validator_test(test)
+      end
+    end
+  end
+end
+