Sync with latest HTML5lib

vendor/plugins/HTML5lib/tests/preamble.rb

@@ -8,9 +8,11 @@ else
   TESTDATA_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'testdata')
 end
 
-$:.unshift File.join(File.dirname(File.dirname(__FILE__)),'lib')
+# $:.unshift File.join(File.dirname(File.dirname(__FILE__)), 'lib')
 
-$:.unshift File.dirname(__FILE__)
+# $:.unshift File.dirname(__FILE__)
+
+require 'core_ext/string'
 
 def html5_test_files(subdirectory)
   Dir[File.join(TESTDATA_DIR, subdirectory, '*.*')]
@@ -42,7 +44,7 @@ module HTML5
     
       def each
         data = {}
-        key=nil
+        key = nil
         @f.each_line do |line|
           if line[0] == ?# and @sections.include?(line[1..-2])
             heading = line[1..-2]
@@ -68,21 +70,3 @@ module HTML5
     end
   end
 end
-
-class String
-  alias old_format %
-  define_method("%") do |data|
-    unless data.kind_of?(Hash)
-      $VERBOSE = false
-      r = old_format(data)
-      $VERBOSE = true
-      r
-    else
-      ret = self.clone
-      data.each do |k,v|
-        ret.gsub!(/\%\(#{k}\)/, v)
-      end
-      ret
-    end
-  end
-end

vendor/plugins/HTML5lib/tests/test_input_stream.rb

@@ -0,0 +1,17 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+require "test/unit"
+require "html5/inputstream"
+
+class TestHtml5Inputstream < Test::Unit::TestCase
+  def test_newline_in_queue
+    stream = HTML5::HTMLInputStream.new("\nfoo")
+    stream.unget(stream.char)
+    assert_equal [1, 0], stream.position
+  end
+  
+  def test_buffer_boundary
+    stream = HTML5::HTMLInputStream.new("abcdefghijklmnopqrstuvwxyz" * 50, :encoding => 'windows-1252')
+    1022.times{stream.char}
+    assert_equal "i", stream.char
+  end
+end

vendor/plugins/HTML5lib/tests/test_treewalkers.rb

@@ -47,31 +47,29 @@ class TestTreeWalkers < Test::Unit::TestCase
     indent = 0
     concatenateCharacterTokens(tokens) do |token|
       case token[:type]
-        when :StartTag, :EmptyTag
-            output << "#{' '*indent}<#{token[:name]}>"
-            indent += 2
-            for name, value in token[:data].to_a.sort
-                next if name=='xmlns'
-                output << "#{' '*indent}#{name}=\"#{value}\""
-            end
-            indent -= 2 if token[:type] == :EmptyTag
-        when :EndTag
-            indent -= 2
-        when :Comment
-            output << "#{' '*indent}<!-- #{token[:data]} -->"
-        when :Doctype
-            if token[:name] and token[:name].any?
-              output << "#{' '*indent}<!DOCTYPE #{token[:name]}>"
-            else
-              output << "#{' '*indent}<!DOCTYPE >"
-            end
-        when :Characters, :SpaceCharacters
-            output << "#{' '*indent}\"#{token[:data]}\""
+      when :StartTag, :EmptyTag
+        output << "#{' '*indent}<#{token[:name]}>"
+        indent += 2
+        for name, value in token[:data].to_a.sort
+          next if name=='xmlns'
+          output << "#{' '*indent}#{name}=\"#{value}\""
+        end
+        indent -= 2 if token[:type] == :EmptyTag
+      when :EndTag
+        indent -= 2
+      when :Comment
+        output << "#{' '*indent}<!-- #{token[:data]} -->"
+      when :Doctype
+        if token[:name] and token[:name].any?
+          output << "#{' '*indent}<!DOCTYPE #{token[:name]}>"
         else
-            # TODO: what to do with errors?
+          output << "#{' '*indent}<!DOCTYPE >"
+        end
+      when :Characters, :SpaceCharacters
+        output << "#{' '*indent}\"#{token[:data]}\""
       end
     end
-    return output.join("\n")
+    output.join("\n")
   end
 
   html5_test_files('tree-construction').each do |test_file|
@@ -113,4 +111,25 @@ class TestTreeWalkers < Test::Unit::TestCase
       end
    end
   end
+
+  def test_all_tokens
+    expected = [
+        {:data => [], :type => :StartTag, :name => 'html'},
+        {:data => [], :type => :StartTag, :name => 'head'},
+        {:data => [], :type => :EndTag,   :name => 'head'},
+        {:data => [], :type => :StartTag, :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'html'}]
+    for treeName, tree_class in $tree_types_to_test
+      p = HTML5::HTMLParser.new(:tree => tree_class[:builder])
+      document = p.parse("<html></html>")
+      # document = tree_class.get(:adapter)(document)
+      output = tree_class[:walker].new(document)
+      expected.zip(output) do |expected_token, output_token|
+        assert_equal(expected_token, output_token)
+      end
+    end
+  end
+
+
 end