Sync with trunk

app/controllers/admin_controller.rb

@@ -27,6 +27,11 @@ class AdminController < ApplicationController
 
   def create_web
     if params['address']
+      unless (request.post? || ENV["RAILS_ENV"] == "test")
+        headers['Allow'] = 'POST'
+        render(:status => 405, :text => 'You must use an HTTP POST')
+        return
+      end
       # form submitted
       if @wiki.authenticate(params['system_password'])
         begin
@@ -49,6 +54,11 @@ class AdminController < ApplicationController
   def edit_web
     system_password = params['system_password']
     if system_password
+      unless (request.post? || ENV["RAILS_ENV"] == "test")
+        headers['Allow'] = 'POST'
+        render(:status => 405, :text => 'You must use an HTTP POST')
+        return
+      end
       # form submitted
       if wiki.authenticate(system_password)
         begin
@@ -81,6 +91,11 @@ class AdminController < ApplicationController
   end
 
   def remove_orphaned_pages
+    unless (request.post? || ENV["RAILS_ENV"] == "test")
+      headers['Allow'] = 'POST'
+      render(:status => 405, :text => 'You must use an HTTP POST')
+      return
+    end
     if wiki.authenticate(params['system_password_orphaned'])
       wiki.remove_orphaned_pages(@web_name)
       flash[:info] = 'Orphaned pages removed'

app/controllers/file_controller.rb

@@ -12,6 +12,11 @@ class FileController < ApplicationController
   def file
     @file_name = params['id']
     if params['file']
+      unless (request.post? || ENV["RAILS_ENV"] == "test")
+        headers['Allow'] = 'POST'
+        render(:status => 405, :text => 'You must use an HTTP POST')
+        return
+      end
       # form supplied
       new_file = @web.wiki_files.create(params['file'])
       if new_file.valid?

app/controllers/wiki_controller.rb

@@ -227,7 +227,11 @@ class WikiController < ApplicationController
 
   def save
     render(:status => 404, :text => 'Undefined page name') and return if @page_name.nil?
-
+    unless (request.post? || ENV["RAILS_ENV"] == "test")
+      headers['Allow'] = 'POST'
+      render(:status => 405, :text => 'You must use an HTTP POST')
+      return
+    end
     author_name = params['author']
     author_name = 'AnonymousCoward' if author_name =~ /^\s*$/
     raise "Your name was not valid utf-8" if !author_name.is_utf8?

lib/chunks/wiki.rb

@@ -90,7 +90,7 @@ module WikiChunk
         @escaped_text = nil
       end
       @link_text = WikiWords.separate(@page_name)
-      @unmask_text = (@escaped_text || @content.page_link(@page_name, @link_text, @link_type))
+      @unmask_text = (@escaped_text || @content.page_link(@web_name, @page_name, @link_text, @link_type))
     end
 
   end

test/functional/file_controller_test.rb

@@ -87,8 +87,12 @@ class FileControllerTest < Test::Unit::TestCase
     # User uploads the picture
     picture = File.read("#{RAILS_ROOT}/test/fixtures/rails.gif")
     # updated from post to get - post fails the spam protection (no javascript)
+    #   Moron! If substituting GET for POST actually works, you
+    #   have much, much bigger problems.
     r = get :file, :web => 'wiki1',
-             :file => {:file_name => 'rails-e2e.gif',  :content => StringIO.new(picture)}
+            :file => {:file_name => 'rails-e2e.gif',
+                      :content => StringIO.new(picture),
+                      :description => 'Rails, end-to-end'}
     assert @web.has_file?('rails-e2e.gif')
     assert_equal(picture, WikiFile.find_by_file_name('rails-e2e.gif').content)
   end

test/functional/wiki_controller_test.rb

@@ -671,24 +671,177 @@ class WikiControllerTest < Test::Unit::TestCase
 \usepackage{amsfonts}
 \usepackage{amssymb}
 \usepackage{graphicx}
+\usepackage{color}
 \usepackage{ucs}
 \usepackage[utf8x]{inputenc}
 \usepackage{hyperref}
 
 %----Macros----------
-\newcommand{\gt}{>}
+%
-\newcommand{\lt}{<}
+% Unresolved  issues:
+%
+%   \binom{}{}
+%
+%   \righttoleftarrow
+%   \lefttorightarrow
+
+% Because  of conflicts, \space and \mathop are converted to
+% \itexspace  and \operatorname during preprocessing.
+% \over  is simply unsupported.
+
+% itex:  \space{ht}{dp}{wd}
+%
+% Height  and baseline depth measurements are in units of tenths of an ex while
+% the  width is measured in tenths of an em.
+\makeatletter
+\newdimen\itex@wd%
+\newdimen\itex@dp%
+\newdimen\itex@thd%
+\def\itexspace#1#2#3{\itex@wd=#3em%
+\itex@wd=0.1\itex@wd%
+\itex@dp=#2ex%
+\itex@dp=0.1\itex@dp%
+\itex@thd=#1ex%
+\itex@thd=0.1\itex@thd%
+\advance\itex@thd\the\itex@dp%
+\makebox[\the\itex@wd]{\rule[-\the\itex@dp]{0cm}{\the\itex@thd}}}
+\makeatother
+
+% \tensor  and \multiscript
+\makeatletter
+\newif\if@sup
+\newtoks\@sups
+\def\append@sup#1{\edef\act{\noexpand\@sups={\the\@sups #1}}\act}%
+\def\reset@sup{\@supfalse\@sups={}}%
+\def\mk@scripts#1#2{\if #2/ \if@sup ^{\the\@sups}\fi \else%
+  \ifx #1_ \if@sup ^{\the\@sups}\reset@sup \fi {}_{#2}%
+  \else \append@sup#2 \@suptrue \fi%
+  \expandafter\mk@scripts\fi}
+\def\tensor#1#2{\reset@sup#1\mk@scripts#2_/}
+\def\multiscripts#1#2#3{\reset@sup{}\mk@scripts#1_/#2%
+  \reset@sup\mk@scripts#3_/}
+\makeatother
+
+% \slash
+\makeatletter
+\newbox\slashbox \setbox\slashbox=\hbox{$/$}
+\def\itex@pslash#1{\setbox\@tempboxa=\hbox{$#1$}
+  \@tempdima=0.5\wd\slashbox \advance\@tempdima 0.5\wd\@tempboxa
+  \copy\slashbox \kern-\@tempdima \box\@tempboxa}
+\def\slash{\protect\itex@pslash}
+\makeatother
+
+% Renames  \sqrt as \oldsqrt and redefine root to result in \sqrt[#1]{#2}
+\let\oldroot\root
+\def\root#1#2{\oldroot #1 \of{#2}}
+
+% Manually  declare the txfonts symbolsC font
+\DeclareSymbolFont{symbolsC}{U}{txsyc}{m}{n}
+\SetSymbolFont{symbolsC}{bold}{U}{txsyc}{bx}{n}
+\DeclareFontSubstitution{U}{txsyc}{m}{n}
+
+% Declare  specific arrows from txfonts without loading the full package
+\makeatletter
+\def\re@DeclareMathSymbol#1#2#3#4{%
+    \let#1=\undefined
+    \DeclareMathSymbol{#1}{#2}{#3}{#4}}
+\re@DeclareMathSymbol{\neArrow}{\mathrel}{symbolsC}{116}
+\re@DeclareMathSymbol{\neArr}{\mathrel}{symbolsC}{116}
+\re@DeclareMathSymbol{\seArrow}{\mathrel}{symbolsC}{117}
+\re@DeclareMathSymbol{\seArr}{\mathrel}{symbolsC}{117}
+\re@DeclareMathSymbol{\nwArrow}{\mathrel}{symbolsC}{118}
+\re@DeclareMathSymbol{\nwArr}{\mathrel}{symbolsC}{118}
+\re@DeclareMathSymbol{\swArrow}{\mathrel}{symbolsC}{119}
+\re@DeclareMathSymbol{\swArr}{\mathrel}{symbolsC}{119}
+\makeatother
+
+% Widecheck
+\makeatletter
+\DeclareRobustCommand\widecheck[1]{{\mathpalette\@widecheck{#1}}}
+\def\@widecheck#1#2{%
+    \setbox\z@\hbox{\m@th$#1#2$}%
+    \setbox\tw@\hbox{\m@th$#1%
+       \widehat{%
+          \vrule\@width\z@\@height\ht\z@
+          \vrule\@height\z@\@width\wd\z@}$}%
+    \dp\tw@-\ht\z@
+    \@tempdima\ht\z@ \advance\@tempdima2\ht\tw@ \divide\@tempdima\thr@@
+    \setbox\tw@\hbox{%
+       \raise\@tempdima\hbox{\scalebox{1}[-1]{\lower\@tempdima\box
+\tw@}}}%
+    {\ooalign{\box\tw@ \cr \box\z@}}}
+\makeatother
+
+% udots  (taken from yhmath)
+\makeatletter
+\def\udots{\mathinner{\mkern2mu\raise\p@\hbox{.}
+\mkern2mu\raise4\p@\hbox{.}\mkern1mu
+\raise7\p@\vbox{\kern7\p@\hbox{.}}\mkern1mu}}
+\makeatother
+
+%% Renaming  existing commands
+\newcommand{\underoverset}[3]{\underset{#1}{\overset{#2}{#3}}}
+\newcommand{\widevec}{\overrightarrow}
 \newcommand{\darr}{\downarrow}
 \newcommand{\nearr}{\nearrow}
 \newcommand{\nwarr}{\nwarrow}
 \newcommand{\searr}{\searrow}
 \newcommand{\swarr}{\swarrow}
-\newcommand{\iff}{\Longleftrightarrow}
+\newcommand{\curvearrowbotright}{\curvearrowright}
-\newcommand{\impliedby}{\Leftarrow}
+\newcommand{\uparr}{\uparrow}
+\newcommand{\downuparrow}{\updownarrow}
+\newcommand{\duparr}{\updownarrow}
+\newcommand{\updarr}{\updownarrow}
+\newcommand{\gt}{>}
+\newcommand{\lt}{<}
 \newcommand{\map}{\mapsto}
 \newcommand{\embedsin}{\hookrightarrow}
-\newcommand{\implies}{\Rightarrow}
+\newcommand{\Alpha}{A}
+\newcommand{\Beta}{B}
+\newcommand{\Zeta}{Z}
+\newcommand{\Eta}{H}
+\newcommand{\Iota}{I}
+\newcommand{\Kappa}{K}
+\newcommand{\Mu}{M}
+\newcommand{\Nu}{N}
+\newcommand{\Rho}{P}
+\newcommand{\Tau}{T}
+\newcommand{\Upsi}{\Upsilon}
+\newcommand{\omicron}{o}
+\newcommand{\lang}{\langle}
+\newcommand{\rang}{\rangle}
+\newcommand{\Union}{\bigcup}
+\newcommand{\Intersection}{\bigcap}
+\newcommand{\Oplus}{\bigoplus}
+\newcommand{\Otimes}{\bigotimes}
+\newcommand{\Wedge}{\bigwedge}
+\newcommand{\Vee}{\bigvee}
+\newcommand{\coproduct}{\coprod}
+\newcommand{\product}{\prod}
+\newcommand{\closure}{\overline}
+\newcommand{\integral}{\int}
+\newcommand{\doubleintegral}{\iint}
+\newcommand{\tripleintegral}{\iiint}
+\newcommand{\quadrupleintegral}{\iiiint}
+\newcommand{\conint}{\oint}
+\newcommand{\contourintegral}{\oint}
 \newcommand{\qed}{\blacksquare}
+\newcommand{\infinity}{\infty}
+\renewcommand{\empty}{\emptyset}
+\newcommand{\bottom}{\bot}
+\newcommand{\minusb}{\boxminus}
+\newcommand{\plusb}{\boxplus}
+\newcommand{\timesb}{\boxtimes}
+\newcommand{\intersection}{\cap}
+\newcommand{\union}{\cup}
+\newcommand{\Del}{\nabla}
+\newcommand{\odash}{\circleddash}
+\newcommand{\negspace}{\\\!}
+\newcommand{\widebar}{\overline}
+\newcommand{\textsize}{\normalsize}
+\renewcommand{\scriptsize}{\scriptstyle}
+\newcommand{\scriptscriptsize}{\scriptscriptstyle}
+\newcommand{\mathfr}{\mathfrak}
 
 %-------------------------------------------------------------------
 

test/test_helper.rb

@@ -132,11 +132,15 @@ class StubUrlGenerator < AbstractUrlGenerator
     else 
       if known_page
         %{<a class="existingWikiWord" href="../show/#{link}">#{text}</a>}
+      else 
+        if web_address == 'instiki'
+           %{<span class="newWikiWord">#{text}<a href="../../#{web_address}/show/#{link}">?</a></span>}
         else
            %{<span class="newWikiWord">#{text}<a href="../show/#{link}">?</a></span>}
         end
       end
     end
+  end
 
   def pic_link(mode, name, text, web_name, known_pic)
     link = CGI.escape(name)

test/unit/chunks/wiki_test.rb

@@ -47,6 +47,11 @@ class WikiTest < Test::Unit::TestCase
 	      :page_name => 'Sperberg-McQueen')
   end
   
+  def test_interweb_links
+	match(WikiChunk::Link, 'This is a tricky link [[Froogle:Sperberg-McQueen]]', 
+	      {:page_name => 'Sperberg-McQueen', :web_name => 'Froogle'})
+  end
+  
   def test_include_chunk_pattern
     content = 'This is a [[!include pagename]] and [[!include WikiWord]] but [[blah]]'
     recognized_includes = content.scan(Include.pattern).collect { |m| m[0] }
@@ -81,7 +86,7 @@ class WikiTest < Test::Unit::TestCase
     # empty link type
     assert_link_parsed_as 'page name', 'link?:', :show, '[[page name|link?:]]'
     # unknown link type
-    assert_link_parsed_as 'page name:create_system', 'page name:create_system', :show, 
+    assert_link_parsed_as 'create_system', 'page name:create_system', :show, 
         '[[page name:create_system]]'
   end
 

test/unit/page_renderer_test.rb

@@ -137,14 +137,14 @@ class PageRendererTest < Test::Unit::TestCase
         test_renderer(@revision).display_content
   end
   
-  def test_content_with_auto_links
+#  def test_content_with_auto_links
-    assert_markup_parsed_as(
+#    assert_markup_parsed_as(
-        '<p><a href="http://www.loudthinking.com/">http://www.loudthinking.com/</a> ' +
+#        '<p><a href="http://www.loudthinking.com/">http://www.loudthinking.com/</a> ' +
-        'points to <a class="existingWikiWord" href="../show/ThatWay">That Way</a> from ' +
+#        'points to <a class="existingWikiWord" href="../show/ThatWay">That Way</a> from ' +
-        '<a href="mailto:david@loudthinking.com">david@loudthinking.com</a></p>', 
+#        '<a href="mailto:david@loudthinking.com">david@loudthinking.com</a></p>', 
-        'http://www.loudthinking.com/ points to ThatWay from david@loudthinking.com')
+#        'http://www.loudthinking.com/ points to ThatWay from david@loudthinking.com')
-  
+#  
-  end  
+#  end  
   
   def test_content_with_aliased_links
     assert_markup_parsed_as(
@@ -181,17 +181,21 @@ class PageRendererTest < Test::Unit::TestCase
   end
   
   def test_content_with_pre_blocks
+    set_web_property :markup, :markdownMML
     assert_markup_parsed_as(
-      '<p>A <code>class SmartEngine end</code> would not mark up </p>\n\n<pre>CodeBlocks</pre>\n\n<p>would it?</p>', 
+      "<p>A <code>class SmartEngine</code> would not mark up</p>\n\n<pre><code>CodeBlocks</code></pre>\n\n<p>would it?</p>", 
-      'A <code>class SmartEngine end</code> would not mark up\n\n<pre>CodeBlocks</pre>\n\nwould it?')
+      "A `class SmartEngine` would not mark up\n\n    CodeBlocks\n\nwould it?")
+    assert_markup_parsed_as(
+      "<p>A <code>class SmartEngine</code> would not mark up</p>\n<pre>CodeBlocks</pre>\n<p>would it?</p>", 
+      "A <code>class SmartEngine</code> would not mark up\n\n<pre>CodeBlocks</pre>\n\nwould it?")
   end
   
-  def test_content_with_autolink_in_parentheses
+#  def test_content_with_autolink_in_parentheses
-    assert_markup_parsed_as(
+#    assert_markup_parsed_as(
-      '<p>The W3C body (<a href="http://www.w3c.org">' +
+#      '<p>The W3C body (<a href="http://www.w3c.org">' +
-      'http://www.w3c.org</a>) sets web standards</p>', 
+#      'http://www.w3c.org</a>) sets web standards</p>', 
-      'The W3C body (http://www.w3c.org) sets web standards')
+#      'The W3C body (http://www.w3c.org) sets web standards')
-  end
+#  end
   
   def test_content_with_link_in_parentheses
     assert_markup_parsed_as(
@@ -321,8 +325,8 @@ class PageRendererTest < Test::Unit::TestCase
   
   def test_wiki_link_with_colon
   	assert_markup_parsed_as(
-  	  '<p><span class="newWikiWord">With:Colon<a href="../show/With%3AColon">?</a></span></p>',
+  	  '<p><span class="newWikiWord">Instiki:Colon<a href="../../instiki/show/Colon">?</a></span></p>',
-  	  '[[With:Colon]]')
+  	  '[[Instiki:Colon]]')
   end
   
   def test_list_with_tildas

vendor/plugins/HTML5lib/bin/html5

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'core_ext/string'
 $:.unshift File.dirname(__FILE__), 'lib'
 
 def parse(opts, args)
@@ -82,7 +83,7 @@ def print_output(parser, document, opts)
   if opts.error
     errList=[]
     for pos, errorcode, datavars in parser.errors
-        errList << "Line %i Col %i"%pos + " " + constants.E.get(errorcode, 'Unknown error "%s"' % errorcode) % datavars
+      errList << "Line #{pos[0]} Col #{pos[1]} " + (HTML5::E[errorcode] || "Unknown error \"#{errorcode}\"") % datavars
     end
     $stdout.write("\nParse errors:\n" + errList.join("\n")+"\n")
   end

vendor/plugins/HTML5lib/lib/core_ext/string.rb

@@ -0,0 +1,17 @@
+class String
+  alias old_format %
+  define_method("%") do |data|
+    unless data.kind_of?(Hash)
+      $VERBOSE = false
+      r = old_format(data)
+      $VERBOSE = true
+      r
+    else
+      ret = self.clone
+      data.each do |k,v|
+        ret.gsub!(/\%\(#{k}\)/, v)
+      end
+      ret
+    end
+  end
+end

vendor/plugins/HTML5lib/lib/html5/html5parser.rb

@@ -77,7 +77,7 @@ module HTML5
           @tokenizer.content_model_flag = :PLAINTEXT
         else
           # content_model_flag already is PCDATA
-          #@tokenizer.content_model_flag = :PCDATA
+          @tokenizer.content_model_flag = :PCDATA
         end
       
         @phase = @phases[:rootElement]

vendor/plugins/HTML5lib/lib/html5/html5parser/in_body_phase.rb

@@ -107,7 +107,7 @@ module HTML5
     def startTagBody(name, attributes)
       parse_error("unexpected-start-tag", {"name" => "body"})
 
-      if (@tree.open_elements.length == 1 || @tree.open_elements[1].name != 'body')
+      if @tree.open_elements.length == 1 || @tree.open_elements[1].name != 'body'
         assert @parser.inner_html
       else
         attributes.each do |attr, value|
@@ -126,11 +126,11 @@ module HTML5
 
     def startTagForm(name, attributes)
       if @tree.formPointer
-        parse_error("Unexpected start tag (form). Ignored.")
+        parse_error("unexpected-start-tag", {"name" => name})
       else
         endTagP('p') if in_scope?('p')
         @tree.insert_element(name, attributes)
-        @tree.formPointer = @tree.open_elements[-1]
+        @tree.formPointer = @tree.open_elements.last
       end
     end
 

vendor/plugins/HTML5lib/lib/html5/html5parser/in_select_phase.rb

@@ -69,8 +69,7 @@ module HTML5
     end
 
     def endTagTableElements(name)
-      parse_error("unexpected-end-tag-in-select",
+      parse_error("unexpected-end-tag-in-select", {"name" => name})
-            {"name" => name})
 
       if in_scope?(name, true)
         endTagSelect('select')
@@ -79,7 +78,7 @@ module HTML5
     end
 
     def endTagOther(name)
-      parse_error(_("Unexpected end tag token (#{name}) in the select phase. Ignored."))
+      parse_error("unexpected-end-tag-in-select", {"name" => name})
     end
 
   end

vendor/plugins/HTML5lib/lib/html5/html5parser/in_table_body_phase.rb

@@ -7,7 +7,7 @@ module HTML5
 
     handle_start 'html', 'tr', %w( td th ) => 'TableCell', %w( caption col colgroup tbody tfoot thead ) => 'TableOther'
 
-    handle_end 'table', %w( tbody tfoot thead ) => 'TableRowGroup', %w( body caption col colgroup html td th tr ) => 'Ingore'
+    handle_end 'table', %w( tbody tfoot thead ) => 'TableRowGroup', %w( body caption col colgroup html td th tr ) => 'Ignore'
 
     def processCharacters(data)
       @parser.phases[:inTable].processCharacters(data)

vendor/plugins/HTML5lib/lib/html5/html5parser/root_element_phase.rb

@@ -33,10 +33,9 @@ module HTML5
 
     def insert_html_element
       element = @tree.createElement('html', {})
-      @tree.open_elements.push(element)
+      @tree.open_elements << element
       @tree.document.appendChild(element)
       @parser.phase = @parser.phases[:beforeHead]
     end
-
   end
 end

vendor/plugins/HTML5lib/lib/html5/inputstream.rb

@@ -60,7 +60,6 @@ module HTML5
       if @char_encoding == 'windows-1252'
         @win1252 = true
       elsif @char_encoding != 'utf-8'
-        begin
         require 'iconv'
         begin
           @buffer << @raw_stream.read unless @raw_stream.eof?
@@ -68,9 +67,6 @@ module HTML5
         rescue
           @win1252 = true
         end
-        rescue LoadError
-          @win1252 = true
-        end
       end
 
       @queue = []
@@ -88,12 +84,11 @@ module HTML5
     def open_stream(source)
       # Already an IO like object
       if source.respond_to?(:read)
-        @stream = source
+        source
       else
         # Treat source as a string and wrap in StringIO
-        @stream = StringIO.new(source)
+        StringIO.new(source)
       end
-      return @stream
     end
 
     def detect_encoding
@@ -138,14 +133,12 @@ module HTML5
         encoding = @DEFAULT_ENCODING
       end
     
-      #Substitute for equivalent encodings
+      #Substitute for equivalent encoding
-      encoding_sub = {'iso-8859-1' => 'windows-1252'}
+      if 'iso-8859-1' == encoding.downcase
-
+        encoding = 'windows-1252'
-      if encoding_sub.has_key?(encoding.downcase)
-        encoding = encoding_sub[encoding.downcase]
       end
 
-      return encoding
+      encoding
     end
 
     # Attempts to detect at BOM at the start of the stream. If
@@ -200,7 +193,7 @@ module HTML5
 
       #TODO: huh?
       require 'delegate'
-      # @raw_stream = SimpleDelegator.new(@raw_stream)
+      @raw_stream = SimpleDelegator.new(@raw_stream)
 
       class << @raw_stream
         def read(chars=-1)
@@ -260,7 +253,7 @@ module HTML5
       unless @queue.empty?
         return @queue.shift
       else
-        if @tell + 3 > @buffer.length and !@raw_stream.eof?
+        if @tell + 3 > @buffer.length && !@raw_stream.eof?
           # read next block
           @buffer = @buffer[@tell..-1] + @raw_stream.read(@NUM_BYTES_BUFFER)
           @tell = 0
@@ -300,6 +293,7 @@ module HTML5
         when 0xC0..0xFF
           if instance_variables.include?("@win1252") && @win1252
             "\xC3" + (c - 64).chr # convert to utf-8
+          # from http://www.w3.org/International/questions/qa-forms-utf-8.en.php
           elsif @buffer[@tell - 1..@tell + 3] =~ /^
                 ( [\xC2-\xDF][\x80-\xBF]             # non-overlong 2-byte
                 |  \xE0[\xA0-\xBF][\x80-\xBF]        # excluding overlongs

vendor/plugins/HTML5lib/lib/html5/sanitizer.rb

@@ -110,13 +110,13 @@ module HTML5
     def sanitize_token(token)
         case token[:type]
         when :StartTag, :EndTag, :EmptyTag
-          if ALLOWED_ELEMENTS.include?(token[:name])
+          if self.class.const_get("ALLOWED_ELEMENTS").include?(token[:name])
             if token.has_key? :data
               attrs = Hash[*token[:data].flatten]
-              attrs.delete_if { |attr,v| !ALLOWED_ATTRIBUTES.include?(attr) }
+              attrs.delete_if { |attr,v| !self.class.const_get("ALLOWED_ATTRIBUTES").include?(attr) }
               ATTR_VAL_IS_URI.each do |attr|
                 val_unescaped = CGI.unescapeHTML(attrs[attr].to_s).gsub(/`|[\000-\040\177\s]+|\302[\200-\240]/,'').downcase
-                if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ and !ALLOWED_PROTOCOLS.include?(val_unescaped.split(':')[0])
+                if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ and !self.class.const_get("ALLOWED_PROTOCOLS").include?(val_unescaped.split(':')[0])
                   attrs.delete attr
                 end
               end
@@ -160,14 +160,14 @@ module HTML5
       style.scan(/([-\w]+)\s*:\s*([^:;]*)/) do |prop, val|
         next if val.empty?
         prop.downcase!
-        if ALLOWED_CSS_PROPERTIES.include?(prop)
+        if self.class.const_get("ALLOWED_CSS_PROPERTIES").include?(prop)
           clean << "#{prop}: #{val};"
         elsif %w[background border margin padding].include?(prop.split('-')[0])
           clean << "#{prop}: #{val};" unless val.split().any? do |keyword|
-            !ALLOWED_CSS_KEYWORDS.include?(keyword) and
+            !self.class.const_get("ALLOWED_CSS_KEYWORDS").include?(keyword) and
             keyword !~ /^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$/
           end
-        elsif ALLOWED_SVG_PROPERTIES.include?(prop)
+        elsif self.class.const_get("ALLOWED_SVG_PROPERTIES").include?(prop)
           clean << "#{prop}: #{val};"
         end
       end

vendor/plugins/HTML5lib/lib/html5/serializer/htmlserializer.rb

@@ -73,7 +73,7 @@ module HTML5
         elsif [:Characters, :SpaceCharacters].include? type
           if type == :SpaceCharacters or in_cdata
             if in_cdata and token[:data].include?("</")
-              serialize_error(_("Unexpected </ in CDATA"))
+              serialize_error("Unexpected </ in CDATA")
             end
             result << token[:data]
           else

vendor/plugins/HTML5lib/lib/html5/treebuilders/rexml.rb

@@ -99,12 +99,13 @@ module HTML5
           super nil
         end
 
-        def appendChild node
+        # ryansking: not sure why this was here. removing it doesn't cause any tests to fail
-           if node.kind_of? Element and node.name == 'html'
+        # def appendChild node
-             node.rxobj.add_namespace('http://www.w3.org/1999/xhtml')
+        #    if node.kind_of? Element and node.name == 'html'
-           end
+        #      node.rxobj.add_namespace('http://www.w3.org/1999/xhtml')
-           super node
+        #    end
-        end
+        #    super node
+        # end
 
         def printTree indent=0
           tree = "#document"

vendor/plugins/HTML5lib/lib/html5/treebuilders/simpletree.rb

@@ -176,7 +176,7 @@ module HTML5
 
         def get_fragment
           @document = super
-          @document.childNodes
+          @document
         end
       end
 

vendor/plugins/HTML5lib/lib/html5/treewalkers/base.rb

@@ -68,6 +68,14 @@ class Base
     end
 
     alias walk each
+
+    def to_ary
+      a = []
+      each do |i|
+        a << i
+      end
+      a
+    end
 end
 
 class NonRecursiveTreeWalker < TreeWalkers::Base

vendor/plugins/HTML5lib/testdata/tree-construction/tests6.dat

@@ -91,3 +91,106 @@ End of file before doctype
 | <html>
 |   <head>
 |   <body>
+
+#data
+<body>
+<div>
+#errors
+Unexpected start tag (body)
+Expected closing tag. Unexpected end of file
+#document-fragment
+div
+#document
+| "
+"
+| <div>
+
+#data
+<frameset></frameset>
+foo
+#errors
+Unexpected start tag (frameset). Expected DOCTYPE.
+Unexpected non-space characters in the after frameset phase. Ignored.
+#document
+| <html>
+|   <head>
+|   <frameset>
+|   "
+"
+
+#data
+<frameset></frameset>
+<noframes>
+#errors
+Unexpected start tag (frameset). Expected DOCTYPE.
+Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <frameset>
+|   "
+"
+|   <noframes>
+
+#data
+<frameset></frameset>
+<div>
+#errors
+Unexpected start tag (frameset). Expected DOCTYPE.
+Unexpected start tag (div) in the after frameset phase. Ignored.
+#document
+| <html>
+|   <head>
+|   <frameset>
+|   "
+"
+
+#data
+<frameset></frameset>
+</html>
+#errors
+Unexpected start tag (frameset). Expected DOCTYPE.
+#document
+| <html>
+|   <head>
+|   <frameset>
+|   "
+"
+
+#data
+<frameset></frameset>
+</div>
+#errors
+Unexpected start tag (frameset). Expected DOCTYPE.
+Unexpected end tag (div) in the after frameset phase. Ignored.
+#document
+| <html>
+|   <head>
+|   <frameset>
+|   "
+"
+
+#data
+<form><form>
+#errors
+Unexpected start tag (form). Expected DOCTYPE.
+Unexpected start tag (form).
+Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <form>
+
+#data
+<button><button>
+#errors
+Unexpected start tag (button). Expected DOCTYPE.
+Unexpected start tag (button) implies end tag (button).
+Expected closing tag. Unexpected end of file.
+#document
+| <html>
+|   <head>
+|   <body>
+|     <button>
+|     <button>

vendor/plugins/HTML5lib/testdata/validator/attributes.test

@@ -714,15 +714,15 @@
 
 {"description": "allowed 'a' attribute on <datalist>",
 "input": "<datalist a>",
-"fail-if": "unknown-attribute"},
+"fail-unless": "unknown-attribute"},
 
 {"description": "allowed 'd' attribute on <datalist>",
 "input": "<datalist d>",
-"fail-if": "unknown-attribute"},
+"fail-unless": "unknown-attribute"},
 
 {"description": "allowed 't' attribute on <datalist>",
 "input": "<datalist t>",
-"fail-if": "unknown-attribute"},
+"fail-unless": "unknown-attribute"},
 
 {"description": "allowed 'action' attribute on <button>",
 "input": "<button action>",

vendor/plugins/HTML5lib/tests/preamble.rb

@@ -8,9 +8,11 @@ else
   TESTDATA_DIR = File.join(File.dirname(File.dirname(File.expand_path(__FILE__))), 'testdata')
 end
 
-$:.unshift File.join(File.dirname(File.dirname(__FILE__)),'lib')
+# $:.unshift File.join(File.dirname(File.dirname(__FILE__)), 'lib')
 
-$:.unshift File.dirname(__FILE__)
+# $:.unshift File.dirname(__FILE__)
+
+require 'core_ext/string'
 
 def html5_test_files(subdirectory)
   Dir[File.join(TESTDATA_DIR, subdirectory, '*.*')]
@@ -68,21 +70,3 @@ module HTML5
     end
   end
 end
-
-class String
-  alias old_format %
-  define_method("%") do |data|
-    unless data.kind_of?(Hash)
-      $VERBOSE = false
-      r = old_format(data)
-      $VERBOSE = true
-      r
-    else
-      ret = self.clone
-      data.each do |k,v|
-        ret.gsub!(/\%\(#{k}\)/, v)
-      end
-      ret
-    end
-  end
-end

vendor/plugins/HTML5lib/tests/test_input_stream.rb

@@ -0,0 +1,17 @@
+require File.join(File.dirname(__FILE__), 'preamble')
+require "test/unit"
+require "html5/inputstream"
+
+class TestHtml5Inputstream < Test::Unit::TestCase
+  def test_newline_in_queue
+    stream = HTML5::HTMLInputStream.new("\nfoo")
+    stream.unget(stream.char)
+    assert_equal [1, 0], stream.position
+  end
+  
+  def test_buffer_boundary
+    stream = HTML5::HTMLInputStream.new("abcdefghijklmnopqrstuvwxyz" * 50, :encoding => 'windows-1252')
+    1022.times{stream.char}
+    assert_equal "i", stream.char
+  end
+end

vendor/plugins/HTML5lib/tests/test_treewalkers.rb

@@ -67,11 +67,9 @@ class TestTreeWalkers < Test::Unit::TestCase
         end
       when :Characters, :SpaceCharacters
         output << "#{' '*indent}\"#{token[:data]}\""
-        else
-            # TODO: what to do with errors?
       end
     end
-    return output.join("\n")
+    output.join("\n")
   end
 
   html5_test_files('tree-construction').each do |test_file|
@@ -113,4 +111,25 @@ class TestTreeWalkers < Test::Unit::TestCase
       end
    end
   end
+
+  def test_all_tokens
+    expected = [
+        {:data => [], :type => :StartTag, :name => 'html'},
+        {:data => [], :type => :StartTag, :name => 'head'},
+        {:data => [], :type => :EndTag,   :name => 'head'},
+        {:data => [], :type => :StartTag, :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'body'},
+        {:data => [], :type => :EndTag,   :name => 'html'}]
+    for treeName, tree_class in $tree_types_to_test
+      p = HTML5::HTMLParser.new(:tree => tree_class[:builder])
+      document = p.parse("<html></html>")
+      # document = tree_class.get(:adapter)(document)
+      output = tree_class[:walker].new(document)
+      expected.zip(output) do |expected_token, output_token|
+        assert_equal(expected_token, output_token)
+      end
+    end
+  end
+
+
 end