Greek &phis; φ, double-struck 𝔸, numeric 𝔸 ⁗, uppercase ™ <
' + output = "Greek \317\225 \317\206, double-struck \360\235\224\270, numeric \360\235\224\270 \342\201\227, uppercase \342\204\242 <
" + output2 = "Greek \317\225 \317\206, double-struck \360\235\224\270, numeric 𝔸 ⁗, uppercase \342\204\242 <
" + assert_equal(output, sanitize_xhtml(input)) + assert_equal(output, sanitize_html(input)) + assert_equal(output, my_rex(input)) + assert_equal(output2, input.to_utf8) + end + + def test_sanitize_malformed_utf8 + input = "\357elephant & \302ivory
" + output = "\357\277\275elephant & \357\277\275ivory
" + check_sanitization(input, output, output, output) + end + + Sanitizer::ALLOWED_ELEMENTS.each do |tag_name| define_method "test_should_allow_#{tag_name}_tag" do - assert_equal "<#{tag_name} title=\"1\">foo <bad>bar</bad> baz", - sanitize_html("<#{tag_name} title='1'>foo' if tag_name == 'br' + rexmloutput = "<#{tag_name} title='1' />" + end + check_sanitization(input, xhtmloutput, xhtmloutput, rexmloutput) end end - Sanitize::ALLOWED_ELEMENTS.each do |tag_name| + Sanitizer::ALLOWED_ELEMENTS.each do |tag_name| define_method "test_should_forbid_#{tag_name.upcase}_tag" do - assert_equal "<#{tag_name.upcase} title=\"1\">foo <bad>bar</bad> baz</#{tag_name.upcase}>", - sanitize_html("<#{tag_name.upcase} title='1'>foo
foo <bad>bar</bad> baz
", - sanitize_html("foo
foo
foo <bad>bar</bad> baz
" + htmloutput = "foo <bad>bar</bad> baz
" + check_sanitization(input, output, output, output) + end + end + + Sanitizer::ALLOWED_ATTRIBUTES.each do |attribute_name| + define_method "test_should_forbid_#{attribute_name.upcase}_attribute" do + input = "foo
foo <bad>bar</bad> baz
" + check_sanitization(input, output, output, output) + end + end + + Sanitizer::ALLOWED_PROTOCOLS.each do |protocol| + define_method "test_should_allow_#{protocol}_uris" do + input = %(foo) + output = "foo" + check_sanitization(input, output, output, output) + end + end + + Sanitizer::ALLOWED_PROTOCOLS.each do |protocol| + define_method "test_should_allow_uppercase_#{protocol}_uris" do + input = %(foo) + output = "foo" + check_sanitization(input, output, output, output) + end + end + + Sanitizer::SVG_ALLOW_LOCAL_HREF.each do |tag_name| + next unless Sanitizer::ALLOWED_ELEMENTS.include?(tag_name) + define_method "test_#{tag_name}_should_allow_local_href_with_ns_decl" do + input = %(<#{tag_name} xlink:href="#foo" xmlns:xlink='http://www.w3.org/1999/xlink'/>) + output = "<#{tag_name.downcase} xlink:href='#foo' xmlns:xlink='http://www.w3.org/1999/xlink'/>" + xhtmloutput = "<#{tag_name} xlink:href='#foo' xmlns:xlink='http://www.w3.org/1999/xlink'/>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + + define_method "test_#{tag_name}_should_allow_local_href_with_newline_and_ns_decl" do + input = %(<#{tag_name} xlink:href="\n#foo" xmlns:xlink='http://www.w3.org/1999/xlink'/>) + output = "<#{tag_name.downcase} xlink:href='\n#foo' xmlns:xlink='http://www.w3.org/1999/xlink'/>" + xhtmloutput = "<#{tag_name} xlink:href='\n#foo' xmlns:xlink='http://www.w3.org/1999/xlink'/>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + + define_method "test_#{tag_name}_should_forbid_local_href_without_ns_decl" do + input = %(<#{tag_name} xlink:href="#foo"/>) + output = "<#{tag_name.downcase} xlink:href='#foo'/>" + xhtmloutput = "<#{tag_name} xlink:href='#foo'></#{tag_name}>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + + define_method "test_#{tag_name}_should_forbid_local_href_with_newline_without_ns_decl" do + input = %(<#{tag_name} xlink:href="\n#foo"/>) + output = "<#{tag_name.downcase} xlink:href='\n#foo'/>" + xhtmloutput = "<#{tag_name} xlink:href='\n#foo'></#{tag_name}>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + + define_method "test_#{tag_name}_should_forbid_nonlocal_href_with_ns_decl" do + input = %(<#{tag_name} xlink:href="http://bad.com/foo" xmlns:xlink='http://www.w3.org/1999/xlink'/>) + output = "<#{tag_name.downcase} xmlns:xlink='http://www.w3.org/1999/xlink'/>" + xhtmloutput = "<#{tag_name} xmlns:xlink='http://www.w3.org/1999/xlink'/>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + + define_method "test_#{tag_name}_should_forbid_nonlocal_href_with_newline_and_ns_decl" do + input = %(<#{tag_name} xlink:href="\nhttp://bad.com/foo" xmlns:xlink='http://www.w3.org/1999/xlink'/>) + output = "<#{tag_name.downcase} xmlns:xlink='http://www.w3.org/1999/xlink'/>" + xhtmloutput = "<#{tag_name} xmlns:xlink='http://www.w3.org/1999/xlink'/>" + check_sanitization(input, xhtmloutput, xhtmloutput, xhtmloutput) + end + end + + def test_should_handle_astral_plane_characters + input = "𝒵 𝔸
" + output = "\360\235\222\265 \360\235\224\270
" + check_sanitization(input, output, output, output) + + input = "foo <bad>bar</bad> baz
", - sanitize_html("foo
foo
',
- sanitize_html(%(foo
))
- end
-
- def test_div_expression
- assert_equal 'foo
',
- sanitize_html(%(foo
))
- end
-
- def test_img_vbscript
- assert_equal '#{sanitized}
")
- sanitized = doc.to_s.gsub(/\A(.*)<\/div>\Z/m, '\1')
- rescue REXML::ParseException
- sanitized = sanitized.escapeHTML
- end
-
-# Sanitize a string, parsed using HTML parsing rules.
-#
-# :call-seq:
-# sanitize_html( string ) -> string
-# sanitize_html( string, {:encoding => 'iso-8859-1', :to_tree => true} ) -> REXML::Document
-#
-# Unless otherwise specified, the string is assumed to be utf-8 encoded.
-# By default, the output is a string. But, optionally, you can return a REXML tree.
-#
-# The string returned is utf-8 encoded. If you want, you can use iconv to convert it to some other encoding.
-# (REXML trees are always utf-8 encoded.)
- def sanitize_html(html, options = {})
- @encoding = 'utf-8'
- @treebuilder = TreeBuilders::REXML::TreeBuilder
- @to_tree = false
- options.each do |name, value|
- next unless %w(encoding treebuilder to_tree).include? name.to_s
- if name.to_s == 'treebuilder'
- @treebuilder = HTML5lib::TreeBuilders.get_tree_builder(value)
- else
- instance_variable_set("@#{name}", value)
- end
- end
- if @encoding == 'utf-8'
- parsed = HTMLParser.parse_fragment(html.to_utf8, {:tokenizer => HTMLSanitizer,
- :encoding => @encoding, :tree => @treebuilder })
- else
- parsed = HTMLParser.parse_fragment(html.to_ncr, {:tokenizer => HTMLSanitizer,
- :encoding => @encoding, :tree => @treebuilder })
- end
- return parsed if @to_tree
- return parsed.to_s
- end
-
-# Sanitize a REXML tree. The output is a string.
-#
-# :call-seq:
-# sanitize_rexml(tree) -> string
-#
- def sanitize_rexml(tree)
- tokens = TreeWalkers.get_tree_walker('rexml2').new(tree)
- XHTMLSerializer.serialize(tokens, {:encoding=>'utf-8',
- :space_before_trailing_solidus => true,
- :inject_meta_charset => false,
- :sanitize => true})
- end
-end
-
-require 'rexml/element'
-module REXML #:nodoc:
- class Element
-
-# Convert XHTML+MathML Named Entities in a REXML::Element to Numeric Character References
-#
-# :call-seq:
-# tree.to_ncr -> REXML::Element
-#
-# REXML, typically, converts NCRs to utf-8 characters, which is what you'll see when you
-# access the resulting REXML document.
-#
-# Note that this method needs to traverse the entire tree, converting text nodes and attributes
-# for each element. This can be SLOW. It will often be faster to serialize to a string and then
-# use String.to_ncr instead.
-#
- def to_ncr
- self.each_element { |el|
- el.texts.each_index {|i|
- el.texts[i].value = el.texts[i].to_s.to_ncr
- }
- el.attributes.each { |name,val|
- el.attributes[name] = val.to_ncr
- }
- el.to_ncr if el.has_elements?
- }
- return self
- end
-
-# Convert XHTML+MathML Named Entities in a REXML::Element to UTF-8
-#
-# :call-seq:
-# tree.to_utf8 -> REXML::Element
-#
-# Note that this method needs to traverse the entire tree, converting text nodes and attributes
-# for each element. This can be SLOW. It will often be faster to serialize to a string and then
-# use String.to_utf8 instead.
-#
- def to_utf8
- self.each_element { |el|
- el.texts.each_index {|i|
- el.texts[i].value = el.texts[i].to_s.to_utf8
- }
- el.attributes.each { |name,val|
- el.attributes[name] = val.to_utf8
- }
- el.to_utf8 if el.has_elements?
- }
- return self
- end
-
- end
-end
-
-module HTML5 #:nodoc: all
- module TreeWalkers
-
- private
-
- class << self
- def [](name)
- case name.to_s.downcase
- when 'rexml'
- require 'html5/treewalkers/rexml'
- REXML::TreeWalker
- when 'rexml2'
- REXML2::TreeWalker
- else
- raise "Unknown TreeWalker #{name}"
- end
- end
-
- alias :get_tree_walker :[]
- end
-
- module REXML2
- class TreeWalker < HTML5::TreeWalkers::NonRecursiveTreeWalker
-
- private
-
- def node_details(node)
- case node
- when ::REXML::Document
- [:DOCUMENT]
- when ::REXML::Element
- if !node.name
- [:DOCUMENT_FRAGMENT]
- else
- [:ELEMENT, node.name,
- node.attributes.map {|name,value| [name,value.to_utf8]},
- node.has_elements? || node.has_text?]
- end
- when ::REXML::Text
- [:TEXT, node.value.to_utf8]
- when ::REXML::Comment
- [:COMMENT, node.string]
- when ::REXML::DocType
- [:DOCTYPE, node.name, node.public, node.system]
- when ::REXML::XMLDecl
- [nil]
- else
- [:UNKNOWN, node.class.inspect]
- end
- end
-
- def first_child(node)
- node.children.first
- end
-
- def next_sibling(node)
- node.next_sibling
- end
-
- def parent(node)
- node.parent
- end
- end
- end
- end
-end
diff --git a/lib/sanitizer.rb b/lib/sanitizer.rb
index 8537fed2..d26d99ae 100644
--- a/lib/sanitizer.rb
+++ b/lib/sanitizer.rb
@@ -169,7 +169,7 @@ module Sanitizer
node.attributes.delete attr; next
end
if ATTR_VAL_IS_URI.include?(attr)
- val_unescaped = val.unescapeHTML.gsub(/`|[\000-\040\177\s]+|\302[\200-\240]/,'').downcase
+ val_unescaped = val.unescapeHTML.as_bytes.gsub(/`|[\000-\040\177\s]+|\302[\200-\240]/,'').downcase
if val_unescaped =~ /^[a-z0-9][-+.a-z0-9]*:/ && !ALLOWED_PROTOCOLS.include?(val_unescaped.split(':')[0])
node.attributes.delete attr; next
end
@@ -206,4 +206,23 @@ module Sanitizer
clean.join(' ')
end
+
+# Sanitize a string, parsed using XHTML parsing rules. Reparse the result to
+# ensure well-formedness.
+#
+# :call-seq:
+# safe_sanitize_xhtml(string) -> string
+#
+# Unless otherwise specified, the string is assumed to be utf-8 encoded.
+#
+# The string returned is utf-8 encoded. If you want, you can use iconv to convert it to some other encoding.
+# (REXML trees are always utf-8 encoded.)
+ def safe_xhtml_sanitize(html, options = {})
+ sanitized = xhtml_sanitize(html.purify)
+ doc = REXML::Document.new("
#{sanitized}
")
+ sanitized = doc.to_s.gsub(/\A(.*)<\/div>\Z/m, '\1')
+ rescue REXML::ParseException
+ sanitized = sanitized.escapeHTML
+ end
+
end
diff --git a/lib/stringsupport.rb b/lib/stringsupport.rb
index ef924d24..731a1f95 100644
--- a/lib/stringsupport.rb
+++ b/lib/stringsupport.rb
@@ -2,6 +2,26 @@
class String
+# A method to allow byte-oriented operations in both Ruby 1.8 and Ruby 1.9
+#
+# Under 1.8, this is a NOOP. Under 1.9, it sets the encoding to "ASCII-8BIT"
+#--
+ def as_bytes
+ force_encoding("ASCII-8BIT") if self.respond_to?(:force_encoding)
+ self
+ end
+
+#++
+# A method to allow string-oriented operations in both Ruby 1.8 and Ruby 1.9
+#
+# Under 1.8, this is a NOOP. Under 1.9, it sets the encoding to "UTF-8"
+#--
+ def as_utf8
+ force_encoding("UTF-8") if self.respond_to?(:force_encoding)
+ self
+ end
+
+#++
# Take a string, and remove any invalid substrings, returning a valid utf-8 string.
#
# :call-seq:
@@ -11,12 +31,16 @@ class String
#--
def purify
text = check_ncrs
- text.split(//u).grep(UTF8_REGEX).join
+ if text.respond_to?(:encoding)
+ text.split(//).collect{|c| c.as_bytes}.grep(UTF8_REGEX).join.as_utf8
+ else
+ text.split(//u).grep(UTF8_REGEX).join
+ end
end
def check_ncrs
- text = gsub(/&#[xX]([a-fA-F0-9]+);/) { |m| [$1.hex].pack('U*') =~ UTF8_REGEX ? m : '' }
- text.gsub(/&#(\d+);/) { |m| [$1.to_i].pack('U*') =~ UTF8_REGEX ? m : '' }
+ text = gsub(/&#[xX]([a-fA-F0-9]+);/) { |m| [$1.hex].pack('U*').as_bytes =~ UTF8_REGEX ? m : '' }
+ text.gsub(/&#(\d+);/) { |m| [$1.to_i].pack('U*').as_bytes =~ UTF8_REGEX ? m : '' }
end
UTF8_REGEX = /\A(
@@ -42,7 +66,7 @@ class String
#--
def is_utf8?
#expand NCRs to utf-8
- text = self.check_ncrs
+ text = self.check_ncrs.as_bytes
# You might think this is faster, but it isn't
#pieces = self.split(/&#[xX]([a-fA-F0-9]+);/)
diff --git a/test/functional/application_test.rb b/test/functional/application_test.rb
index 5dfad5b1..cda76b26 100755
--- a/test/functional/application_test.rb
+++ b/test/functional/application_test.rb
@@ -1,6 +1,6 @@
# Unit tests for ApplicationController (the abstract controller class)
-require File.dirname(__FILE__) + '/../test_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
require 'wiki_controller'
# Need some concrete class to test the abstract class features
diff --git a/test/functional/file_controller_test.rb b/test/functional/file_controller_test.rb
index 348407b0..d9c67cfe 100755
--- a/test/functional/file_controller_test.rb
+++ b/test/functional/file_controller_test.rb
@@ -1,6 +1,6 @@
#!/usr/bin/env ruby
-require File.dirname(__FILE__) + '/../test_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
require 'file_controller'
require 'fileutils'
require 'stringio'
diff --git a/test/functional/routes_test.rb b/test/functional/routes_test.rb
index 177b2f4d..f16645db 100644
--- a/test/functional/routes_test.rb
+++ b/test/functional/routes_test.rb
@@ -1,6 +1,6 @@
#!/usr/bin/env ruby
-require File.dirname(__FILE__) + '/../test_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
require 'action_controller/routing'
diff --git a/test/functional/wiki_controller_test.rb b/test/functional/wiki_controller_test.rb
index 6729004c..a1a73daf 100755
--- a/test/functional/wiki_controller_test.rb
+++ b/test/functional/wiki_controller_test.rb
@@ -9,6 +9,7 @@ require 'wiki_controller'
require 'rexml/document'
require 'tempfile'
require 'zip/zipfilesystem'
+require 'stringsupport'
# Raise errors beyond the default web-based presentation
class WikiController; def rescue_action(e) logger.error(e); raise e end; end
@@ -935,7 +936,7 @@ class WikiControllerTest < ActionController::TestCase
r = process('show', 'id' => 'HomePage', 'web' => 'wiki1')
assert_response :success
- assert_match /Recursive include detected: HomePage \342\206\222 HomePage<\/em>/, r.body
+ assert_match /Recursive include detected: HomePage \342\206\222 HomePage<\/em>/, r.body.as_bytes
end
def test_recursive_include_II
@@ -947,7 +948,7 @@ class WikiControllerTest < ActionController::TestCase
r = process('show', 'id' => 'HomePage', 'web' => 'wiki1')
assert_response :success
- assert_match /
Recursive-include:<\/p>\n\n
extra fun Recursive include detected: Foo \342\206\222 Foo<\/em><\/p>/, r.body
+ assert_match / Recursive-include:<\/p>\n\n extra fun Recursive include detected: Foo \342\206\222 Foo<\/em><\/p>/, r.body.as_bytes
end
def test_recursive_include_III
@@ -961,7 +962,7 @@ class WikiControllerTest < ActionController::TestCase
r = process('show', 'id' => 'HomePage', 'web' => 'wiki1')
assert_response :success
- assert_match / Recursive-include:<\/p>\n\n extra fun<\/p>\nRecursive include detected: Bar \342\206\222 Bar<\/em>/, r.body
+ assert_match / Recursive-include:<\/p>\n\n extra fun<\/p>\nRecursive include detected: Bar \342\206\222 Bar<\/em>/, r.body.as_bytes
end
def test_nonrecursive_include
diff --git a/test/unit/chunks/category_test.rb b/test/unit/chunks/category_test.rb
index b105801a..b76e1074 100755
--- a/test/unit/chunks/category_test.rb
+++ b/test/unit/chunks/category_test.rb
@@ -1,6 +1,6 @@
#!/usr/bin/env ruby
-require File.dirname(__FILE__) + '/../../test_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
require 'chunks/category'
class CategoryTest < Test::Unit::TestCase
diff --git a/test/unit/chunks/nowiki_test.rb b/test/unit/chunks/nowiki_test.rb
index 8307e089..f9b1ff0c 100755
--- a/test/unit/chunks/nowiki_test.rb
+++ b/test/unit/chunks/nowiki_test.rb
@@ -1,6 +1,6 @@
#!/usr/bin/env ruby
-require File.dirname(__FILE__) + '/../../test_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
require 'chunks/nowiki'
class NoWikiTest < Test::Unit::TestCase
@@ -26,25 +26,25 @@ class NoWikiTest < Test::Unit::TestCase
def test_sanitize_nowiki
match(NoWiki, 'This sentence contains equation 