Ruby 1.9 Compatibility

Completely removed the html5lib sanitizer. Fixed the string-handling to work in both Ruby 1.8.x and 1.9.2. There are still, inexplicably, two functional tests that fail. But the rest seems to work quite well.
2009-11-30 16:28:18 -06:00 · 2009-11-30 16:28:18 -06:00 · a6429f8c22
commit a6429f8c22
parent 79c8572053
142 changed files with 519 additions and 843 deletions
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/contentModelFlags.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/contentModelFlags.test
@ -0,0 +1,48 @@
+{"tests": [
+
+{"description":"PLAINTEXT content model flag",
+"contentModelFlags":["PLAINTEXT"],
+"input":"<head>&body;",
+"output":[["Character", "<head>&body;"]]},
+
+{"description":"End tag closing RCDATA or CDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo</bar>",
+"output":[["Character", "foo"], ["EndTag", "bar"]]},
+
+{"description":"End tag closing RCDATA or CDATA (case-insensitivity)",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo</bAr>",
+"output":[["Character", "foo"], ["EndTag", "bar"]]},
+
+{"description":"End tag with incorrect name in RCDATA or CDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"baz",
+"input":"</foo>bar</baz>",
+"output":[["Character", "</foo>bar"], ["EndTag", "baz"]]},
+
+{"description":"End tag with incorrect name in RCDATA or CDATA (starting like correct name)",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"baz",
+"input":"</foo>bar</bazaar>",
+"output":[["Character", "</foo>bar</bazaar>"]]},
+
+{"description":"End tag closing RCDATA or CDATA, switching back to PCDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo</bar></baz>",
+"output":[["Character", "foo"], ["EndTag", "bar"], ["EndTag", "baz"]]},
+
+{"description":"CDATA w/ something looking like an entity",
+"contentModelFlags":["CDATA"],
+"input":"&foo;",
+"output":[["Character", "&foo;"]]},
+
+{"description":"RCDATA w/ an entity",
+"contentModelFlags":["RCDATA"],
+"input":"&lt;",
+"output":[["Character", "<"]]}
+
+]}
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/entities.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/entities.test
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/escapeFlag.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/escapeFlag.test
@ -0,0 +1,21 @@
+{"tests": [
+
+{"description":"Commented close tag in [R]CDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo<!--</bar>--></bar>",
+"output":[["Character", "foo<!--</bar>-->"], ["EndTag", "bar"]]},
+
+{"description":"Bogus comment in [R]CDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo<!-->baz</bar>",
+"output":[["Character", "foo<!-->baz"], ["EndTag", "bar"]]},
+
+{"description":"End tag surrounded by bogus comment in [R]CDATA",
+"contentModelFlags":["RCDATA", "CDATA"],
+"lastStartTag":"bar",
+"input":"foo<!--></bar><!-->baz</bar>",
+"output":[["Character", "foo<!-->"], ["EndTag", "bar"], "ParseError", ["Comment", ""], ["Character", "baz"], ["EndTag", "bar"]]}
+
+]}
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test1.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test1.test
@ -0,0 +1,172 @@
+{"tests": [
+
+{"description":"Correct Doctype lowercase",
+"input":"<!DOCTYPE html>",
+"output":[["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Correct Doctype uppercase",
+"input":"<!DOCTYPE HTML>",
+"output":[["DOCTYPE", "HTML", null, null, true]]},
+
+{"description":"Correct Doctype mixed case",
+"input":"<!DOCTYPE HtMl>", 
+"output":[["DOCTYPE", "HtMl", null, null, true]]},
+
+{"description":"Truncated doctype start",
+"input":"<!DOC>", 
+"output":["ParseError", ["Comment", "DOC"]]},
+
+{"description":"Doctype in error",
+"input":"<!DOCTYPE foo>", 
+"output":[["DOCTYPE", "foo", null, null, true]]},
+
+{"description":"Single Start Tag",
+"input":"<h>",
+"output":[["StartTag", "h", {}]]},
+
+{"description":"Empty end tag",
+"input":"</>",
+"output":["ParseError"]},
+
+{"description":"Empty start tag",
+"input":"<>",
+"output":["ParseError", ["Character", "<>"]]},
+
+{"description":"Start Tag w/attribute",
+"input":"<h a='b'>",
+"output":[["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Start Tag w/attribute no quotes",
+"input":"<h a=b>",
+"output":[["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Start/End Tag",
+"input":"<h></h>",
+"output":[["StartTag", "h", {}], ["EndTag", "h"]]},
+
+{"description":"Two unclosed start tags",
+"input":"<p>One<p>Two",
+"output":[["StartTag", "p", {}], ["Character", "One"], ["StartTag", "p", {}], ["Character", "Two"]]},
+
+{"description":"End Tag w/attribute",
+"input":"<h></h a='b'>",
+"output":[["StartTag", "h", {}], "ParseError", ["EndTag", "h"]]},
+
+{"description":"Multiple atts",
+"input":"<h a='b' c='d'>",
+"output":[["StartTag", "h", {"a":"b", "c":"d"}]]},
+
+{"description":"Multiple atts no space",
+"input":"<h a='b'c='d'>",
+"output":[["StartTag", "h", {"a":"b", "c":"d"}]]},
+
+{"description":"Repeated attr",
+ "input":"<h a='b' a='d'>",
+ "output":["ParseError", ["StartTag", "h", {"a":"b"}]]},
+
+{"description":"Simple comment",
+ "input":"<!--comment-->",
+ "output":[["Comment", "comment"]]},
+
+{"description":"Comment, Central dash no space",
+ "input":"<!----->",
+ "output":["ParseError", ["Comment", "-"]]},
+
+{"description":"Comment, two central dashes",
+"input":"<!-- --comment -->",
+"output":["ParseError", ["Comment", " --comment "]]},
+
+{"description":"Unfinished comment",
+"input":"<!--comment",
+"output":["ParseError", ["Comment", "comment"]]},
+
+{"description":"Start of a comment",
+"input":"<!-",
+"output":["ParseError", ["Comment", "-"]]},
+
+{"description":"Short comment",
+ "input":"<!-->",
+ "output":["ParseError", ["Comment", ""]]},
+
+{"description":"Short comment two",
+ "input":"<!--->",
+ "output":["ParseError", ["Comment", ""]]},
+
+{"description":"Short comment three",
+ "input":"<!---->",
+ "output":[["Comment", ""]]},
+
+
+{"description":"Ampersand EOF",
+"input":"&",
+"output":[["Character", "&"]]},
+
+{"description":"Ampersand ampersand EOF",
+"input":"&&",
+"output":[["Character", "&&"]]},
+
+{"description":"Ampersand space EOF",
+"input":"& ",
+"output":[["Character", "& "]]},
+
+{"description":"Unfinished entity",
+"input":"&f",
+"output":["ParseError", ["Character", "&f"]]},
+
+{"description":"Ampersand, number sign",
+"input":"&#",
+"output":["ParseError", ["Character", "&#"]]},
+
+{"description":"Unfinished numeric entity",
+"input":"&#x",
+"output":["ParseError", ["Character", "&#x"]]},
+
+{"description":"Entity with trailing semicolon (1)",
+"input":"I'm &not;it",
+"output":[["Character","I'm ¬it"]]},
+
+{"description":"Entity with trailing semicolon (2)",
+"input":"I'm &notin;",
+"output":[["Character","I'm ∉"]]},
+
+{"description":"Entity without trailing semicolon (1)",
+"input":"I'm &notit",
+"output":[["Character","I'm "], "ParseError", ["Character", "¬it"]]},
+
+{"description":"Entity without trailing semicolon (2)",
+"input":"I'm &notin",
+"output":[["Character","I'm "], "ParseError", ["Character", "¬in"]]},
+
+{"description":"Partial entity match at end of file",
+"input":"I'm &no",
+"output":[["Character","I'm "], "ParseError", ["Character", "&no"]]},
+
+{"description":"ASCII decimal entity",
+"input":"&#0036;",
+"output":[["Character","$"]]},
+
+{"description":"ASCII hexadecimal entity",
+"input":"&#x3f;",
+"output":[["Character","?"]]},
+
+{"description":"Hexadecimal entity in attribute",
+"input":"<h a='&#x3f;'></h>",
+"output":[["StartTag", "h", {"a":"?"}], ["EndTag", "h"]]},
+
+{"description":"Entity in attribute without semicolon ending in x",
+"input":"<h a='&notx'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&notx"}]]},
+
+{"description":"Entity in attribute without semicolon ending in 1",
+"input":"<h a='&not1'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&not1"}]]},
+
+{"description":"Entity in attribute without semicolon ending in i",
+"input":"<h a='&noti'>",
+"output":["ParseError", ["StartTag", "h", {"a":"&noti"}]]},
+
+{"description":"Entity in attribute without semicolon",
+"input":"<h a='&COPY'>",
+"output":["ParseError", ["StartTag", "h", {"a":"©"}]]}
+
+]}
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test2.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test2.test
@ -0,0 +1,129 @@
+{"tests": [
+
+{"description":"DOCTYPE without name",
+"input":"<!DOCTYPE>",
+"output":["ParseError", "ParseError", ["DOCTYPE", "", null, null, false]]},
+
+{"description":"DOCTYPE without space before name",
+"input":"<!DOCTYPEhtml>",
+"output":["ParseError", ["DOCTYPE", "html", null, null, true]]},
+
+{"description":"Incorrect DOCTYPE without a space before name",
+"input":"<!DOCTYPEfoo>",
+"output":["ParseError", ["DOCTYPE", "foo", null, null, true]]},
+
+{"description":"DOCTYPE with publicId",
+"input":"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", "-//W3C//DTD HTML Transitional 4.01//EN", null, true]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC",
+"input":"<!DOCTYPE html PUBLIC",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC '",
+"input":"<!DOCTYPE html PUBLIC '",
+"output":["ParseError", ["DOCTYPE", "html", "", null, false]]},
+
+{"description":"DOCTYPE with EOF after PUBLIC 'x",
+"input":"<!DOCTYPE html PUBLIC 'x",
+"output":["ParseError", ["DOCTYPE", "html", "x", null, false]]},
+
+{"description":"DOCTYPE with systemId",
+"input":"<!DOCTYPE html SYSTEM \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", null, "-//W3C//DTD HTML Transitional 4.01//EN", true]]},
+
+{"description":"DOCTYPE with publicId and systemId",
+"input":"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML Transitional 4.01//EN\" \"-//W3C//DTD HTML Transitional 4.01//EN\">",
+"output":[["DOCTYPE", "html", "-//W3C//DTD HTML Transitional 4.01//EN", "-//W3C//DTD HTML Transitional 4.01//EN", true]]},
+
+{"description":"Incomplete doctype",
+"input":"<!DOCTYPE html ",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Numeric entity representing the NUL character",
+"input":"&#0000;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity representing the NUL character",
+"input":"&#x0000;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Numeric entity representing a codepoint after 1114111 (U+10FFFF)",
+"input":"&#2225222;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity representing a codepoint after 1114111 (U+10FFFF)",
+"input":"&#x1010FFFF;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity pair representing a surrogate pair",
+"input":"&#xD869;&#xDED6;",
+"output":["ParseError", ["Character", "\uFFFD"], "ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Hexadecimal entity with mixed uppercase and lowercase",
+"input":"&#xaBcD;",
+"output":[["Character", "\uABCD"]]},
+
+{"description":"Entity without a name",
+"input":"&;",
+"output":["ParseError", ["Character", "&;"]]},
+
+{"description":"Unescaped ampersand in attribute value",
+"input":"<h a='&'>",
+"output":["ParseError", ["StartTag", "h", { "a":"&" }]]},
+
+{"description":"StartTag containing <",
+"input":"<a<b>",
+"output":[["StartTag", "a<b", { }]]},
+
+{"description":"Non-void element containing trailing /",
+"input":"<h/>",
+"output":["ParseError", ["StartTag", "h", { }]]},
+
+{"description":"Void element with permitted slash",
+"input":"<br/>",
+"output":[["StartTag", "br", { }]]},
+
+{"description":"StartTag containing /",
+"input":"<h/a='b'>",
+"output":["ParseError", ["StartTag", "h", { "a":"b" }]]},
+
+{"description":"Double-quoted attribute value",
+"input":"<h a=\"b\">",
+"output":[["StartTag", "h", { "a":"b" }]]},
+
+{"description":"Unescaped </",
+"input":"</",
+"output":["ParseError", ["Character", "</"]]},
+
+{"description":"Illegal end tag name",
+"input":"</1>",
+"output":["ParseError", ["Comment", "1"]]},
+
+{"description":"Simili processing instruction",
+"input":"<?namespace>",
+"output":["ParseError", ["Comment", "?namespace"]]},
+
+{"description":"A bogus comment stops at >, even if preceeded by two dashes",
+"input":"<?foo-->",
+"output":["ParseError", ["Comment", "?foo--"]]},
+
+{"description":"Unescaped <",
+"input":"foo < bar",
+"output":[["Character", "foo "], "ParseError", ["Character", "< bar"]]},
+
+{"description":"Null Byte Replacement",
+"input":"\u0000",
+"output":["ParseError", ["Character", "\ufffd"]]},
+
+{"description":"Comment with dash",
+"input":"<!---x",
+"output":["ParseError", ["Comment", "-x"]]},
+
+{"description":"Entity + newline",
+"input":"\nx\n&gt;\n",
+"output":[["Character","\nx\n>\n"]]}
+
+]}
+
+
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test3.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test3.test
@ -0,0 +1,367 @@
+{"tests": [
+
+{"description":"<",
+"input":"<",
+"output":["ParseError", ["Character", "<"]]},
+
+{"description":"<>",
+"input":"<>",
+"output":["ParseError", ["Character", "<>"]]},
+
+{"description":"<!",
+"input":"<!",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!>",
+"input":"<!>",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!--",
+"input":"<!--",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!-->",
+"input":"<!-->",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!---",
+"input":"<!---",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!--->",
+"input":"<!--->",
+"output":["ParseError", ["Comment", ""]]},
+
+{"description":"<!---->",
+"input":"<!---->",
+"output":[["Comment", ""]]},
+
+{"description":"<!-----",
+"input":"<!-----",
+"output":["ParseError", "ParseError", ["Comment", "-"]]},
+
+{"description":"<!----.",
+"input":"<!----.",
+"output":["ParseError", "ParseError", ["Comment", "--."]]},
+
+{"description":"<!---?",
+"input":"<!---?",
+"output":["ParseError", ["Comment", "-?"]]},
+
+{"description":"<!--?-",
+"input":"<!--?-",
+"output":["ParseError", ["Comment", "?"]]},
+
+{"description":"<!--?--",
+"input":"<!--?--",
+"output":["ParseError", ["Comment", "?"]]},
+
+{"description":"<!--?-.",
+"input":"<!--?-.",
+"output":["ParseError", ["Comment", "?-."]]},
+
+{"description":"<!--?.",
+"input":"<!--?.",
+"output":["ParseError", ["Comment", "?."]]},
+
+{"description":"<?>",
+"input":"<?>",
+"output":["ParseError", ["Comment", "?"]]},
+
+{"description":"<??",
+"input":"<??",
+"output":["ParseError", ["Comment", "??"]]},
+
+{"description":"</",
+"input":"</",
+"output":["ParseError", ["Character", "</"]]},
+
+{"description":"</>",
+"input":"</>",
+"output":["ParseError"]},
+
+{"description":"</?",
+"input":"</?",
+"output":["ParseError", ["Comment", "?"]]},
+
+{"description":">",
+"input":">",
+"output":[["Character", ">"]]},
+
+{"description":"-",
+"input":"-",
+"output":[["Character", "-"]]},
+
+{"description":"?",
+"input":"?",
+"output":[["Character", "?"]]},
+
+{"description":"&",
+"input":"&",
+"output":[["Character", "&"]]},
+
+{"description":"&#",
+"input":"&#",
+"output":["ParseError", ["Character", "&#"]]},
+
+{"description":"&#9",
+"input":"&#9",
+"output":["ParseError", ["Character", "\t"]]},
+
+{"description":"<!doctype >",
+"input":"<!doctype >",
+"output":["ParseError", ["DOCTYPE", "", null, null, false]]},
+
+{"description":"<!doctype  ",
+"input":"<!doctype  ",
+"output":["ParseError", ["DOCTYPE", "", null, null, false]]},
+
+{"description":"<!doctype!>",
+"input":"<!doctype!>",
+"output":["ParseError", ["DOCTYPE", "!", null, null, true]]},
+
+{"description":"<!doctype! >",
+"input":"<!doctype! >",
+"output":["ParseError", ["DOCTYPE", "!", null, null, true]]},
+
+{"description":"<!doctype!  ",
+"input":"<!doctype!  ",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! ?>",
+"input":"<!doctype! ?>",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! ??",
+"input":"<!doctype! ??",
+"output":["ParseError", "ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype!?",
+"input":"<!doctype!?",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!?", null, null, false]]},
+
+{"description":"<!doctype! public>",
+"input":"<!doctype! public>",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! public ",
+"input":"<!doctype! public ",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! public?",
+"input":"<!doctype! public?",
+"output":["ParseError", "ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! public''",
+"input":"<!doctype! public''",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "", null, false]]},
+
+{"description":"<!doctype! public'(",
+"input":"<!doctype! public'(",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "(", null, false]]},
+
+{"description":"<!doctype! public\"\">",
+"input":"<!doctype! public\"\">",
+"output":["ParseError", ["DOCTYPE", "!", "", null, true]]},
+
+{"description":"<!doctype! public\"\" ",
+"input":"<!doctype! public\"\" ",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "", null, false]]},
+
+{"description":"<!doctype! public\"\"?",
+"input":"<!doctype! public\"\"?",
+"output":["ParseError", "ParseError", "ParseError", ["DOCTYPE", "!", "", null, false]]},
+
+{"description":"<!doctype! public\"\"'",
+"input":"<!doctype! public\"\"'",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "", "", false]]},
+
+{"description":"<!doctype! public\"\"\"",
+"input":"<!doctype! public\"\"\"",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "", "", false]]},
+
+{"description":"<!doctype! public\"#",
+"input":"<!doctype! public\"#",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", "#", null, false]]},
+
+{"description":"<!doctype! system>",
+"input":"<!doctype! system>",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! system ",
+"input":"<!doctype! system ",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! system?",
+"input":"<!doctype! system?",
+"output":["ParseError", "ParseError", "ParseError", ["DOCTYPE", "!", null, null, false]]},
+
+{"description":"<!doctype! system''",
+"input":"<!doctype! system''",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, "", false]]},
+
+{"description":"<!doctype! system'(",
+"input":"<!doctype! system'(",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, "(", false]]},
+
+{"description":"<!doctype! system\"\">",
+"input":"<!doctype! system\"\">",
+"output":["ParseError", ["DOCTYPE", "!", null, "", true]]},
+
+{"description":"<!doctype! system\"\" ",
+"input":"<!doctype! system\"\" ",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, "", false]]},
+
+{"description":"<!doctype! system\"\"?",
+"input":"<!doctype! system\"\"?",
+"output":["ParseError", "ParseError", "ParseError", ["DOCTYPE", "!", null, "", false]]},
+
+{"description":"<!doctype! system\"#",
+"input":"<!doctype! system\"#",
+"output":["ParseError", "ParseError", ["DOCTYPE", "!", null, "#", false]]},
+
+{"description":"</z",
+"input":"</z",
+"output":["ParseError", ["EndTag", "z"]]},
+
+{"description":"<z>",
+"input":"<z>",
+"output":[["StartTag", "z", {}]]},
+
+{"description":"<z ",
+"input":"<z ",
+"output":["ParseError", ["StartTag", "z", {}]]},
+
+{"description":"<z/>",
+"input":"<z/>",
+"output":["ParseError", ["StartTag", "z", {}]]},
+
+{"description":"<z/ ",
+"input":"<z/ ",
+"output":["ParseError", "ParseError", ["StartTag", "z", {}]]},
+
+{"description":"<z//",
+"input":"<z//",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {}]]},
+
+{"description":"<z",
+"input":"<z",
+"output":["ParseError", ["StartTag", "z", {}]]},
+
+{"description":"</z",
+"input":"</z",
+"output":["ParseError", ["EndTag", "z"]]},
+
+{"description":"<z0",
+"input":"<z0",
+"output":["ParseError", ["StartTag", "z0", {}]]},
+
+{"description":"<z/0=>",
+"input":"<z/0=>",
+"output":["ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0= ",
+"input":"<z/0= ",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0=?>",
+"input":"<z/0=?>",
+"output":["ParseError", ["StartTag", "z", {"0": "?"}]]},
+
+{"description":"<z/0=? ",
+"input":"<z/0=? ",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "?"}]]},
+
+{"description":"<z/0=??",
+"input":"<z/0=??",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "??"}]]},
+
+{"description":"<z/0=''",
+"input":"<z/0=''",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0='&",
+"input":"<z/0='&",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "&"}]]},
+
+{"description":"<z/0='%",
+"input":"<z/0='%",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "%"}]]},
+
+{"description":"<z/0=\"'",
+"input":"<z/0=\"'",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "'"}]]},
+
+{"description":"<z/0=\"\"",
+"input":"<z/0=\"\"",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0=\"&",
+"input":"<z/0=\"&",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "&"}]]},
+
+{"description":"<z/0=&",
+"input":"<z/0=&",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "&"}]]},
+
+{"description":"<z/0>",
+"input":"<z/0>",
+"output":["ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0 =",
+"input":"<z/0 =",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0 >",
+"input":"<z/0 >",
+"output":["ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0  ",
+"input":"<z/0  ",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0 /",
+"input":"<z/0 /",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0/",
+"input":"<z/0/",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/00",
+"input":"<z/00",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"00": ""}]]},
+
+{"description":"<z/0 0",
+"input":"<z/0 0",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": ""}]]},
+
+{"description":"<z/0='&#9",
+"input":"<z/0='&#9",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": "\t"}]]},
+
+{"description":"<z/0=\"&#9",
+"input":"<z/0=\"&#9",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": "\t"}]]},
+
+{"description":"<z/0=&#9",
+"input":"<z/0=&#9",
+"output":["ParseError", "ParseError", "ParseError", ["StartTag", "z", {"0": "\t"}]]},
+
+{"description":"<z/0z",
+"input":"<z/0z",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0z": ""}]]},
+
+{"description":"<z/0 z",
+"input":"<z/0 z",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "", "z": ""}]]},
+
+{"description":"<zz",
+"input":"<zz",
+"output":["ParseError", ["StartTag", "zz", {}]]},
+
+{"description":"<z/z",
+"input":"<z/z",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"z": ""}]]}
+
+]}
--- a/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test4.test
+++ b/attic/vendor/plugins/HTML5lib/testdata/tokenizer/test4.test
@ -0,0 +1,198 @@
+{"tests": [
+
+{"description":"< in attribute name",
+"input":"<z/0  <",
+"output":["ParseError", "ParseError", ["StartTag", "z", {"0": "", "<": ""}]]},
+
+{"description":"< in attribute value",
+"input":"<z x=<",
+"output":["ParseError", ["StartTag", "z", {"x": "<"}]]},
+
+{"description":"CR EOF after doctype name",
+"input":"<!doctype html \r",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"CR EOF in tag name",
+"input":"<z\r",
+"output":["ParseError", ["StartTag", "z", {}]]},
+
+{"description":"Zero hex numeric entity",
+"input":"&#x0",
+"output":["ParseError", "ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Zero decimal numeric entity",
+"input":"&#0",
+"output":["ParseError", "ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Zero-prefixed hex numeric entity",
+"input":"&#x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000041;",
+"output":[["Character", "A"]]},
+
+{"description":"Zero-prefixed decimal numeric entity",
+"input":"&#000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000065;",
+"output":[["Character", "A"]]},
+
+{"description":"Empty hex numeric entities",
+"input":"&#x &#X ",
+"output":["ParseError", ["Character", "&#x "], "ParseError", ["Character", "&#X "]]},
+
+{"description":"Empty decimal numeric entities",
+"input":"&# &#; ",
+"output":["ParseError", ["Character", "&# "], "ParseError", ["Character", "&#; "]]},
+
+{"description":"Non-BMP numeric entity",
+"input":"&#x10000;",
+"output":[["Character", "\uD800\uDC00"]]},
+
+{"description":"Maximum non-BMP numeric entity",
+"input":"&#X10FFFF;",
+"output":[["Character", "\uDBFF\uDFFF"]]},
+
+{"description":"Above maximum numeric entity",
+"input":"&#x110000;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"32-bit hex numeric entity",
+"input":"&#x80000041;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"33-bit hex numeric entity",
+"input":"&#x100000041;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"33-bit decimal numeric entity",
+"input":"&#4294967361;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"65-bit hex numeric entity",
+"input":"&#x10000000000000041;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"65-bit decimal numeric entity",
+"input":"&#18446744073709551681;",
+"output":["ParseError", ["Character", "\uFFFD"]]},
+
+{"description":"Surrogate code point edge cases",
+"input":"&#xD7FF;&#xD800;&#xD801;&#xDFFE;&#xDFFF;&#xE000;",
+"output":[["Character", "\uD7FF"], "ParseError", ["Character", "\uFFFD"], "ParseError", ["Character", "\uFFFD"], "ParseError", ["Character", "\uFFFD"], "ParseError", ["Character", "\uFFFD\uE000"]]},
+
+{"description":"Uppercase start tag name",
+"input":"<X>",
+"output":[["StartTag", "x", {}]]},
+
+{"description":"Uppercase end tag name",
+"input":"</X>",
+"output":[["EndTag", "x"]]},
+
+{"description":"Uppercase attribute name",
+"input":"<x X>",
+"output":[["StartTag", "x", { "x":"" }]]},
+
+{"description":"Tag/attribute name case edge values",
+"input":"<x@AZ[`az{ @AZ[`az{>",
+"output":[["StartTag", "x@az[`az{", { "@az[`az{":"" }]]},
+
+{"description":"Duplicate different-case attributes",
+"input":"<x x=1 x=2 X=3>",
+"output":["ParseError", "ParseError", ["StartTag", "x", { "x":"1" }]]},
+
+{"description":"Uppercase close tag attributes",
+"input":"</x X>",
+"output":["ParseError", ["EndTag", "x"]]},
+
+{"description":"Duplicate close tag attributes",
+"input":"</x x x>",
+"output":["ParseError", "ParseError", ["EndTag", "x"]]},
+
+{"description":"Permitted slash",
+"input":"<br/>",
+"output":[["StartTag", "br", {}]]},
+
+{"description":"Non-permitted slash",
+"input":"<xr/>",
+"output":["ParseError", ["StartTag", "xr", {}]]},
+
+{"description":"Permitted slash but in close tag",
+"input":"</br/>",
+"output":["ParseError", ["EndTag", "br"]]},
+
+{"description":"Doctype public case-sensitivity (1)",
+"input":"<!DoCtYpE HtMl PuBlIc \"AbC\" \"XyZ\">",
+"output":[["DOCTYPE", "HtMl", "AbC", "XyZ", true]]},
+
+{"description":"Doctype public case-sensitivity (2)",
+"input":"<!dOcTyPe hTmL pUbLiC \"aBc\" \"xYz\">",
+"output":[["DOCTYPE", "hTmL", "aBc", "xYz", true]]},
+
+{"description":"Doctype system case-sensitivity (1)",
+"input":"<!DoCtYpE HtMl SyStEm \"XyZ\">",
+"output":[["DOCTYPE", "HtMl", null, "XyZ", true]]},
+
+{"description":"Doctype system case-sensitivity (2)",
+"input":"<!dOcTyPe hTmL sYsTeM \"xYz\">",
+"output":[["DOCTYPE", "hTmL", null, "xYz", true]]},
+
+{"description":"U+0000 in lookahead region after non-matching character",
+"input":"<!doc>\u0000",
+"output":["ParseError", ["Comment", "doc"], "ParseError", ["Character", "\uFFFD"]],
+"ignoreErrorOrder":true},
+
+{"description":"U+0000 in lookahead region",
+"input":"<!doc\u0000",
+"output":["ParseError", "ParseError", ["Comment", "doc\uFFFD"]],
+"ignoreErrorOrder":true},
+
+{"description":"CR followed by U+0000",
+"input":"\r\u0000",
+"output":["ParseError", ["Character", "\n\uFFFD"]],
+"ignoreErrorOrder":true},
+
+{"description":"CR followed by non-LF",
+"input":"\r?",
+"output":[["Character", "\n?"]]},
+
+{"description":"CR at EOF",
+"input":"\r",
+"output":[["Character", "\n"]]},
+
+{"description":"LF at EOF",
+"input":"\n",
+"output":[["Character", "\n"]]},
+
+{"description":"CR LF",
+"input":"\r\n",
+"output":[["Character", "\n"]]},
+
+{"description":"CR CR",
+"input":"\r\r",
+"output":[["Character", "\n\n"]]},
+
+{"description":"LF LF",
+"input":"\n\n",
+"output":[["Character", "\n\n"]]},
+
+{"description":"LF CR",
+"input":"\n\r",
+"output":[["Character", "\n\n"]]},
+
+{"description":"text CR CR CR text",
+"input":"text\r\r\rtext",
+"output":[["Character", "text\n\n\ntext"]]},
+
+{"description":"Doctype publik",
+"input":"<!DOCTYPE html PUBLIK \"AbC\" \"XyZ\">",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Doctype publi",
+"input":"<!DOCTYPE html PUBLI",
+"output":["ParseError", "ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Doctype sistem",
+"input":"<!DOCTYPE html SISTEM \"AbC\">",
+"output":["ParseError", ["DOCTYPE", "html", null, null, false]]},
+
+{"description":"Doctype sys",
+"input":"<!DOCTYPE html SYS",
+"output":["ParseError", "ParseError", ["DOCTYPE", "html", null, null, false]]}
+
+]}