549 lines
16 KiB
Ruby
Executable file
549 lines
16 KiB
Ruby
Executable file
require 'stringio'
|
|
require 'html5lib/constants'
|
|
|
|
module HTML5lib
|
|
|
|
# Provides a unicode stream of characters to the HTMLTokenizer.
|
|
|
|
# This class takes care of character encoding and removing or replacing
|
|
# incorrect byte-sequences and also provides column and line tracking.
|
|
|
|
class HTMLInputStream
|
|
|
|
attr_accessor :queue, :charEncoding
|
|
|
|
# Initialises the HTMLInputStream.
|
|
#
|
|
# HTMLInputStream(source, [encoding]) -> Normalized stream from source
|
|
# for use by the HTML5Lib.
|
|
#
|
|
# source can be either a file-object, local filename or a string.
|
|
#
|
|
# The optional encoding parameter must be a string that indicates
|
|
# the encoding. If specified, that encoding will be used,
|
|
# regardless of any BOM or later declaration (such as in a meta
|
|
# element)
|
|
#
|
|
# parseMeta - Look for a <meta> element containing encoding information
|
|
|
|
def initialize(source, options = {})
|
|
@encoding = nil
|
|
@parseMeta = true
|
|
@chardet = true
|
|
|
|
options.each { |name, value| instance_variable_set("@#{name}", value) }
|
|
|
|
# List of where new lines occur
|
|
@newLines = []
|
|
|
|
# Raw Stream
|
|
@rawStream = openStream(source)
|
|
|
|
# Encoding Information
|
|
#Number of bytes to use when looking for a meta element with
|
|
#encoding information
|
|
@NUM_BYTES_META = 512
|
|
#Encoding to use if no other information can be found
|
|
@DEFAULT_ENCODING = 'windows-1252'
|
|
|
|
#Detect encoding iff no explicit "transport level" encoding is supplied
|
|
if @encoding.nil? or not HTML5lib.isValidEncoding(@encoding)
|
|
@charEncoding = detectEncoding
|
|
else
|
|
@charEncoding = @encoding
|
|
end
|
|
|
|
# Read bytes from stream decoding them into Unicode
|
|
uString = @rawStream.read
|
|
unless @charEncoding == 'utf-8'
|
|
begin
|
|
require 'iconv'
|
|
uString = Iconv.iconv('utf-8', @encoding, uString)[0]
|
|
rescue
|
|
end
|
|
end
|
|
|
|
# Normalize newlines and null characters
|
|
uString.gsub!(/\r\n?/, "\n")
|
|
uString.gsub!("\x00", [0xFFFD].pack('U'))
|
|
|
|
# Convert the unicode string into a list to be used as the data stream
|
|
@dataStream = uString
|
|
|
|
@queue = []
|
|
|
|
# Reset position in the list to read from
|
|
reset
|
|
end
|
|
|
|
# Produces a file object from source.
|
|
#
|
|
# source can be either a file object, local filename or a string.
|
|
def openStream(source)
|
|
# Already an IO like object
|
|
if source.respond_to?(:read)
|
|
@stream = source
|
|
else
|
|
# Treat source as a string and wrap in StringIO
|
|
@stream = StringIO.new(source)
|
|
end
|
|
return @stream
|
|
end
|
|
|
|
def detectEncoding
|
|
|
|
#First look for a BOM
|
|
#This will also read past the BOM if present
|
|
encoding = detectBOM
|
|
#If there is no BOM need to look for meta elements with encoding
|
|
#information
|
|
if encoding.nil? and @parseMeta
|
|
encoding = detectEncodingMeta
|
|
end
|
|
#Guess with chardet, if avaliable
|
|
if encoding.nil? and @chardet
|
|
begin
|
|
require 'rubygems'
|
|
require 'UniversalDetector' # gem install chardet
|
|
buffer = @rawStream.read
|
|
encoding = UniversalDetector::chardet(buffer)['encoding']
|
|
@rawStream = openStream(buffer)
|
|
rescue LoadError
|
|
end
|
|
end
|
|
# If all else fails use the default encoding
|
|
if encoding.nil?
|
|
encoding = @DEFAULT_ENCODING
|
|
end
|
|
|
|
#Substitute for equivalent encodings:
|
|
encodingSub = {'ascii' => 'windows-1252', 'iso-8859-1' => 'windows-1252'}
|
|
|
|
if encodingSub.has_key?(encoding.downcase)
|
|
encoding = encodingSub[encoding.downcase]
|
|
end
|
|
|
|
return encoding
|
|
end
|
|
|
|
# Attempts to detect at BOM at the start of the stream. If
|
|
# an encoding can be determined from the BOM return the name of the
|
|
# encoding otherwise return nil
|
|
def detectBOM
|
|
bomDict = {
|
|
"\xef\xbb\xbf" => 'utf-8',
|
|
"\xff\xfe" => 'utf-16-le',
|
|
"\xfe\xff" => 'utf-16-be',
|
|
"\xff\xfe\x00\x00" => 'utf-32-le',
|
|
"\x00\x00\xfe\xff" => 'utf-32-be'
|
|
}
|
|
|
|
# Go to beginning of file and read in 4 bytes
|
|
@rawStream.seek(0)
|
|
string = @rawStream.read(4)
|
|
return nil unless string
|
|
|
|
# Try detecting the BOM using bytes from the string
|
|
encoding = bomDict[string[0...3]] # UTF-8
|
|
seek = 3
|
|
unless encoding
|
|
# Need to detect UTF-32 before UTF-16
|
|
encoding = bomDict[string] # UTF-32
|
|
seek = 4
|
|
unless encoding
|
|
encoding = bomDict[string[0...2]] # UTF-16
|
|
seek = 2
|
|
end
|
|
end
|
|
|
|
#AT - move this to the caller?
|
|
# Set the read position past the BOM if one was found, otherwise
|
|
# set it to the start of the stream
|
|
@rawStream.seek(encoding ? seek : 0)
|
|
|
|
return encoding
|
|
end
|
|
|
|
# Report the encoding declared by the meta element
|
|
def detectEncodingMeta
|
|
parser = EncodingParser.new(@rawStream.read(@NUM_BYTES_META))
|
|
@rawStream.seek(0)
|
|
return parser.getEncoding
|
|
end
|
|
|
|
def determineNewLines
|
|
# Looks through the stream to find where new lines occur so
|
|
# the position method can tell where it is.
|
|
@newLines.push(0)
|
|
(0...@dataStream.length).each { |i| @newLines.push(i) if @dataStream[i] == ?\n }
|
|
end
|
|
|
|
# Returns (line, col) of the current position in the stream.
|
|
def position
|
|
# Generate list of new lines first time around
|
|
determineNewLines if @newLines.empty?
|
|
line = 0
|
|
tell = @tell
|
|
@newLines.each do |pos|
|
|
break unless pos < tell
|
|
line += 1
|
|
end
|
|
col = tell - @newLines[line-1] - 1
|
|
return [line, col]
|
|
end
|
|
|
|
# Resets the position in the stream back to the start.
|
|
def reset
|
|
@tell = 0
|
|
end
|
|
|
|
# Read one character from the stream or queue if available. Return
|
|
# EOF when EOF is reached.
|
|
def char
|
|
unless @queue.empty?
|
|
return @queue.shift
|
|
else
|
|
begin
|
|
@tell += 1
|
|
return @dataStream[@tell - 1].chr
|
|
rescue
|
|
return :EOF
|
|
end
|
|
end
|
|
end
|
|
|
|
# Returns a string of characters from the stream up to but not
|
|
# including any character in characters or EOF. characters can be
|
|
# any container that supports the in method being called on it.
|
|
def charsUntil(characters, opposite = false)
|
|
charStack = [char]
|
|
|
|
unless charStack[0] == :EOF
|
|
while (characters.include? charStack[-1]) == opposite
|
|
unless @queue.empty?
|
|
# First from the queue
|
|
charStack.push(@queue.shift)
|
|
break if charStack[-1] == :EOF
|
|
else
|
|
# Then the rest
|
|
begin
|
|
charStack.push(@dataStream[@tell].chr)
|
|
@tell += 1
|
|
rescue
|
|
charStack.push(:EOF)
|
|
break
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
# Put the character stopped on back to the front of the queue
|
|
# from where it came.
|
|
@queue.insert(0, charStack.pop)
|
|
return charStack.join('')
|
|
end
|
|
end
|
|
|
|
# String-like object with an assosiated position and various extra methods
|
|
# If the position is ever greater than the string length then an exception is raised
|
|
class EncodingBytes < String
|
|
|
|
attr_accessor :position
|
|
|
|
def initialize(value)
|
|
super(value)
|
|
@position = -1
|
|
end
|
|
|
|
def each
|
|
while @position < length
|
|
@position += 1
|
|
yield self[@position]
|
|
end
|
|
rescue EOF
|
|
end
|
|
|
|
def currentByte
|
|
raise EOF if @position >= length
|
|
return self[@position].chr
|
|
end
|
|
|
|
# Skip past a list of characters
|
|
def skip(chars = SPACE_CHARACTERS)
|
|
while chars.include?(currentByte)
|
|
@position += 1
|
|
end
|
|
end
|
|
|
|
# Look for a sequence of bytes at the start of a string. If the bytes
|
|
# are found return true and advance the position to the byte after the
|
|
# match. Otherwise return false and leave the position alone
|
|
def matchBytes(bytes, lower = false)
|
|
data = self[position ... position+bytes.length]
|
|
data.downcase! if lower
|
|
rv = (data == bytes)
|
|
@position += bytes.length if rv == true
|
|
return rv
|
|
end
|
|
|
|
# Look for the next sequence of bytes matching a given sequence. If
|
|
# a match is found advance the position to the last byte of the match
|
|
def jumpTo(bytes)
|
|
newPosition = self[position .. -1].index(bytes)
|
|
if newPosition
|
|
@position += (newPosition + bytes.length-1)
|
|
return true
|
|
else
|
|
raise EOF
|
|
end
|
|
end
|
|
|
|
# Move the pointer so it points to the next byte in a set of possible
|
|
# bytes
|
|
def findNext(byteList)
|
|
until byteList.include?(currentByte)
|
|
@position += 1
|
|
end
|
|
end
|
|
end
|
|
|
|
# Mini parser for detecting character encoding from meta elements
|
|
class EncodingParser
|
|
|
|
# string - the data to work on for encoding detection
|
|
def initialize(data)
|
|
@data = EncodingBytes.new(data.to_s)
|
|
@encoding = nil
|
|
end
|
|
|
|
@@method_dispatch = [
|
|
['<!--', :handleComment],
|
|
['<meta', :handleMeta],
|
|
['</', :handlePossibleEndTag],
|
|
['<!', :handleOther],
|
|
['<?', :handleOther],
|
|
['<', :handlePossibleStartTag]
|
|
]
|
|
|
|
def getEncoding
|
|
@data.each do |byte|
|
|
keepParsing = true
|
|
@@method_dispatch.each do |(key, method)|
|
|
if @data.matchBytes(key, lower = true)
|
|
keepParsing = send(method)
|
|
break
|
|
end
|
|
end
|
|
break unless keepParsing
|
|
end
|
|
@encoding = @encoding.strip unless @encoding.nil?
|
|
return @encoding
|
|
end
|
|
|
|
# Skip over comments
|
|
def handleComment
|
|
return @data.jumpTo('-->')
|
|
end
|
|
|
|
def handleMeta
|
|
# if we have <meta not followed by a space so just keep going
|
|
return true unless SPACE_CHARACTERS.include?(@data.currentByte)
|
|
|
|
#We have a valid meta element we want to search for attributes
|
|
while true
|
|
#Try to find the next attribute after the current position
|
|
attr = getAttribute
|
|
|
|
return true if attr.nil?
|
|
|
|
if attr[0] == 'charset'
|
|
tentativeEncoding = attr[1]
|
|
if HTML5lib.isValidEncoding(tentativeEncoding)
|
|
@encoding = tentativeEncoding
|
|
return false
|
|
end
|
|
elsif attr[0] == 'content'
|
|
contentParser = ContentAttrParser.new(EncodingBytes.new(attr[1]))
|
|
tentativeEncoding = contentParser.parse
|
|
if HTML5lib.isValidEncoding(tentativeEncoding)
|
|
@encoding = tentativeEncoding
|
|
return false
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
def handlePossibleStartTag
|
|
return handlePossibleTag(false)
|
|
end
|
|
|
|
def handlePossibleEndTag
|
|
@data.position+=1
|
|
return handlePossibleTag(true)
|
|
end
|
|
|
|
def handlePossibleTag(endTag)
|
|
unless ASCII_LETTERS.include?(@data.currentByte)
|
|
#If the next byte is not an ascii letter either ignore this
|
|
#fragment (possible start tag case) or treat it according to
|
|
#handleOther
|
|
if endTag
|
|
@data.position -= 1
|
|
handleOther
|
|
end
|
|
return true
|
|
end
|
|
|
|
@data.findNext(SPACE_CHARACTERS + ['<', '>'])
|
|
|
|
if @data.currentByte == '<'
|
|
#return to the first step in the overall "two step" algorithm
|
|
#reprocessing the < byte
|
|
@data.position -= 1
|
|
else
|
|
#Read all attributes
|
|
{} until getAttribute.nil?
|
|
end
|
|
return true
|
|
end
|
|
|
|
def handleOther
|
|
return @data.jumpTo('>')
|
|
end
|
|
|
|
# Return a name,value pair for the next attribute in the stream,
|
|
# if one is found, or nil
|
|
def getAttribute
|
|
@data.skip(SPACE_CHARACTERS + ['/'])
|
|
|
|
if @data.currentByte == '<'
|
|
@data.position -= 1
|
|
return nil
|
|
elsif @data.currentByte == '>'
|
|
return nil
|
|
end
|
|
|
|
attrName = []
|
|
attrValue = []
|
|
spaceFound = false
|
|
#Step 5 attribute name
|
|
while true
|
|
if @data.currentByte == '=' and attrName:
|
|
break
|
|
elsif SPACE_CHARACTERS.include?(@data.currentByte)
|
|
spaceFound = true
|
|
break
|
|
elsif ['/', '<', '>'].include?(@data.currentByte)
|
|
return [attrName.join(''), '']
|
|
elsif ASCII_UPPERCASE.include?(@data.currentByte)
|
|
attrName.push(@data.currentByte.downcase)
|
|
else
|
|
attrName.push(@data.currentByte)
|
|
end
|
|
#Step 6
|
|
@data.position += 1
|
|
end
|
|
#Step 7
|
|
if spaceFound
|
|
@data.skip
|
|
#Step 8
|
|
unless @data.currentByte == '='
|
|
@data.position -= 1
|
|
return [attrName.join(''), '']
|
|
end
|
|
end
|
|
#XXX need to advance position in both spaces and value case
|
|
#Step 9
|
|
@data.position += 1
|
|
#Step 10
|
|
@data.skip
|
|
#Step 11
|
|
if ["'", '"'].include?(@data.currentByte)
|
|
#11.1
|
|
quoteChar = @data.currentByte
|
|
while true
|
|
@data.position+=1
|
|
#11.3
|
|
if @data.currentByte == quoteChar
|
|
@data.position += 1
|
|
return [attrName.join(''), attrValue.join('')]
|
|
#11.4
|
|
elsif ASCII_UPPERCASE.include?(@data.currentByte)
|
|
attrValue.push(@data.currentByte.downcase)
|
|
#11.5
|
|
else
|
|
attrValue.push(@data.currentByte)
|
|
end
|
|
end
|
|
elsif ['>', '<'].include?(@data.currentByte)
|
|
return [attrName.join(''), '']
|
|
elsif ASCII_UPPERCASE.include?(@data.currentByte)
|
|
attrValue.push(@data.currentByte.downcase)
|
|
else
|
|
attrValue.push(@data.currentByte)
|
|
end
|
|
while true
|
|
@data.position +=1
|
|
if (SPACE_CHARACTERS + ['>', '<']).include?(@data.currentByte)
|
|
return [attrName.join(''), attrValue.join('')]
|
|
elsif ASCII_UPPERCASE.include?(@data.currentByte)
|
|
attrValue.push(@data.currentByte.downcase)
|
|
else
|
|
attrValue.push(@data.currentByte)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
class ContentAttrParser
|
|
def initialize(data)
|
|
@data = data
|
|
end
|
|
def parse
|
|
begin
|
|
#Skip to the first ";"
|
|
@data.position = 0
|
|
@data.jumpTo(';')
|
|
@data.position += 1
|
|
@data.skip
|
|
#Check if the attr name is charset
|
|
#otherwise return
|
|
@data.jumpTo('charset')
|
|
@data.position += 1
|
|
@data.skip
|
|
unless @data.currentByte == '='
|
|
#If there is no = sign keep looking for attrs
|
|
return nil
|
|
end
|
|
@data.position += 1
|
|
@data.skip
|
|
#Look for an encoding between matching quote marks
|
|
if ['"', "'"].include?(@data.currentByte)
|
|
quoteMark = @data.currentByte
|
|
@data.position += 1
|
|
oldPosition = @data.position
|
|
@data.jumpTo(quoteMark)
|
|
return @data[oldPosition ... @data.position]
|
|
else
|
|
#Unquoted value
|
|
oldPosition = @data.position
|
|
begin
|
|
@data.findNext(SPACE_CHARACTERS)
|
|
return @data[oldPosition ... @data.position]
|
|
rescue EOF
|
|
#Return the whole remaining value
|
|
return @data[oldPosition .. -1]
|
|
end
|
|
end
|
|
rescue EOF
|
|
return nil
|
|
end
|
|
end
|
|
end
|
|
|
|
# Determine if a string is a supported encoding
|
|
def self.isValidEncoding(encoding)
|
|
(not encoding.nil? and encoding.kind_of?(String) and ENCODINGS.include?(encoding.downcase.strip))
|
|
end
|
|
|
|
end
|