require 'html5lib/constants'
require 'html5lib/inputstream'

module HTML5lib

# This class takes care of tokenizing HTML.
#
# * @currentToken
#   Holds the token that is currently being processed.
#
# * @state
#   Holds a reference to the method to be invoked... XXX
#
# * @states
#   Holds a mapping between states and methods that implement the state.
#
# * @stream
#   Points to HTMLInputStream object.

class HTMLTokenizer
    attr_accessor :contentModelFlag, :currentToken
    attr_reader :stream

    # XXX need to fix documentation

    def initialize(stream, options={})
        @stream = HTMLInputStream.new(stream, options)

        @states = {
            :data => :dataState,
            :entityData => :entityDataState,
            :tagOpen => :tagOpenState,
            :closeTagOpen => :closeTagOpenState,
            :tagName => :tagNameState,
            :beforeAttributeName => :beforeAttributeNameState,
            :attributeName => :attributeNameState,
            :afterAttributeName => :afterAttributeNameState,
            :beforeAttributeValue => :beforeAttributeValueState,
            :attributeValueDoubleQuoted => :attributeValueDoubleQuotedState,
            :attributeValueSingleQuoted => :attributeValueSingleQuotedState,
            :attributeValueUnQuoted => :attributeValueUnQuotedState,
            :bogusComment => :bogusCommentState,
            :markupDeclarationOpen => :markupDeclarationOpenState,
            :comment => :commentState,
            :commentDash => :commentDashState,
            :commentEnd => :commentEndState,
            :doctype => :doctypeState,
            :beforeDoctypeName => :beforeDoctypeNameState,
            :doctypeName => :doctypeNameState,
            :afterDoctypeName => :afterDoctypeNameState,
            :bogusDoctype => :bogusDoctypeState
        }

        # Setup the initial tokenizer state
        @contentModelFlag = :PCDATA
        @state = @states[:data]

        # The current token being created
        @currentToken = nil

        # Tokens to be processed.
        @tokenQueue = []
    end

    # This is where the magic happens.
    #
    # We do our usually processing through the states and when we have a token
    # to return we yield the token which pauses processing until the next token
    # is requested.
    def each
        @stream.reset
        @tokenQueue = []
        # Start processing. When EOF is reached @state will return false
        # instead of true and the loop will terminate.
        while send @state
            while not @tokenQueue.empty?
                yield @tokenQueue.shift
            end
        end
    end

    # Below are various helper functions the tokenizer states use worked out.
    
    # If the next character is a '>', convert the currentToken into
    # an EmptyTag

    def processSolidusInTag

        # We need to consume another character to make sure it's a ">"
        data = @stream.char

        if @currentToken[:type] == :StartTag and data == ">"
            @currentToken[:type] = :EmptyTag
        else
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Solidus (/) incorrectly placed in tag.")})
        end

        # The character we just consumed need to be put back on the stack so it
        # doesn't get lost...
        @stream.queue.push(data)
    end

    # This function returns either U+FFFD or the character based on the
    # decimal or hexadecimal representation. It also discards ";" if present.
    # If not present @tokenQueue.push({:type => :ParseError}") is invoked.

    def consumeNumberEntity(isHex)

        # XXX More need to be done here. For instance, #13 should prolly be
        # converted to #10 so we don't get \r (#13 is \r right?) in the DOM and
        # such. Thoughts on this appreciated.
        allowed = DIGITS
        radix = 10
        if isHex
            allowed = HEX_DIGITS
            radix = 16
        end

        char = [0xFFFD].pack('U')
        charStack = []

        # Consume all the characters that are in range while making sure we
        # don't hit an EOF.
        c = @stream.char
        while allowed.include?(c) and c != :EOF
            charStack.push(c)
            c = @stream.char
        end

        # Convert the set of characters consumed to an int.
        charAsInt = charStack.join('').to_i(radix)

        # If the integer is between 127 and 160 (so 128 and bigger and 159 and
        # smaller) we need to do the "windows trick".
        if (127...160).include? charAsInt
            #XXX - removed parse error from windows 1252 entity for now
            #we may want to reenable this later
            #@tokenQueue.push({:type => :ParseError, :data =>
            #  _("Entity used with illegal number (windows-1252 reference).")})

            charAsInt = ENTITIES_WINDOWS1252[charAsInt - 128]
        end

        # 0 is not a good number.
        if charAsInt == 0
            charAsInt = 65533
        end

        if charAsInt <= 0x10FFF
            char = [charAsInt].pack('U')
        else
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Numeric entity couldn't be converted to character.")})
        end

        # Discard the ; if present. Otherwise, put it back on the queue and
        # invoke parseError on parser.
        if c != ";"
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Numeric entity didn't end with ';'.")})
            @stream.queue.push(c)
        end

        return char
    end

    def consumeEntity
        char = nil
        charStack = [@stream.char]
        if charStack[0] == "#"
            # We might have a number entity here.
            charStack += [@stream.char, @stream.char]
            if charStack.include? :EOF
                # If we reach the end of the file put everything up to :EOF
                # back in the queue
                charStack = charStack[0...charStack.index(:EOF)]
                @stream.queue+= charStack
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Numeric entity expected. Got end of file instead.")})
            else
                if charStack[1].downcase == "x" \
                  and HEX_DIGITS.include? charStack[2]
                    # Hexadecimal entity detected.
                    @stream.queue.push(charStack[2])
                    char = consumeNumberEntity(true)
                elsif DIGITS.include? charStack[1]
                    # Decimal entity detected.
                    @stream.queue += charStack[1..-1]
                    char = consumeNumberEntity(false)
                else
                    # No number entity detected.
                    @stream.queue += charStack
                    @tokenQueue.push({:type => :ParseError, :data =>
                      _("Numeric entity expected but none found.")})
                end
            end
        # Break out if we reach the end of the file
        elsif charStack[0] == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Entity expected. Got end of file instead.")})
        else
            # At this point in the process might have named entity. Entities
            # are stored in the global variable "entities".
            #
            # Consume characters and compare to these to a substring of the
            # entity names in the list until the substring no longer matches.
            filteredEntityList = ENTITIES.keys
            filteredEntityList.reject! {|e| e[0].chr != charStack[0]}
            entityName = nil

            while charStack[-1] != :EOF
                name = charStack.join('')
                if filteredEntityList.any? {|e| e[0...name.length] == name}
                    filteredEntityList.reject! {|e| e[0...name.length] != name}
                    charStack.push(@stream.char)
                else
                    break
                end

                if ENTITIES.include? name
                    entityName = name
                end
            end

            if entityName != nil
                char = ENTITIES[entityName]

                # Check whether or not the last character returned can be
                # discarded or needs to be put back.
                if not charStack[-1] == ";"
                    @tokenQueue.push({:type => :ParseError, :data =>
                      _("Named entity didn't end with ';'.")})
                    @stream.queue += charStack[entityName.length..-1]
                end
            else
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Named entity expected. Got none.")})
                @stream.queue += charStack
            end
        end
        return char
    end

    # This method replaces the need for "entityInAttributeValueState".
    def processEntityInAttribute
        entity = consumeEntity
        if entity
            @currentToken[:data][-1][1] += entity
        else
            @currentToken[:data][-1][1] += "&"
        end
    end

    # This method is a generic handler for emitting the tags. It also sets
    # the state to "data" because that's what's needed after a token has been
    # emitted.
    def emitCurrentToken
        # Add token to the queue to be yielded
        @tokenQueue.push(@currentToken)
        @state = @states[:data]
    end


    # Below are the various tokenizer states worked out.

    # XXX AT Perhaps we should have Hixie run some evaluation on billions of
    # documents to figure out what the order of the various if and elsif
    # statements should be.

    def dataState
        data = @stream.char
        if data == "&" and (@contentModelFlag == :PCDATA or
            @contentModelFlag == :RCDATA)
            @state = @states[:entityData]
        elsif data == "<" and @contentModelFlag != :PLAINTEXT
            @state = @states[:tagOpen]
        elsif data == :EOF
            # Tokenization ends.
            return false
        elsif SPACE_CHARACTERS.include? data
            # Directly after emitting a token you switch back to the "data
            # state". At that point SPACE_CHARACTERS are important so they are
            # emitted separately.
            # XXX need to check if we don't need a special "spaces" flag on
            # characters.
            @tokenQueue.push({:type => :SpaceCharacters, :data =>
              data + @stream.charsUntil(SPACE_CHARACTERS, true)})
        else
            @tokenQueue.push({:type => :Characters, :data => 
              data + @stream.charsUntil(["&", "<"])})
        end
        return true
    end

    def entityDataState
        entity = consumeEntity
        if entity
            @tokenQueue.push({:type => :Characters, :data => entity})
        else
            @tokenQueue.push({:type => :Characters, :data => "&"})
        end
        @state = @states[:data]
        return true
    end

    def tagOpenState
        data = @stream.char
        if @contentModelFlag == :PCDATA
            if data == "!"
                @state = @states[:markupDeclarationOpen]
            elsif data == "/"
                @state = @states[:closeTagOpen]
            elsif data != :EOF and ASCII_LETTERS.include? data
                @currentToken =\
                  {:type => :StartTag, :name => data, :data => []}
                @state = @states[:tagName]
            elsif data == ">"
                # XXX In theory it could be something besides a tag name. But
                # do we really care?
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected tag name. Got '>' instead.")})
                @tokenQueue.push({:type => :Characters, :data => "<>"})
                @state = @states[:data]
            elsif data == "?"
                # XXX In theory it could be something besides a tag name. But
                # do we really care?
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected tag name. Got '?' instead (HTML doesn't " +
                  "support processing instructions).")})
                @stream.queue.push(data)
                @state = @states[:bogusComment]
            else
                # XXX
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected tag name. Got something else instead")})
                @tokenQueue.push({:type => :Characters, :data => "<"})
                @stream.queue.push(data)
                @state = @states[:data]
            end
        else
            # We know the content model flag is set to either RCDATA or CDATA
            # now because this state can never be entered with the PLAINTEXT
            # flag.
            if data == "/"
                @state = @states[:closeTagOpen]
            else
                @tokenQueue.push({:type => :Characters, :data => "<"})
                @stream.queue.insert(0, data)
                @state = @states[:data]
            end
        end
        return true
    end

    def closeTagOpenState
        if (@contentModelFlag == :RCDATA or @contentModelFlag == :CDATA)
            if @currentToken
                charStack = []

                # So far we know that "</" has been consumed. We now need to know
                # whether the next few characters match the name of last emitted
                # start tag which also happens to be the currentToken. We also need
                # to have the character directly after the characters that could
                # match the start tag name.
                (@currentToken[:name].length + 1).times do
                    charStack.push(@stream.char)
                    # Make sure we don't get hit by :EOF
                    break if charStack[-1] == :EOF
                end

                # Since this is just for checking. We put the characters back on
                # the stack.
                @stream.queue += charStack
            end

            if @currentToken and
              @currentToken[:name].downcase == 
                charStack[0...-1].join('').downcase and
              (SPACE_CHARACTERS + [">", "/", "<", :EOF]).include? charStack[-1]
                # Because the characters are correct we can safely switch to
                # PCDATA mode now. This also means we don't have to do it when
                # emitting the end tag token.
                @contentModelFlag = :PCDATA
            else
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected closing tag after seeing '</'. None found.")})
                @tokenQueue.push({:type => :Characters, :data => "</"})
                @state = @states[:data]

                # Need to return here since we don't want the rest of the
                # method to be walked through.
                return true
            end
        end

        if @contentModelFlag == :PCDATA
            data = @stream.char
            if data == :EOF
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected closing tag. Unexpected end of file.")})
                @tokenQueue.push({:type => :Characters, :data => "</"})
                @state = @states[:data]
            elsif ASCII_LETTERS.include? data
                @currentToken =\
                  {:type => :EndTag, :name => data, :data => []}
                @state = @states[:tagName]
            elsif data == ">"
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected closing tag. Got '>' instead. Ignoring '</>'.")})
                @state = @states[:data]
            else
                # XXX data can be _'_...
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected closing tag. Unexpected character '" + data + "' found.")})
                @stream.queue.push(data)
                @state = @states[:bogusComment]
            end
        end
        return true
    end

    def tagNameState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @state = @states[:beforeAttributeName]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in the tag name.")})
            emitCurrentToken
        elsif ASCII_LETTERS.include? data
            @currentToken[:name] += data +\
              @stream.charsUntil(ASCII_LETTERS, true)
        elsif data == ">"
            emitCurrentToken
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character when getting the tag name.")})
            emitCurrentToken
        elsif data == "/"
            processSolidusInTag
            @state = @states[:beforeAttributeName]
        else
            @currentToken[:name] += data
        end
        return true
    end

    def beforeAttributeNameState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @stream.charsUntil(SPACE_CHARACTERS, true)
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file. Expected attribute name instead.")})
            emitCurrentToken
        elsif ASCII_LETTERS.include? data
            @currentToken[:data].push([data, ""])
            @state = @states[:attributeName]
        elsif data == ">"
            emitCurrentToken
        elsif data == "/"
            processSolidusInTag
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character. Expected attribute name instead.")})
            emitCurrentToken
        else
            @currentToken[:data].push([data, ""])
            @state = @states[:attributeName]
        end
        return true
    end

    def attributeNameState
        data = @stream.char
        leavingThisState = true
        if data == "="
            @state = @states[:beforeAttributeValue]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in attribute name.")})
            emitCurrentToken
            leavingThisState = false
        elsif ASCII_LETTERS.include? data
            @currentToken[:data][-1][0] += data +\
              @stream.charsUntil(ASCII_LETTERS, true)
            leavingThisState = false
        elsif data == ">"
            # XXX If we emit here the attributes are converted to a dict
            # without being checked and when the code below runs we error
            # because data is a dict not a list
        elsif SPACE_CHARACTERS.include? data
            @state = @states[:afterAttributeName]
        elsif data == "/"
            processSolidusInTag
            @state = @states[:beforeAttributeName]
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character in attribute name.")})
            emitCurrentToken
            leavingThisState = false
        else
            @currentToken[:data][-1][0] += data
            leavingThisState = false
        end

        if leavingThisState
            # Attributes are not dropped at this stage. That happens when the
            # start tag token is emitted so values can still be safely appended
            # to attributes, but we do want to report the parse error in time.
            @currentToken[:data][0...-1].each {|name,value|
                if @currentToken[:data][-1][0] == name
                    @tokenQueue.push({:type => :ParseError, :data =>
                      _("Dropped duplicate attribute on tag.")})
                end
            }
            # XXX Fix for above XXX
            if data == ">"
                emitCurrentToken
            end
        end
        return true
    end

    def afterAttributeNameState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @stream.charsUntil(SPACE_CHARACTERS, true)
        elsif data == "="
            @state = @states[:beforeAttributeValue]
        elsif data == ">"
            emitCurrentToken
        elsif ASCII_LETTERS.include? data
            @currentToken[:data].push([data, ""])
            @state = @states[:attributeName]
        elsif data == "/"
            processSolidusInTag
            @state = @states[:beforeAttributeName]
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character. Expected = or end of tag.")})
            emitCurrentToken
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file. Expected = or end of tag.")})
            emitCurrentToken
        else
            @currentToken[:data].push([data, ""])
            @state = @states[:attributeName]
        end
        return true
    end

    def beforeAttributeValueState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @stream.charsUntil(SPACE_CHARACTERS, true)
        elsif data == "\""
            @state = @states[:attributeValueDoubleQuoted]
        elsif data == "&"
            @state = @states[:attributeValueUnQuoted]
            @stream.queue.push(data);
        elsif data == "'"
            @state = @states[:attributeValueSingleQuoted]
        elsif data == ">"
            emitCurrentToken
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character. Expected attribute value.")})
            emitCurrentToken
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file. Expected attribute value.")})
            emitCurrentToken
        else
            @currentToken[:data][-1][1] += data
            @state = @states[:attributeValueUnQuoted]
        end
        return true
    end

    def attributeValueDoubleQuotedState
        data = @stream.char
        if data == "\""
            @state = @states[:beforeAttributeName]
        elsif data == "&"
            processEntityInAttribute
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in attribute value (\").")})
            emitCurrentToken
        else
            @currentToken[:data][-1][1] += data +\
              @stream.charsUntil(["\"", "&"])
        end
        return true
    end

    def attributeValueSingleQuotedState
        data = @stream.char
        if data == "'"
            @state = @states[:beforeAttributeName]
        elsif data == "&"
            processEntityInAttribute
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in attribute value (').")})
            emitCurrentToken
        else
            @currentToken[:data][-1][1] += data +\
              @stream.charsUntil(["'", "&"])
        end
        return true
    end

    def attributeValueUnQuotedState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @state = @states[:beforeAttributeName]
        elsif data == "&"
            processEntityInAttribute
        elsif data == ">"
            emitCurrentToken
        elsif data == "<"
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected < character in attribute value.")})
            emitCurrentToken
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in attribute value.")})
            emitCurrentToken
        else
            @currentToken[:data][-1][1] += data + 
              @stream.charsUntil(["&", ">","<"] + SPACE_CHARACTERS)
        end
        return true
    end

    def bogusCommentState
        # Make a new comment token and give it as value all the characters
        # until the first > or :EOF (charsUntil checks for :EOF automatically)
        # and emit it.
        @tokenQueue.push(
          {:type => :Comment, :data => @stream.charsUntil((">"))})

        # Eat the character directly after the bogus comment which is either a
        # ">" or an :EOF.
        @stream.char
        @state = @states[:data]
        return true
    end

    def markupDeclarationOpenState
        charStack = [@stream.char, @stream.char]
        if charStack == ["-", "-"]
            @currentToken = {:type => :Comment, :data => ""}
            @state = @states[:comment]
        else
            5.times { charStack.push(@stream.char) }
            # Put in explicit :EOF check
            if ((not charStack.include? :EOF) and
                charStack.join("").upcase == "DOCTYPE")
                @currentToken =\
                  {:type => :Doctype, :name => "", :data => true}
                @state = @states[:doctype]
            else
                @tokenQueue.push({:type => :ParseError, :data =>
                  _("Expected '--' or 'DOCTYPE'. Not found.")})
                @stream.queue += charStack
                @state = @states[:bogusComment]
            end
        end
        return true
    end

    def commentState
        data = @stream.char
        if data == "-"
            @state = @states[:commentDash]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in comment.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            @currentToken[:data] += data + @stream.charsUntil("-")
        end
        return true
    end

    def commentDashState
        data = @stream.char
        if data == "-"
            @state = @states[:commentEnd]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in comment (-)")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            @currentToken[:data] += "-" + data +\
              @stream.charsUntil("-")
            # Consume the next character which is either a "-" or an :EOF as
            # well so if there's a "-" directly after the "-" we go nicely to
            # the "comment end state" without emitting a ParseError there.
            @stream.char
        end
        return true
    end

    def commentEndState
        data = @stream.char
        if data == ">"
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        elsif data == "-"
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected '-' after '--' found in comment.")})
            @currentToken[:data] += data
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in comment (--).")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            # XXX
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected character in comment found.")})
            @currentToken[:data] += "--" + data
            @state = @states[:comment]
        end
        return true
    end

    def doctypeState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
            @state = @states[:beforeDoctypeName]
        else
            @tokenQueue.push({:type => :ParseError, :data =>
              _("No space after literal string 'DOCTYPE'.")})
            @stream.queue.push(data)
            @state = @states[:beforeDoctypeName]
        end
        return true
    end

    def beforeDoctypeNameState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
        elsif ASCII_LOWERCASE.include? data
            @currentToken[:name] = data.upcase
            @state = @states[:doctypeName]
        elsif data == ">"
            # Character needs to be consumed per the specification so don't
            # invoke emitCurrentTokenWithParseError with :data as argument.
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected > character. Expected DOCTYPE name.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file. Expected DOCTYPE name.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            @currentToken[:name] = data
            @state = @states[:doctypeName]
        end
        return true
    end

    def doctypeNameState
        data = @stream.char
        needsDoctypeCheck = false
        if SPACE_CHARACTERS.include? data
            @state = @states[:afterDoctypeName]
            needsDoctypeCheck = true
        elsif data == ">"
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        elsif data == :EOF
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in DOCTYPE name.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            # We can't just uppercase everything that arrives here. For
            # instance, non-ASCII characters.
            if ASCII_LOWERCASE.include? data
                data = data.upcase
            end
            @currentToken[:name] += data
            needsDoctypeCheck = true
        end

        # After some iterations through this state it should eventually say
        # "HTML". Otherwise there's an error.
        if needsDoctypeCheck and @currentToken[:name] == "HTML"
            @currentToken[:data] = false
        end
        return true
    end

    def afterDoctypeNameState
        data = @stream.char
        if SPACE_CHARACTERS.include? data
        elsif data == ">"
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        elsif data == :EOF
            @currentToken[:data] = true
            # XXX EMIT
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in DOCTYPE.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        else
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Expected space or '>'. Got '" + data + "'")})
            @currentToken[:data] = true
            @state = @states[:bogusDoctype]
        end
        return true
    end

    def bogusDoctypeState
        data = @stream.char
        if data == ">"
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        elsif data == :EOF
            # XXX EMIT
            @stream.queue.push(data)
            @tokenQueue.push({:type => :ParseError, :data =>
              _("Unexpected end of file in bogus doctype.")})
            @tokenQueue.push(@currentToken)
            @state = @states[:data]
        end
        return true
    end

    def _(string); string; end
end

end