Purify Categories

Apply the same methodology, as in Revision 432,
to the category chunk-handler. This completes
the replacement of all the code that looks like

  if string.is_utf8?
    do something
  else
    complain
  end

with code that looks like

  string.purify
  do something

lib/chunks/category.rb

@@ -18,7 +18,8 @@ class Category < Chunk::Abstract
 def initialize(match_data, content)
     super(match_data, content)
     @hidden = match_data[1]
-    @list = match_data[2].split(',').map { |c| c.to_s.is_utf8? ? c.strip.escapeHTML : nil }
+#    @list = match_data[2].split(',').map { |c| clean = c.purify; clean.strip.escapeHTML if clean }
+    @list = match_data[2].split(',').map { |c| clean = c.purify.strip.escapeHTML; clean if clean != ''}
     @list.compact!
     @unmask_text = ''
     if @hidden

lib/stringsupport.rb

@@ -31,7 +31,7 @@ class String
        |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
        | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
        |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
-     )+\Z/nx;
+     )*\Z/nx;
 #++
      
 # Check whether a string is valid utf-8

test/unit/chunks/category_test.rb

@@ -12,6 +12,12 @@ class CategoryTest < Test::Unit::TestCase
 	match(Category, ':category: test', :list => ['test'], :hidden => ':')
   end
 
+  def test_no_category
+	match(Category, 'category:  ', :list => [], :hidden => nil)
+	match(Category, 'category :   chunk test  , ', :list => ['chunk test'], :hidden => nil)
+	match(Category, ':category:', :list => [], :hidden => ':')
+  end
+
   def test_multiple_categories
 	match(Category, 'category: test, multiple', :list => ['test', 'multiple'], :hidden => nil)
 	match(Category, 'category : chunk test , multi category,regression test case  ', 
@@ -26,4 +32,9 @@ class CategoryTest < Test::Unit::TestCase
 	)
   end
 
+  def test_multiple_categories_invalid_utf8
+	match(Category, "category: test, multiple,\000egg", :list => ['test', 'multiple', 'egg'], :hidden => nil)
+	match(Category, "category : chunk test , multi category,,e\000gg", :list => ['chunk test', 'multi category', 'egg'], :hidden => nil)
+  end
+
 end