From 116255dc0d3b6092d419d9d6c6e08b515fa3094d Mon Sep 17 00:00:00 2001
From: Jacques Distler <distler@golem.ph.utexas.edu>
Date: Mon, 7 Sep 2009 20:38:09 -0500
Subject: [PATCH] Purify Categories

Apply the same methodology, as in Revision 432,
to the category chunk-handler. This completes
the replacement of all the code that looks like

  if string.is_utf8?
    do something
  else
    complain
  end

with code that looks like

  string.purify
  do something
---
 lib/chunks/category.rb            |  3 ++-
 lib/stringsupport.rb              |  2 +-
 test/unit/chunks/category_test.rb | 11 +++++++++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/lib/chunks/category.rb b/lib/chunks/category.rb
index ee473ae0..fce66589 100644
--- a/lib/chunks/category.rb
+++ b/lib/chunks/category.rb
@@ -18,7 +18,8 @@ class Category < Chunk::Abstract
 def initialize(match_data, content)
     super(match_data, content)
     @hidden = match_data[1]
-    @list = match_data[2].split(',').map { |c| c.to_s.is_utf8? ? c.strip.escapeHTML : nil }
+#    @list = match_data[2].split(',').map { |c| clean = c.purify; clean.strip.escapeHTML if clean }
+    @list = match_data[2].split(',').map { |c| clean = c.purify.strip.escapeHTML; clean if clean != ''}
     @list.compact!
     @unmask_text = ''
     if @hidden
diff --git a/lib/stringsupport.rb b/lib/stringsupport.rb
index 58f74592..2a051d3f 100644
--- a/lib/stringsupport.rb
+++ b/lib/stringsupport.rb
@@ -31,7 +31,7 @@ class String
        |  \xF0[\x90-\xBF][\x80-\xBF]{2}     # planes 1-3
        | [\xF1-\xF3][\x80-\xBF]{3}          # planes 4-15
        |  \xF4[\x80-\x8F][\x80-\xBF]{2}     # plane 16
-     )+\Z/nx;
+     )*\Z/nx;
 #++
      
 # Check whether a string is valid utf-8
diff --git a/test/unit/chunks/category_test.rb b/test/unit/chunks/category_test.rb
index 86c1ca51..b105801a 100755
--- a/test/unit/chunks/category_test.rb
+++ b/test/unit/chunks/category_test.rb
@@ -12,6 +12,12 @@ class CategoryTest < Test::Unit::TestCase
 	match(Category, ':category: test', :list => ['test'], :hidden => ':')
   end
 
+  def test_no_category
+	match(Category, 'category:  ', :list => [], :hidden => nil)
+	match(Category, 'category :   chunk test  , ', :list => ['chunk test'], :hidden => nil)
+	match(Category, ':category:', :list => [], :hidden => ':')
+  end
+
   def test_multiple_categories
 	match(Category, 'category: test, multiple', :list => ['test', 'multiple'], :hidden => nil)
 	match(Category, 'category : chunk test , multi category,regression test case  ', 
@@ -26,4 +32,9 @@ class CategoryTest < Test::Unit::TestCase
 	)
   end
 
+  def test_multiple_categories_invalid_utf8
+	match(Category, "category: test, multiple,\000egg", :list => ['test', 'multiple', 'egg'], :hidden => nil)
+	match(Category, "category : chunk test , multi category,,e\000gg", :list => ['chunk test', 'multi category', 'egg'], :hidden => nil)
+  end
+
 end