[FIXES BUILD] Fixed categories behavior and added id generation in import_storage. Something is still wrong with orphaned pages though

app/controllers/wiki_controller.rb

@@ -289,20 +289,13 @@ class WikiController < ApplicationController
 
   def parse_category
     @category = @params['category']
-    @categories = []
+    @categories = WikiReference.list_categories.sort
-    @pages_in_category = @web.select do |page|
+    page_names_in_category = WikiReference.pages_in_category(@category)
-      # FIXME: was PageRenderer.new(page.revisions.last).display_content.find_chunks(Category),
+    if (page_names_in_category.empty?)
-      # heinously slow
+      @pages_in_category = @web.select_all.by_name
-      page_categories = [] 
-      page_categories = page_categories.map { |cat| cat.list }.flatten
-      page_categories.each {|c| @categories << c unless @categories.include? c }
-      page_categories.include?(@category) 
-    end
-    @categories.sort!
-    if (@pages_in_category.empty?)
-      @pages_in_category = PageSet.new(@web).by_name
       @set_name = 'the web'
     else 
+      @pages_in_category = @web.select { |page| page_names_in_category.include?(page.name) }.by_name
       @set_name = "category '#{@category}'"
     end
   end

app/models/page.rb

@@ -1,7 +1,7 @@
 class Page < ActiveRecord::Base
   belongs_to :web
   has_many :revisions, :order => 'id'
-  has_many :wiki_references, :order => 'referenced_page_name'
+  has_many :wiki_references, :order => 'referenced_name'
   has_one :current_revision, :class_name => 'Revision', :order => 'id DESC'
 
   def revise(content, time, author, renderer)
@@ -15,7 +15,7 @@ class Page < ActiveRecord::Base
 
     # Try to render content to make sure that markup engine can take it,
     renderer.revision = Revision.new(
-        :page => self, :content => content, :author => author, :revised_at => time)
+       :page => self, :content => content, :author => author, :revised_at => time)
     renderer.display_content
 
     # A user may change a page, look at it and make some more changes - several times.
@@ -25,7 +25,7 @@ class Page < ActiveRecord::Base
     if (revisions_size > 0) && continous_revision?(time, author)
       current_revision.update_attributes(:content => content, :revised_at => time)
     else
-      Revision.create(:page => self, :content => content, :author => author, :revised_at => time)
+      revisions.create(:content => content, :author => author, :revised_at => time)
     end
     save
     self

app/models/page_observer.rb

@@ -3,13 +3,13 @@ class PageObserver < ActiveRecord::Observer
   
   def after_create(page)
     WikiReference.update_all("link_type = '#{WikiReference::LINKED_PAGE}'", 
-        ['referenced_page_name = ?', page.name])
+        ['referenced_name = ?', page.name])
   end
 
   def before_destroy(page)
     WikiReference.delete_all ['page_id = ?', page.id]
     WikiReference.update_all("link_type = '#{WikiReference::WANTED_PAGE}'", 
-        ['referenced_page_name = ?', page.name])
+        ['referenced_name = ?', page.name])
   end
 
 end

app/models/page_set.rb

@@ -82,7 +82,9 @@ class PageSet < Array
 
   def wiki_words
     self.inject([]) { |wiki_words, page|
-      wiki_words + page.wiki_references.map { |ref| ref.referenced_page_name }
+      wiki_words + page.wiki_references.
+          select { |ref| ref.link_type != WikiReference::CATEGORY }.
+          map { |ref| ref.referenced_name }
     }.flatten.uniq
   end
 

app/models/web.rb

@@ -56,6 +56,10 @@ class Web < ActiveRecord::Base
     PageSet.new(self, pages, condition)
   end
   
+  def select_all
+    PageSet.new(self, pages, nil)
+  end
+  
   private
 
     # Returns an array of all the wiki words in any current revision

app/models/wiki_reference.rb

@@ -3,9 +3,12 @@ class WikiReference < ActiveRecord::Base
   LINKED_PAGE = 'L'
   WANTED_PAGE = 'W'
   INCLUDED_PAGE = 'I'
+  CATEGORY = 'C'
 
   belongs_to :page
-  validates_inclusion_of :link_type, :in => [LINKED_PAGE, WANTED_PAGE, INCLUDED_PAGE]
+  validates_inclusion_of :link_type, :in => [LINKED_PAGE, WANTED_PAGE, INCLUDED_PAGE, CATEGORY]
+
+  # FIXME all finders below MUST restrict their results to pages belonging to a particular web
 
   def self.link_type(web, page_name)
     web.has_page?(page_name) ? LINKED_PAGE : WANTED_PAGE
@@ -13,24 +16,37 @@ class WikiReference < ActiveRecord::Base
 
   def self.pages_that_reference(page_name)
     query = 'SELECT name FROM pages JOIN wiki_references ON pages.id = wiki_references.page_id ' +
-        'WHERE wiki_references.referenced_page_name = ?'
+        'WHERE wiki_references.referenced_name = ?' +
+        "AND wiki_references.link_type in ('#{LINKED_PAGE}', '#{WANTED_PAGE}', '#{INCLUDED_PAGE}')"
     names = connection.select_all(sanitize_sql([query, page_name])).map { |row| row['name'] }
   end
 
   def self.pages_that_link_to(page_name)
     query = 'SELECT name FROM pages JOIN wiki_references ON pages.id = wiki_references.page_id ' +
-        'WHERE wiki_references.referenced_page_name = ? ' +
+        'WHERE wiki_references.referenced_name = ? ' +
         "AND wiki_references.link_type in ('#{LINKED_PAGE}', '#{WANTED_PAGE}')"
     names = connection.select_all(sanitize_sql([query, page_name])).map { |row| row['name'] }
   end
 
   def self.pages_that_include(page_name)
     query = 'SELECT name FROM pages JOIN wiki_references ON pages.id = wiki_references.page_id ' +
-        'WHERE wiki_references.referenced_page_name = ? ' +
+        'WHERE wiki_references.referenced_name = ? ' +
         "AND wiki_references.link_type = '#{INCLUDED_PAGE}'"
     names = connection.select_all(sanitize_sql([query, page_name])).map { |row| row['name'] }
   end
 
+  def self.pages_in_category(category)
+    query = 'SELECT name FROM pages JOIN wiki_references ON pages.id = wiki_references.page_id ' +
+        'WHERE wiki_references.referenced_name = ? ' +
+        "AND wiki_references.link_type = '#{CATEGORY}'"
+    names = connection.select_all(sanitize_sql([query, category])).map { |row| row['name'] }
+  end
+  
+  def self.list_categories
+    query = "SELECT DISTINCT referenced_name FROM wiki_references WHERE link_type = '#{CATEGORY}'"
+    connection.select_all(query).map { |row| row['referenced_name'] }
+  end
+
   def wiki_link?
     linked_page? or wanted_page?
   end

db/wiki_references.erbsql

@@ -4,6 +4,6 @@ CREATE TABLE wiki_references (
     updated_at <%= @datetime %> NOT NULL,
 
     page_id INTEGER NOT NULL,
-    referenced_page_name VARCHAR NOT NULL,
+    referenced_name VARCHAR NOT NULL,
     link_type CHAR(1) NOT NULL
 ) <%= create_options %>;

lib/page_renderer.rb

@@ -95,22 +95,27 @@ class PageRenderer
     wiki_word_chunks = result.find_chunks(WikiChunk::WikiLink)
     wiki_words = wiki_word_chunks.map { |c| ( c.escaped? ? nil : c.page_name ) }.compact.uniq
     
-    wiki_words.each do |referenced_page_name|
+    wiki_words.each do |referenced_name|
       # Links to self are always considered linked
-      if referenced_page_name == @revision.page.name
+      if referenced_name == @revision.page.name
         link_type = WikiReference::LINKED_PAGE
       else
-        link_type = WikiReference.link_type(@revision.page.web, referenced_page_name)
+        link_type = WikiReference.link_type(@revision.page.web, referenced_name)
       end
-      references.create :referenced_page_name => referenced_page_name, :link_type => link_type
+      references.create :referenced_name => referenced_name, :link_type => link_type
     end
     
     include_chunks = result.find_chunks(Include)
     includes = include_chunks.map { |c| ( c.escaped? ? nil : c.page_name ) }.compact.uniq
     includes.each do |included_page_name|
-      references.create :referenced_page_name => included_page_name, 
+      references.create :referenced_name => included_page_name, 
           :link_type => WikiReference::INCLUDED_PAGE
     end
+    
+    categories = result.find_chunks(Category).map { |cat| cat.list }.flatten
+    categories.each do |category|
+      references.create :referenced_name => category, :link_type => WikiReference::CATEGORY
+    end
     result
   end
 end

script/import_storage

@@ -76,7 +76,7 @@ expected_page_rb_path = File.join(OPTIONS[:instiki], 'app/models/page.rb')
 raise "Instiki installation not found in #{OPTIONS[:instiki]}" unless File.file?(expected_page_rb_path)
 
 expected_snapshot_pattern = File.join(OPTIONS[:storage], '*.snapshot')
-raise "No snapshots found in OPTIONS[:storage]" if Dir[expected_snapshot_pattern].empty?
+raise "No snapshots found in #{expected_snapshot_pattern}" if Dir[expected_snapshot_pattern].empty?
 
 INSTIKI_ROOT = File.expand_path(OPTIONS[:instiki])
 
@@ -109,31 +109,61 @@ class Time
 end
 
 def sql_insert(table, hash)
-  output = "INSERT INTO #{table} ("
+  columns = hash.keys
-  output << hash.keys.join(", ")
 
-  output << ") VALUES ('"
+  values = columns.map { |column| hash[column] }
-  output << hash.values.map do |v|
+  values = values.map do |val|
-    case OPTIONS[:database] 
+    if val.nil?
-    when 'mysql', 'postgres'
+      'NULL'
-      v.to_s.gsub("'", "\\\\'")
+    else
-    when 'sqlite'
+      case OPTIONS[:database]
-      v.to_s.gsub("'", "''")
+      when 'mysql', 'postgres'
-    else 
+        escaped_value = val.to_s.gsub("'", "\\\\'")
-      raise "Unsupported database option #{OPTIONS[:database]}"
+      when 'sqlite'
+        escaped_value = val.to_s.gsub("'", "''")
+      else 
+       raise "Unsupported database option #{OPTIONS[:database]}"
+      end
+      "'#{escaped_value}'"
     end
-  end.join("', '")
+  end
-  output << "');"
+  
+  output = "INSERT INTO #{table} ("
+  output << columns.join(", ")
+
+  output << ") VALUES ("
+  output << values.join(", ")
+  output << ");"
   output
 end
 
+def delete_all(outfile)
+  %w(wiki_references revisions pages system webs).each { |table| outfile.puts "DELETE FROM #{table};" }
+end
+
+def next_id(key)
+  $ids ||= {}
+  if $ids[key].nil?
+    $ids[key] = 1
+  else
+    $ids[key] = $ids[key] + 1
+  end
+  $ids[key]
+end
+
+def current_id(key)
+  $ids[key] or raise "No curent ID for #{key.inspect}"
+end
+
 WikiService.storage_path = OPTIONS[:storage]
 wiki = WikiService.instance
 
 File.open(OPTIONS[:outfile], 'w') { |outfile|
+  delete_all(outfile)
+
   wiki.webs.each_pair do |web_name, web|
     outfile.puts sql_insert(:webs, {
-      :id               => web.object_id,
+      :id               => next_id(:web),
       :name             => web.name,
       :address          => web.address,
       :password         => web.password,
@@ -152,9 +182,12 @@ File.open(OPTIONS[:outfile], 'w') { |outfile|
 
     puts "Web #{web_name} has #{web.pages.keys.size} pages"
     web.pages.each_pair do |page_name, page|
+
+      outfile.puts "BEGIN TRANSACTION;"
+
       outfile.puts sql_insert(:pages, {
-        :id         => page.object_id,
+        :id         => next_id(:page),
-        :web_id     => web.object_id,
+        :web_id     => current_id(:web),
         :locked_by  => page.locked_by,
         :name       => page.name,
         :created_at => page.revisions.first.created_at.ansi,
@@ -165,8 +198,8 @@ File.open(OPTIONS[:outfile], 'w') { |outfile|
       page.revisions.each_with_index do |rev, i|
       
         outfile.puts sql_insert(:revisions, {
-          :id         => rev.object_id,
+          :id         => next_id(:revision),
-          :page_id    => page.object_id,
+          :page_id    => current_id(:page),
           :content    => rev.content,
           :author     => rev.author.to_s,
           :ip         => (rev.author.is_a?(Author) ? rev.author.ip : 'N/A'),
@@ -176,6 +209,9 @@ File.open(OPTIONS[:outfile], 'w') { |outfile|
         })
         puts "    Revision #{i} created at #{rev.created_at.ansi}"
       end
+
+      outfile.puts "COMMIT;"
+
     end
   end
 }

test/fixtures/wiki_references.yml

@@ -1,7 +1,7 @@
 my_way_1:
   id: 1
   page_id: 2
-  referenced_page_name: MyWay
+  referenced_name: MyWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -9,7 +9,7 @@ my_way_1:
 smart_engine_1:
   id: 2
   page_id: 3
-  referenced_page_name: SmartEngine
+  referenced_name: SmartEngine
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -17,7 +17,7 @@ smart_engine_1:
 that_way_1:
   id: 3
   page_id: 4
-  referenced_page_name: ThatWay
+  referenced_name: ThatWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -25,7 +25,7 @@ that_way_1:
 home_page_1:
   id: 4
   page_id: 1
-  referenced_page_name: HisWay
+  referenced_name: HisWay
   link_type: W
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -33,7 +33,7 @@ home_page_1:
 home_page_2:
   id: 5
   page_id: 1
-  referenced_page_name: MyWay
+  referenced_name: MyWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -41,7 +41,7 @@ home_page_2:
 home_page_3:
   id: 6
   page_id: 1
-  referenced_page_name: ThatWay
+  referenced_name: ThatWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -49,7 +49,7 @@ home_page_3:
 home_page_4:
   id: 7
   page_id: 1
-  referenced_page_name: SmartEngine
+  referenced_name: SmartEngine
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -57,7 +57,7 @@ home_page_4:
 first_page_1:
   id: 8
   page_id: 6
-  referenced_page_name: HisWay
+  referenced_name: HisWay
   link_type: W
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -65,7 +65,7 @@ first_page_1:
 first_page_2:
   id: 9
   page_id: 6
-  referenced_page_name: MyWay
+  referenced_name: MyWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -73,7 +73,7 @@ first_page_2:
 first_page_3:
   id: 10
   page_id: 6
-  referenced_page_name: ThatWay
+  referenced_name: ThatWay
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -81,7 +81,7 @@ first_page_3:
 first_page_4:
   id: 11
   page_id: 6
-  referenced_page_name: OverThere
+  referenced_name: OverThere
   link_type: W
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
@@ -89,7 +89,24 @@ first_page_4:
 first_page_5:
   id: 12
   page_id: 6
-  referenced_page_name: SmartEngine
+  referenced_name: SmartEngine
   link_type: L
   created_at: <%= Time.now %>
   updated_at: <%= Time.now %>
+
+oak_1:
+  id: 13
+  page_id: 7
+  referenced_name: trees
+  link_type: C
+  created_at: <%= Time.now %>
+  updated_at: <%= Time.now %>
+
+elephant_1:
+  id: 14
+  page_id: 8
+  referenced_name: animals
+  link_type: C
+  created_at: <%= Time.now %>
+  updated_at: <%= Time.now %>
+  

test/unit/page_renderer_test.rb

@@ -332,9 +332,9 @@ class PageRendererTest < Test::Unit::TestCase
         
     references = new_page.wiki_references(true)
     assert_equal 2, references.size
-    assert_equal 'HomePage', references[0].referenced_page_name
+    assert_equal 'HomePage', references[0].referenced_name
     assert_equal WikiReference::LINKED_PAGE, references[0].link_type
-    assert_equal 'NewPage', references[1].referenced_page_name
+    assert_equal 'NewPage', references[1].referenced_name
     assert_equal WikiReference::LINKED_PAGE, references[1].link_type
   end
 
@@ -344,10 +344,20 @@ class PageRendererTest < Test::Unit::TestCase
         
     references = new_page.wiki_references(true)
     assert_equal 1, references.size
-    assert_equal 'IncludedPage', references[0].referenced_page_name
+    assert_equal 'IncludedPage', references[0].referenced_name
     assert_equal WikiReference::INCLUDED_PAGE, references[0].link_type
   end
 
+  def test_references_creation_categories
+    new_page = @web.add_page('NewPage', "Foo\ncategory: NewPageCategory",
+        Time.local(2004, 4, 4, 16, 50), 'AlexeyVerkhovsky', test_renderer)
+
+    references = new_page.wiki_references(true)
+    assert_equal 1, references.size
+    assert_equal 'NewPageCategory', references[0].referenced_name
+    assert_equal WikiReference::CATEGORY, references[0].link_type
+  end
+
   private
 
   def add_sample_pages

test/unit/page_test.rb

@@ -93,9 +93,9 @@ class PageTest < Test::Unit::TestCase
     
     references = new_page.wiki_references(true)
     assert_equal 2, references.size
-    assert_equal 'WantedPage', references[0].referenced_page_name
+    assert_equal 'WantedPage', references[0].referenced_name
     assert_equal WikiReference::WANTED_PAGE, references[0].link_type
-    assert_equal 'WantedPage2', references[1].referenced_page_name
+    assert_equal 'WantedPage2', references[1].referenced_name
     assert_equal WikiReference::WANTED_PAGE, references[1].link_type
 
     wanted_page = Page.new(:web => web, :name => 'WantedPage')
@@ -105,9 +105,9 @@ class PageTest < Test::Unit::TestCase
     # reference NewPage -> WantedPage2 should remain the same
     references = new_page.wiki_references(true)
     assert_equal 2, references.size
-    assert_equal 'WantedPage', references[0].referenced_page_name
+    assert_equal 'WantedPage', references[0].referenced_name
     assert_equal WikiReference::LINKED_PAGE, references[0].link_type
-    assert_equal 'WantedPage2', references[1].referenced_page_name
+    assert_equal 'WantedPage2', references[1].referenced_name
     assert_equal WikiReference::WANTED_PAGE, references[1].link_type
   end