CouchRest::Pager for paging through views

lib/pager.rb

@@ -0,0 +1,89 @@
+# paginate though 'gcharts/mp3-trk-dom-map' view and save
+
+# get 1000 records
+# truncate so that the key of the last record is not included in the page
+#  that key will be the first of the next page
+# (if the last key equals the first key, up the page size)
+# group the results by key
+# yield the group
+
+require File.dirname(__FILE__) + '/couchrest'
+
+module Enumerable
+  def group_by
+    inject({}) do |grouped, element|
+      (grouped[yield(element)] ||= []) << element
+      grouped
+    end
+  end unless [].respond_to?(:group_by)
+end
+
+class CouchRest
+  class Pager
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    def key_reduce(view, count, firstkey = nil, lastkey = nil, &block)
+      # start with no keys
+      startkey = firstkey
+      # lastprocessedkey = nil
+      keepgoing = true
+      
+      while keepgoing && viewrows = request_view(view, count, startkey)
+        startkey = viewrows.first['key']
+        endkey = viewrows.last['key']
+
+        if (startkey == endkey)
+          # we need to rerequest to get a bigger page
+          # so we know we have all the rows for that key
+          viewrows = @db.view(view, :key => startkey)['rows']
+          # we need to do an offset thing to find the next startkey
+          # otherwise we just get stuck
+          lastdocid = viewrows.last['id']
+          fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :count => 2)['rows']
+
+          newendkey = fornextloop.last['key']
+          if (newendkey == endkey)
+            keepgoing = false
+          else
+            startkey = newendkey
+          end
+          rows = viewrows
+        else
+          rows = []
+          for r in viewrows
+            if (lastkey && r['key'] == lastkey)
+              keepgoing = false
+              break
+            end
+            break if (r['key'] == endkey)
+            rows << r
+          end   
+          startkey = endkey
+        end
+
+        grouped = rows.group_by{|r|r['key']}
+        grouped.each do |k, rs|
+          vs = rs.collect{|r|r['value']}
+          yield(k,vs)
+        end
+        
+        # lastprocessedkey = rows.last['key']
+      end
+    end
+
+    def request_view view, count = nil, startkey = nil, endkey = nil
+      opts = {}
+      opts[:count] = count if count
+      opts[:startkey] = startkey if startkey
+      opts[:endkey] = endkey if endkey
+      
+      results = @db.view(view, opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+  end
+end

spec/pager_spec.rb

@@ -0,0 +1,94 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe CouchRest::Pager do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    begin
+      @cr.database(TESTDB).delete!
+    rescue RestClient::Request::RequestFailed
+    end
+    begin
+      @db = @cr.create_db(TESTDB)
+    rescue RestClient::Request::RequestFailed
+    end
+    @pager = CouchRest::Pager.new(@db)
+  end
+  
+  after(:all) do
+    begin
+      @db.delete!
+    rescue RestClient::Request::RequestFailed
+    end
+  end
+  
+  it "should store the db" do
+    @pager.db.should == @db
+  end
+  
+  describe "Pager with a view and docs" do
+    before(:all) do
+      @docs = []
+      100.times do |i|
+        @docs << ({:number => (i % 10)})
+      end
+      @db.bulk_save(@docs)
+      @db.save({
+        '_id' => '_design/magic',
+        'views' => {
+          'number' => {
+            'map' => 'function(doc){emit(doc.number,null)}'
+          }
+        }
+      })
+    end
+    
+    it "should have docs" do
+      @docs.length.should == 100
+      @db.documents['rows'].length.should == 101
+    end
+    
+    it "should have a view" do
+      @db.view('magic/number', :count => 10)['rows'][0]['key'].should == 0
+    end
+    
+    it "should yield once per key" do
+      results = {}
+      @pager.key_reduce('magic/number', 20) do |k,vs|
+        results[k] = vs.length
+      end
+      results[0].should == 10
+      results[3].should == 10
+    end
+    
+    it "with a small step size should yield once per key" do
+      results = {}
+      @pager.key_reduce('magic/number', 7) do |k,vs|
+        results[k] = vs.length
+      end
+      results[0].should == 10
+      results[3].should == 10
+      results[9].should == 10
+    end
+    it "with a large step size should yield once per key" do
+      results = {}
+      @pager.key_reduce('magic/number', 1000) do |k,vs|
+        results[k] = vs.length
+      end
+      results[0].should == 10
+      results[3].should == 10
+      results[9].should == 10
+    end
+    it "with a begin and end should only yield in the range (and leave out the lastkey)" do
+      results = {}
+      @pager.key_reduce('magic/number', 1000, 4, 7) do |k,vs|
+        results[k] = vs.length
+      end
+      results[0].should be_nil
+      results[4].should == 10
+      results[6].should == 10
+      results[7].should be_nil
+      results[8].should be_nil
+      results[9].should be_nil
+    end
+  end
+end

spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require File.dirname(__FILE__) + '/../lib/couchrest'
+
+COUCHHOST = "http://localhost:5985"
+TESTDB = 'couchrest-test'