Make bulk saving more flexible.
* Database#save(doc, true) caches the doc in a database-specific bulk cache. * Database#save(doc, false), default, saves normally, bulk saving and emptying the cache if one exists. * The cache is automatically saved on Database#save if it excedes a configurable limit, 50 by default. * Database#bulk_save without arguments saves and clears the bulk save cache.
This commit is contained in:
parent
dd7f109878
commit
d8d5645ebd
|
@ -4,6 +4,7 @@ require "base64"
|
||||||
module CouchRest
|
module CouchRest
|
||||||
class Database
|
class Database
|
||||||
attr_reader :server, :host, :name, :root
|
attr_reader :server, :host, :name, :root
|
||||||
|
attr_accessor :bulk_save_cache_limit
|
||||||
|
|
||||||
# Create a CouchRest::Database adapter for the supplied CouchRest::Server
|
# Create a CouchRest::Database adapter for the supplied CouchRest::Server
|
||||||
# and database name.
|
# and database name.
|
||||||
|
@ -18,6 +19,8 @@ module CouchRest
|
||||||
@host = server.uri
|
@host = server.uri
|
||||||
@root = "#{host}/#{name}"
|
@root = "#{host}/#{name}"
|
||||||
@streamer = Streamer.new(self)
|
@streamer = Streamer.new(self)
|
||||||
|
@bulk_save_cache = []
|
||||||
|
@bulk_save_cache_limit = 50
|
||||||
end
|
end
|
||||||
|
|
||||||
# returns the database's uri
|
# returns the database's uri
|
||||||
|
@ -106,10 +109,20 @@ module CouchRest
|
||||||
# documents on the client side because POST has the curious property of
|
# documents on the client side because POST has the curious property of
|
||||||
# being automatically retried by proxies in the event of network
|
# being automatically retried by proxies in the event of network
|
||||||
# segmentation and lost responses.
|
# segmentation and lost responses.
|
||||||
def save doc
|
#
|
||||||
|
# If <tt>bulk</tt> is true (false by default) the document is cached for bulk-saving later.
|
||||||
|
# Bulk saving happens automatically when #bulk_save_cache limit is exceded, or on the next non bulk save.
|
||||||
|
def save (doc, bulk = false)
|
||||||
if doc['_attachments']
|
if doc['_attachments']
|
||||||
doc['_attachments'] = encode_attachments(doc['_attachments'])
|
doc['_attachments'] = encode_attachments(doc['_attachments'])
|
||||||
end
|
end
|
||||||
|
if bulk
|
||||||
|
@bulk_save_cache << doc
|
||||||
|
return bulk_save if @bulk_save_cache.length >= @bulk_save_cache_limit
|
||||||
|
return doc
|
||||||
|
elsif !bulk && @bulk_save_cache.length > 0
|
||||||
|
bulk_save
|
||||||
|
end
|
||||||
result = if doc['_id']
|
result = if doc['_id']
|
||||||
slug = CGI.escape(doc['_id'])
|
slug = CGI.escape(doc['_id'])
|
||||||
CouchRest.put "#{@root}/#{slug}", doc
|
CouchRest.put "#{@root}/#{slug}", doc
|
||||||
|
@ -131,7 +144,13 @@ module CouchRest
|
||||||
|
|
||||||
# POST an array of documents to CouchDB. If any of the documents are
|
# POST an array of documents to CouchDB. If any of the documents are
|
||||||
# missing ids, supply one from the uuid cache.
|
# missing ids, supply one from the uuid cache.
|
||||||
def bulk_save docs
|
#
|
||||||
|
# If called with no arguments, bulk saves the cache of documents to be bulk saved.
|
||||||
|
def bulk_save (docs = nil)
|
||||||
|
if docs.nil?
|
||||||
|
docs = @bulk_save_cache
|
||||||
|
@bulk_save_cache = []
|
||||||
|
end
|
||||||
ids, noids = docs.partition{|d|d['_id']}
|
ids, noids = docs.partition{|d|d['_id']}
|
||||||
uuid_count = [noids.length, @server.uuid_batch_count].max
|
uuid_count = [noids.length, @server.uuid_batch_count].max
|
||||||
noids.each do |doc|
|
noids.each do |doc|
|
||||||
|
|
|
@ -190,6 +190,16 @@ describe CouchRest::Database do
|
||||||
@db.get('twoB')
|
@db.get('twoB')
|
||||||
end.should raise_error(RestClient::ResourceNotFound)
|
end.should raise_error(RestClient::ResourceNotFound)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it "should empty the bulk save cache if no documents are given" do
|
||||||
|
@db.save({"_id" => "bulk_cache_1", "val" => "test"}, true)
|
||||||
|
lambda do
|
||||||
|
@db.get('bulk_cache_1')
|
||||||
|
end.should raise_error(RestClient::ResourceNotFound)
|
||||||
|
@db.bulk_save
|
||||||
|
@db.get("bulk_cache_1")["val"].should == "test"
|
||||||
|
end
|
||||||
|
|
||||||
it "should raise an error that is useful for recovery" do
|
it "should raise an error that is useful for recovery" do
|
||||||
@r = @db.save({"_id" => "taken", "field" => "stuff"})
|
@r = @db.save({"_id" => "taken", "field" => "stuff"})
|
||||||
begin
|
begin
|
||||||
|
@ -400,7 +410,48 @@ describe CouchRest::Database do
|
||||||
now['them-keys'].should == 'huge'
|
now['them-keys'].should == 'huge'
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "cached bulk save" do
|
||||||
|
it "stores documents in a database-specific cache" do
|
||||||
|
td = {"_id" => "btd1", "val" => "test"}
|
||||||
|
@db.save(td, true)
|
||||||
|
@db.instance_variable_get("@bulk_save_cache").should == [td]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
it "doesn't save to the database until the configured cache size is exceded" do
|
||||||
|
@db.bulk_save_cache_limit = 3
|
||||||
|
td1 = {"_id" => "td1", "val" => true}
|
||||||
|
td2 = {"_id" => "td2", "val" => 4}
|
||||||
|
@db.save(td1, true)
|
||||||
|
@db.save(td2, true)
|
||||||
|
lambda do
|
||||||
|
@db.get(td1["_id"])
|
||||||
|
end.should raise_error(RestClient::ResourceNotFound)
|
||||||
|
lambda do
|
||||||
|
@db.get(td2["_id"])
|
||||||
|
end.should raise_error(RestClient::ResourceNotFound)
|
||||||
|
td3 = {"_id" => "td3", "val" => "foo"}
|
||||||
|
@db.save(td3, true)
|
||||||
|
@db.get(td1["_id"])["val"].should == td1["val"]
|
||||||
|
@db.get(td2["_id"])["val"].should == td2["val"]
|
||||||
|
@db.get(td3["_id"])["val"].should == td3["val"]
|
||||||
|
end
|
||||||
|
|
||||||
|
it "clears the bulk save cache the first time a non bulk save is requested" do
|
||||||
|
td1 = {"_id" => "blah", "val" => true}
|
||||||
|
td2 = {"_id" => "steve", "val" => 3}
|
||||||
|
@db.bulk_save_cache_limit = 50
|
||||||
|
@db.save(td1, true)
|
||||||
|
lambda do
|
||||||
|
@db.get(td1["_id"])
|
||||||
|
end.should raise_error(RestClient::ResourceNotFound)
|
||||||
|
@db.save(td2)
|
||||||
|
@db.get(td1["_id"])["val"].should == td1["val"]
|
||||||
|
@db.get(td2["_id"])["val"].should == td2["val"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
describe "DELETE existing document" do
|
describe "DELETE existing document" do
|
||||||
before(:each) do
|
before(:each) do
|
||||||
@r = @db.save({'lemons' => 'from texas', 'and' => 'spain'})
|
@r = @db.save({'lemons' => 'from texas', 'and' => 'spain'})
|
||||||
|
|
Loading…
Reference in a new issue