From dd7f1098782e9fe94f6b9aeccb0a1384c02f110b Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Sun, 14 Dec 2008 17:29:15 -0600 Subject: [PATCH 1/3] Add support for database compaction. --- lib/couchrest/core/database.rb | 7 ++++++- spec/couchrest/core/database_spec.rb | 12 +++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/couchrest/core/database.rb b/lib/couchrest/core/database.rb index 6f19128..184d362 100644 --- a/lib/couchrest/core/database.rb +++ b/lib/couchrest/core/database.rb @@ -149,7 +149,12 @@ module CouchRest slug = CGI.escape(doc['_id']) CouchRest.delete "#{@root}/#{slug}?rev=#{doc['_rev']}" end - + + # Compact the database, removing old document revisions and optimizing space use. + def compact! + CouchRest.post "#{@root}/_compact" + end + # DELETE the database itself. This is not undoable and could be rather # catastrophic. Use with care! def delete! diff --git a/spec/couchrest/core/database_spec.rb b/spec/couchrest/core/database_spec.rb index 5fb2e70..1be153c 100644 --- a/spec/couchrest/core/database_spec.rb +++ b/spec/couchrest/core/database_spec.rb @@ -461,6 +461,16 @@ describe CouchRest::Database do end end + + describe "compacting a database" do + it "should compact the database" do + db = @cr.database('couchrest-test') + # r = + db.compact! + # r['ok'].should == true + end + end + describe "deleting a database" do it "should start with the test database" do @cr.databases.should include('couchrest-test') @@ -475,4 +485,4 @@ describe CouchRest::Database do end -end \ No newline at end of file +end From d8d5645ebd0b457633d27b469a96685fc016dccd Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Sun, 14 Dec 2008 23:17:35 -0600 Subject: [PATCH 2/3] Make bulk saving more flexible. * Database#save(doc, true) caches the doc in a database-specific bulk cache. * Database#save(doc, false), default, saves normally, bulk saving and emptying the cache if one exists. * The cache is automatically saved on Database#save if it excedes a configurable limit, 50 by default. * Database#bulk_save without arguments saves and clears the bulk save cache. --- lib/couchrest/core/database.rb | 23 ++++++++++-- spec/couchrest/core/database_spec.rb | 53 +++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/lib/couchrest/core/database.rb b/lib/couchrest/core/database.rb index 184d362..6aecf70 100644 --- a/lib/couchrest/core/database.rb +++ b/lib/couchrest/core/database.rb @@ -4,6 +4,7 @@ require "base64" module CouchRest class Database attr_reader :server, :host, :name, :root + attr_accessor :bulk_save_cache_limit # Create a CouchRest::Database adapter for the supplied CouchRest::Server # and database name. @@ -18,6 +19,8 @@ module CouchRest @host = server.uri @root = "#{host}/#{name}" @streamer = Streamer.new(self) + @bulk_save_cache = [] + @bulk_save_cache_limit = 50 end # returns the database's uri @@ -106,10 +109,20 @@ module CouchRest # documents on the client side because POST has the curious property of # being automatically retried by proxies in the event of network # segmentation and lost responses. - def save doc + # + # If bulk is true (false by default) the document is cached for bulk-saving later. + # Bulk saving happens automatically when #bulk_save_cache limit is exceded, or on the next non bulk save. + def save (doc, bulk = false) if doc['_attachments'] doc['_attachments'] = encode_attachments(doc['_attachments']) end + if bulk + @bulk_save_cache << doc + return bulk_save if @bulk_save_cache.length >= @bulk_save_cache_limit + return doc + elsif !bulk && @bulk_save_cache.length > 0 + bulk_save + end result = if doc['_id'] slug = CGI.escape(doc['_id']) CouchRest.put "#{@root}/#{slug}", doc @@ -131,7 +144,13 @@ module CouchRest # POST an array of documents to CouchDB. If any of the documents are # missing ids, supply one from the uuid cache. - def bulk_save docs + # + # If called with no arguments, bulk saves the cache of documents to be bulk saved. + def bulk_save (docs = nil) + if docs.nil? + docs = @bulk_save_cache + @bulk_save_cache = [] + end ids, noids = docs.partition{|d|d['_id']} uuid_count = [noids.length, @server.uuid_batch_count].max noids.each do |doc| diff --git a/spec/couchrest/core/database_spec.rb b/spec/couchrest/core/database_spec.rb index 1be153c..f30ef06 100644 --- a/spec/couchrest/core/database_spec.rb +++ b/spec/couchrest/core/database_spec.rb @@ -190,6 +190,16 @@ describe CouchRest::Database do @db.get('twoB') end.should raise_error(RestClient::ResourceNotFound) end + + it "should empty the bulk save cache if no documents are given" do + @db.save({"_id" => "bulk_cache_1", "val" => "test"}, true) + lambda do + @db.get('bulk_cache_1') + end.should raise_error(RestClient::ResourceNotFound) + @db.bulk_save + @db.get("bulk_cache_1")["val"].should == "test" + end + it "should raise an error that is useful for recovery" do @r = @db.save({"_id" => "taken", "field" => "stuff"}) begin @@ -400,7 +410,48 @@ describe CouchRest::Database do now['them-keys'].should == 'huge' end end - + + describe "cached bulk save" do + it "stores documents in a database-specific cache" do + td = {"_id" => "btd1", "val" => "test"} + @db.save(td, true) + @db.instance_variable_get("@bulk_save_cache").should == [td] + + end + + it "doesn't save to the database until the configured cache size is exceded" do + @db.bulk_save_cache_limit = 3 + td1 = {"_id" => "td1", "val" => true} + td2 = {"_id" => "td2", "val" => 4} + @db.save(td1, true) + @db.save(td2, true) + lambda do + @db.get(td1["_id"]) + end.should raise_error(RestClient::ResourceNotFound) + lambda do + @db.get(td2["_id"]) + end.should raise_error(RestClient::ResourceNotFound) + td3 = {"_id" => "td3", "val" => "foo"} + @db.save(td3, true) + @db.get(td1["_id"])["val"].should == td1["val"] + @db.get(td2["_id"])["val"].should == td2["val"] + @db.get(td3["_id"])["val"].should == td3["val"] + end + + it "clears the bulk save cache the first time a non bulk save is requested" do + td1 = {"_id" => "blah", "val" => true} + td2 = {"_id" => "steve", "val" => 3} + @db.bulk_save_cache_limit = 50 + @db.save(td1, true) + lambda do + @db.get(td1["_id"]) + end.should raise_error(RestClient::ResourceNotFound) + @db.save(td2) + @db.get(td1["_id"])["val"].should == td1["val"] + @db.get(td2["_id"])["val"].should == td2["val"] + end + end + describe "DELETE existing document" do before(:each) do @r = @db.save({'lemons' => 'from texas', 'and' => 'spain'}) From 84382d8af4bce5a9190a76c41060560f83aa44c1 Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Mon, 15 Dec 2008 10:27:53 -0600 Subject: [PATCH 3/3] Removed model create/update callbacks and integrated with new bulk save infrastructure. --- lib/couchrest/core/database.rb | 2 +- lib/couchrest/core/document.rb | 7 ++--- lib/couchrest/core/model.rb | 38 +++++++--------------------- spec/couchrest/core/document_spec.rb | 17 ++++++++++++- spec/couchrest/core/model_spec.rb | 6 ++--- 5 files changed, 33 insertions(+), 37 deletions(-) diff --git a/lib/couchrest/core/database.rb b/lib/couchrest/core/database.rb index 6aecf70..751c239 100644 --- a/lib/couchrest/core/database.rb +++ b/lib/couchrest/core/database.rb @@ -119,7 +119,7 @@ module CouchRest if bulk @bulk_save_cache << doc return bulk_save if @bulk_save_cache.length >= @bulk_save_cache_limit - return doc + return {"ok" => true} # Compatibility with Document#save elsif !bulk && @bulk_save_cache.length > 0 bulk_save end diff --git a/lib/couchrest/core/document.rb b/lib/couchrest/core/document.rb index 1df61ea..71a5135 100644 --- a/lib/couchrest/core/document.rb +++ b/lib/couchrest/core/document.rb @@ -35,9 +35,10 @@ module CouchRest # Saves the document to the db using create or update. Also runs the :save # callbacks. Sets the _id and _rev fields based on # CouchDB's response. - def save + # If bulk is true (defaults to false) the document is cached for bulk save. + def save(bulk = false) raise ArgumentError, "doc.database required for saving" unless database - result = database.save self + result = database.save self, bulk result['ok'] end @@ -57,4 +58,4 @@ module CouchRest end -end \ No newline at end of file +end diff --git a/lib/couchrest/core/model.rb b/lib/couchrest/core/model.rb index 0fccb27..56333d9 100644 --- a/lib/couchrest/core/model.rb +++ b/lib/couchrest/core/model.rb @@ -215,8 +215,9 @@ module CouchRest # on the document whenever saving occurs. CouchRest uses a pretty # decent time format by default. See Time#to_json def timestamps! - before(:create) do - self['updated_at'] = self['created_at'] = Time.now + before(:save) do + self['updated_at'] = Time.now + self['created_at'] = self['updated_at'] if new_document? end before(:update) do self['updated_at'] = Time.now @@ -478,19 +479,10 @@ module CouchRest # for compatibility with old-school frameworks alias :new_record? :new_document? - # We override this to create the create and update callback opportunities. - # I think we should drop those and just have save. If you care, in your callback, - # check new_document? - def save actually=false - if actually - super() - else - if new_document? - create - else - update - end - end + # Overridden to set the unique ID. + def save bulk = false + set_unique_id if self.respond_to?(:set_unique_id) + super(bulk) end # Saves the document to the db using create or update. Raises an exception @@ -498,10 +490,6 @@ module CouchRest def save! raise "#{self.inspect} failed to save" unless self.save end - - def update - save :actually - end # Deletes the document from the database. Runs the :delete callbacks. # Removes the _id and _rev fields, preparing the @@ -515,12 +503,6 @@ module CouchRest result['ok'] end - def create - # can we use the callbacks for this? - set_unique_id if self.respond_to?(:set_unique_id) - save :actually - end - private def apply_defaults @@ -549,9 +531,7 @@ module CouchRest end include ::Extlib::Hook - # todo: drop create and update hooks... - # (use new_record? in callbacks if you care) - register_instance_hooks :save, :create, :update, :destroy + register_instance_hooks :save, :destroy end # class Model -end # module CouchRest \ No newline at end of file +end # module CouchRest diff --git a/spec/couchrest/core/document_spec.rb b/spec/couchrest/core/document_spec.rb index 31c7151..485f638 100644 --- a/spec/couchrest/core/document_spec.rb +++ b/spec/couchrest/core/document_spec.rb @@ -53,6 +53,21 @@ describe CouchRest::Document, "saving using a database" do end end +describe CouchRest::Document, "bulk saving" do + before :all do + @db = reset_test_db! + end + + it "should use the document bulk save cache" do + doc = CouchRest::Document.new({"_id" => "bulkdoc", "val" => 3}) + doc.database = @db + doc.save(true) + lambda { doc.database.get(doc["_id"]) }.should raise_error(RestClient::ResourceNotFound) + doc.database.bulk_save + doc.database.get(doc["_id"])["val"].should == doc["val"] + end +end + describe "getting from a database" do before(:all) do @db = reset_test_db! @@ -93,4 +108,4 @@ describe "destroying a document from a db" do @doc = CouchRest::Document.new("key" => [1,2,3], :more => "values") lambda{@doc.destroy}.should raise_error(ArgumentError) end -end \ No newline at end of file +end diff --git a/spec/couchrest/core/model_spec.rb b/spec/couchrest/core/model_spec.rb index 466b850..9d4a4e3 100644 --- a/spec/couchrest/core/model_spec.rb +++ b/spec/couchrest/core/model_spec.rb @@ -71,9 +71,9 @@ class Article < CouchRest::Model timestamps! - before(:create, :generate_slug_from_title) + before(:save, :generate_slug_from_title) def generate_slug_from_title - self['slug'] = title.downcase.gsub(/[^a-z0-9]/,'-').squeeze('-').gsub(/^\-|\-$/,'') + self['slug'] = title.downcase.gsub(/[^a-z0-9]/,'-').squeeze('-').gsub(/^\-|\-$/,'') if new_document? end end @@ -679,4 +679,4 @@ describe CouchRest::Model do lambda{Basic.get(@obj.id)}.should raise_error end end -end \ No newline at end of file +end