updater is simpler now that I learned about open_revs=all

This commit is contained in:
Chris Anderson 2009-03-14 19:00:26 -07:00
parent 8964a9b282
commit fbc21aacd9
2 changed files with 15 additions and 36 deletions

View file

@ -80,12 +80,12 @@ module CouchRest
def get(id, params = {}) def get(id, params = {})
slug = escape_docid(id) slug = escape_docid(id)
url = CouchRest.paramify_url("#{@uri}/#{slug}", params) url = CouchRest.paramify_url("#{@uri}/#{slug}", params)
puts url result = CouchRest.get(url)
hash = CouchRest.get(url) return result unless result.is_a?(Hash)
doc = if /^_design/ =~ hash["_id"] doc = if /^_design/ =~ result["_id"]
Design.new(hash) Design.new(result)
else else
Document.new(hash) Document.new(result)
end end
doc.database = self doc.database = self
doc doc

View file

@ -11,7 +11,7 @@ module CouchRest
puts "#{dbname} - #{olddb.info['doc_count']} docs" puts "#{dbname} - #{olddb.info['doc_count']} docs"
streamer = CouchRest::Streamer.new(olddb) streamer = CouchRest::Streamer.new(olddb)
streamer.view("_all_docs_by_seq") do |row| streamer.view("_all_docs_by_seq") do |row|
load_doc_for_row(row) if row load_row_docs(row) if row
maybe_flush_bulks maybe_flush_bulks
end end
flush_bulks! flush_bulks!
@ -20,53 +20,32 @@ module CouchRest
private private
def maybe_flush_bulks def maybe_flush_bulks
flush_bulks! if (@bulk_docs.length > 1000) flush_bulks! if (@bulk_docs.length > 99)
end end
def flush_bulks! def flush_bulks!
url = CouchRest.paramify_url "#{@newdb.uri}/_bulk_docs", {:all_or_nothing => true} url = CouchRest.paramify_url "#{@newdb.uri}/_bulk_docs", {:all_or_nothing => true}
puts "posting bulk to #{url}" puts "posting #{@bulk_docs.length} bulk docs to #{url}"
puts @bulk_docs.collect{|b|b.id}
begin begin
CouchRest.post url, {:docs => @bulk_docs} CouchRest.post url, {:docs => @bulk_docs}
@bulk_docs = []
rescue Exception => e rescue Exception => e
puts e.response puts e.response
raise e
end end
end end
def load_doc_for_row(row) def load_row_docs(row)
if row["value"]["conflicts"]
puts "doc #{row["id"]} has conflicts #{row.inspect}"
# load the doc, and it's conflicts
load_conflicts_doc(row)
elsif row["value"]["deleted_conflicts"]
puts "doc #{row["id"]} has deleted conflicts"
elsif row["value"]["deleted"]
# puts "doc #{row["id"]} is deleted"
@bulk_docs << {"_id" => row["id"], "_deleted" => true}
else
load_normal_doc(row)
end
end
def load_conflicts_doc(row)
results = @olddb.get(row["id"], {:open_revs => "all", :attachments => true}) results = @olddb.get(row["id"], {:open_revs => "all", :attachments => true})
results.select{|r|r["ok"]}.each do |r| results.select{|r|r["ok"]}.each do |r|
doc = r["ok"] doc = r["ok"]
if /^_/.match(doc["_id"]) && !/^_design/.match(doc["_id"])
puts "invalid docid #{doc["_id"]} -- trimming"
doc["_id"] = doc["_id"].sub('_','')
end
doc.delete('_rev') doc.delete('_rev')
@bulk_docs << doc @bulk_docs << doc
end end
end end
def load_normal_doc(row)
# puts row["id"]
doc = @olddb.get(row["id"], :attachments => true)
if /^_/.match(doc.id) && !/^_design/.match(doc.id)
puts "trimming invalid docid #{doc.id}"
doc["_id"] = doc.id.sub('_','')
end
doc.delete("_rev");
@bulk_docs << doc
end
end end
end end