most specs passing with CouchRest module
This commit is contained in:
Chris Anderson
parent
7028f7f7b3
commit
211331f4a6
5 changed files with 38 additions and 33 deletions
223
lib/couchrest/helper/file_manager.rb
Normal file
223
lib/couchrest/helper/file_manager.rb
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
require 'digest/md5'
|
||||
|
||||
module CouchRest
|
||||
class FileManager
|
||||
attr_reader :db
|
||||
attr_accessor :loud
|
||||
|
||||
LANGS = {"rb" => "ruby", "js" => "javascript"}
|
||||
MIMES = {
|
||||
"html" => "text/html",
|
||||
"htm" => "text/html",
|
||||
"png" => "image/png",
|
||||
"css" => "text/css",
|
||||
"js" => "test/javascript"
|
||||
}
|
||||
def initialize(dbname, host="http://localhost:5984")
|
||||
@db = CouchRest.new(host).database(dbname)
|
||||
end
|
||||
|
||||
def push_directory(push_dir, docid=nil)
|
||||
docid ||= push_dir.split('/').reverse.find{|part|!part.empty?}
|
||||
|
||||
pushfiles = Dir["#{push_dir}/**/*.*"].collect do |f|
|
||||
{f.split("#{push_dir}/").last => open(f).read}
|
||||
end
|
||||
|
||||
return if pushfiles.empty?
|
||||
|
||||
@attachments = {}
|
||||
@signatures = {}
|
||||
pushfiles.each do |file|
|
||||
name = file.keys.first
|
||||
value = file.values.first
|
||||
@signatures[name] = md5(value)
|
||||
|
||||
@attachments[name] = {
|
||||
"data" => value,
|
||||
"content_type" => MIMES[name.split('.').last]
|
||||
}
|
||||
end
|
||||
|
||||
doc = @db.get(docid) rescue nil
|
||||
|
||||
unless doc
|
||||
say "creating #{docid}"
|
||||
@db.save({"_id" => docid, "_attachments" => @attachments, "signatures" => @signatures})
|
||||
return
|
||||
end
|
||||
|
||||
# remove deleted docs
|
||||
to_be_removed = doc["signatures"].keys.select do |d|
|
||||
!pushfiles.collect{|p| p.keys.first}.include?(d)
|
||||
end
|
||||
|
||||
to_be_removed.each do |p|
|
||||
say "deleting #{p}"
|
||||
doc["signatures"].delete(p)
|
||||
doc["_attachments"].delete(p)
|
||||
end
|
||||
|
||||
# update existing docs:
|
||||
doc["signatures"].each do |path, sig|
|
||||
if (@signatures[path] == sig)
|
||||
say "no change to #{path}. skipping..."
|
||||
else
|
||||
say "replacing #{path}"
|
||||
doc["signatures"][path] = md5(@attachments[path]["data"])
|
||||
doc["_attachments"][path].delete("stub")
|
||||
doc["_attachments"][path].delete("length")
|
||||
doc["_attachments"][path]["data"] = @attachments[path]["data"]
|
||||
doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
# add in new files
|
||||
new_files = pushfiles.select{|d| !doc["signatures"].keys.include?( d.keys.first) }
|
||||
|
||||
new_files.each do |f|
|
||||
say "creating #{f}"
|
||||
path = f.keys.first
|
||||
content = f.values.first
|
||||
doc["signatures"][path] = md5(content)
|
||||
|
||||
doc["_attachments"][path] = {
|
||||
"data" => content,
|
||||
"content_type" => MIMES[path.split('.').last]
|
||||
}
|
||||
end
|
||||
|
||||
begin
|
||||
@db.save(doc)
|
||||
rescue Exception => e
|
||||
say e.message
|
||||
end
|
||||
end
|
||||
|
||||
def push_views(view_dir)
|
||||
designs = {}
|
||||
|
||||
Dir["#{view_dir}/**/*.*"].each do |design_doc|
|
||||
design_doc_parts = design_doc.split('/')
|
||||
next if /^lib\..*$/.match design_doc_parts.last
|
||||
pre_normalized_view_name = design_doc_parts.last.split("-")
|
||||
view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
|
||||
|
||||
folder = design_doc_parts[-2]
|
||||
|
||||
designs[folder] ||= {}
|
||||
designs[folder]["views"] ||= {}
|
||||
design_lang = design_doc_parts.last.split(".").last
|
||||
designs[folder]["language"] ||= LANGS[design_lang]
|
||||
|
||||
libs = ""
|
||||
Dir["#{view_dir}/lib.#{design_lang}"].collect do |global_lib|
|
||||
libs << open(global_lib).read
|
||||
libs << "\n"
|
||||
end
|
||||
Dir["#{view_dir}/#{folder}/lib.#{design_lang}"].collect do |global_lib|
|
||||
libs << open(global_lib).read
|
||||
libs << "\n"
|
||||
end
|
||||
if design_doc_parts.last =~ /-map/
|
||||
designs[folder]["views"]["#{view_name}-map"] ||= {}
|
||||
|
||||
designs[folder]["views"]["#{view_name}-map"]["map"] = read(design_doc, libs)
|
||||
|
||||
designs[folder]["views"]["#{view_name}-reduce"] ||= {}
|
||||
designs[folder]["views"]["#{view_name}-reduce"]["map"] = read(design_doc, libs)
|
||||
end
|
||||
|
||||
if design_doc_parts.last =~ /-reduce/
|
||||
designs[folder]["views"]["#{view_name}-reduce"] ||= {}
|
||||
|
||||
designs[folder]["views"]["#{view_name}-reduce"]["reduce"] = read(design_doc, libs)
|
||||
end
|
||||
end
|
||||
|
||||
# cleanup empty maps and reduces
|
||||
designs.each do |name, props|
|
||||
props["views"].each do |view, funcs|
|
||||
next unless view.include?("reduce")
|
||||
props["views"].delete(view) unless funcs.keys.include?("reduce")
|
||||
end
|
||||
end
|
||||
|
||||
designs.each do |k,v|
|
||||
create_or_update("_design/#{k}", v)
|
||||
end
|
||||
|
||||
designs
|
||||
end
|
||||
|
||||
def pull_views(view_dir)
|
||||
prefix = "_design"
|
||||
ds = db.documents(:startkey => '#{prefix}/', :endkey => '#{prefix}/ZZZZZZZZZ')
|
||||
ds['rows'].collect{|r|r['id']}.each do |id|
|
||||
puts directory = id.split('/').last
|
||||
FileUtils.mkdir_p(File.join(view_dir,directory))
|
||||
views = db.get(id)['views']
|
||||
|
||||
vgroups = views.keys.group_by{|k|k.sub(/\-(map|reduce)$/,'')}
|
||||
vgroups.each do|g,vs|
|
||||
mapname = vs.find {|v|views[v]["map"]}
|
||||
if mapname
|
||||
# save map
|
||||
mapfunc = views[mapname]["map"]
|
||||
mapfile = File.join(view_dir, directory, "#{g}-map.js") # todo support non-js views
|
||||
File.open(mapfile,'w') do |f|
|
||||
f.write mapfunc
|
||||
end
|
||||
end
|
||||
|
||||
reducename = vs.find {|v|views[v]["reduce"]}
|
||||
if reducename
|
||||
# save reduce
|
||||
reducefunc = views[reducename]["reduce"]
|
||||
reducefile = File.join(view_dir, directory, "#{g}-reduce.js") # todo support non-js views
|
||||
File.open(reducefile,'w') do |f|
|
||||
f.write reducefunc
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
|
||||
private
|
||||
|
||||
def say words
|
||||
puts words if @loud
|
||||
end
|
||||
|
||||
def md5 string
|
||||
Digest::MD5.hexdigest(string)
|
||||
end
|
||||
|
||||
def read(file, libs=nil)
|
||||
st = open(file).read
|
||||
st.sub!(/(\/\/|#)include-lib/,libs) if libs
|
||||
st
|
||||
end
|
||||
|
||||
def create_or_update(id, fields)
|
||||
existing = @db.get(id) rescue nil
|
||||
|
||||
if existing
|
||||
updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
|
||||
if existing != updated
|
||||
say "replacing #{id}"
|
||||
db.save(updated)
|
||||
else
|
||||
say "skipping #{id}"
|
||||
end
|
||||
else
|
||||
say "creating #{id}"
|
||||
db.save(fields.merge({"_id" => id}))
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
103
lib/couchrest/helper/pager.rb
Normal file
103
lib/couchrest/helper/pager.rb
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
module CouchRest
|
||||
class Pager
|
||||
attr_accessor :db
|
||||
def initialize db
|
||||
@db = db
|
||||
end
|
||||
|
||||
def all_docs(count=100, &block)
|
||||
startkey = nil
|
||||
oldend = nil
|
||||
|
||||
while docrows = request_all_docs(count+1, startkey)
|
||||
startkey = docrows.last['key']
|
||||
docrows.pop if docrows.length > count
|
||||
if oldend == startkey
|
||||
break
|
||||
end
|
||||
yield(docrows)
|
||||
oldend = startkey
|
||||
end
|
||||
end
|
||||
|
||||
def key_reduce(view, count, firstkey = nil, lastkey = nil, &block)
|
||||
# start with no keys
|
||||
startkey = firstkey
|
||||
# lastprocessedkey = nil
|
||||
keepgoing = true
|
||||
|
||||
while keepgoing && viewrows = request_view(view, count, startkey)
|
||||
startkey = viewrows.first['key']
|
||||
endkey = viewrows.last['key']
|
||||
|
||||
if (startkey == endkey)
|
||||
# we need to rerequest to get a bigger page
|
||||
# so we know we have all the rows for that key
|
||||
viewrows = @db.view(view, :key => startkey)['rows']
|
||||
# we need to do an offset thing to find the next startkey
|
||||
# otherwise we just get stuck
|
||||
lastdocid = viewrows.last['id']
|
||||
fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :count => 2)['rows']
|
||||
|
||||
newendkey = fornextloop.last['key']
|
||||
if (newendkey == endkey)
|
||||
keepgoing = false
|
||||
else
|
||||
startkey = newendkey
|
||||
end
|
||||
rows = viewrows
|
||||
else
|
||||
rows = []
|
||||
for r in viewrows
|
||||
if (lastkey && r['key'] == lastkey)
|
||||
keepgoing = false
|
||||
break
|
||||
end
|
||||
break if (r['key'] == endkey)
|
||||
rows << r
|
||||
end
|
||||
startkey = endkey
|
||||
end
|
||||
|
||||
key = :begin
|
||||
values = []
|
||||
|
||||
rows.each do |r|
|
||||
if key != r['key']
|
||||
# we're on a new key, yield the old first and then reset
|
||||
yield(key, values) if key != :begin
|
||||
key = r['key']
|
||||
values = []
|
||||
end
|
||||
# keep accumulating
|
||||
values << r['value']
|
||||
end
|
||||
yield(key, values)
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def request_all_docs count, startkey = nil
|
||||
opts = {}
|
||||
opts[:count] = count if count
|
||||
opts[:startkey] = startkey if startkey
|
||||
results = @db.documents(opts)
|
||||
rows = results['rows']
|
||||
rows unless rows.length == 0
|
||||
end
|
||||
|
||||
def request_view view, count = nil, startkey = nil, endkey = nil
|
||||
opts = {}
|
||||
opts[:count] = count if count
|
||||
opts[:startkey] = startkey if startkey
|
||||
opts[:endkey] = endkey if endkey
|
||||
|
||||
results = @db.view(view, opts)
|
||||
rows = results['rows']
|
||||
rows unless rows.length == 0
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
29
lib/couchrest/helper/streamer.rb
Normal file
29
lib/couchrest/helper/streamer.rb
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
module CouchRest
|
||||
class Streamer
|
||||
attr_accessor :db
|
||||
def initialize db
|
||||
@db = db
|
||||
end
|
||||
|
||||
def view name, params = nil
|
||||
urlst = /^_/.match(name) ? "#{@db.root}/#{name}" : "#{@db.root}/_view/#{name}"
|
||||
url = CouchRest.paramify_url urlst, params
|
||||
IO.popen("curl --silent #{url}") do |view|
|
||||
view.gets # discard header
|
||||
while row = parse_line(view.gets)
|
||||
yield row
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def parse_line line
|
||||
return nil unless line
|
||||
if /(\{.*\}),?/.match(line.chomp)
|
||||
JSON.parse($1)
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue