brought file manager back in as a class

This commit is contained in:
Chris Anderson 2008-08-02 00:03:54 -07:00
parent 93f95741c4
commit 541a3cac74
11 changed files with 539 additions and 6 deletions

View file

@ -1,4 +1,5 @@
require File.dirname(__FILE__) + '/lib/couchrest'
require File.dirname(__FILE__) + '/lib/database'
require File.dirname(__FILE__) + '/lib/pager'
require File.dirname(__FILE__) + '/lib/file_manager'

397
lib/file_manager.rb Normal file
View file

@ -0,0 +1,397 @@
require 'rubygems'
require 'couchrest'
require 'digest/md5'
# todo = ARGV
# todo = ["views", "public", "controllers"] if ARGV.include? "all"
#
#
# PROJECT_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(PROJECT_ROOT)
# DBNAME = JSON.load( open("#{PROJECT_ROOT}/config.json").read )["db"]
class CouchRest
class FileManager
attr_reader :db
LANGS = {"rb" => "ruby", "js" => "javascript"}
MIMES = {
"html" => "text/html",
"htm" => "text/html",
"png" => "image/png",
"css" => "text/css",
"js" => "test/javascript"
}
def initialize(dbname, host="http://localhost:5984")
@db = CouchRest.new(host).database(dbname)
end
def push_directory(push_dir, docid=nil)
docid ||= push_dir.split('/').reverse.find{|part|!part.empty?}
pushfiles = Dir["#{push_dir}/**/*.*"].collect do |f|
{f.split("#{push_dir}/").last => open(f).read}
end
return if pushfiles.empty?
@attachments = {}
@signatures = {}
pushfiles.each do |file|
name = file.keys.first
value = file.values.first
@signatures[name] = md5(value)
@attachments[name] = {
"data" => value,
"content_type" => MIMES[name.split('.').last]
}
end
doc = @db.get(docid) rescue nil
unless doc
# puts "creating public"
@db.save({"_id" => docid, "_attachments" => @attachments, "signatures" => @signatures})
return
end
# remove deleted docs
to_be_removed = doc["signatures"].keys.select{|d| !couch["public"].collect{|p| p.keys.first}.include?(d) }
to_be_removed.each do |p|
# puts "deleting #{p}"
doc["signatures"].delete(p)
doc["_attachments"].delete(p)
end
# update existing docs:
doc["signatures"].each do |path, sig|
if (@signatures[path] == sig)
# puts "no change to #{path}. skipping..."
else
# puts "replacing #{path}"
doc["signatures"][path] = md5(@attachments[path]["data"])
doc["_attachments"][path].delete("stub")
doc["_attachments"][path].delete("length")
doc["_attachments"][path]["data"] = @attachments[path]["data"]
doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
end
end
# add in new files
new_files = couch["public"].select{|d| !doc["signatures"].keys.include?( d.keys.first) }
new_files.each do |f|
# puts "creating #{f}"
path = f.keys.first
content = f.values.first
doc["signatures"][path] = md5(content)
doc["_attachments"][path] = {
"data" => content,
"content_type" => @content_types[path.split('.').last]
}
end
begin
@db.save(doc)
rescue Exception => e
# puts e.message
end
end
def push_views(view_dir)
designs = {}
Dir["#{view_dir}/**/*.*"].collect do |design_doc|
design_doc_parts = design_doc.split('/')
pre_normalized_view_name = design_doc_parts.last.split("-")
view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
folder = design_doc_parts[-2]
designs[folder] ||= {}
designs[folder]["views"] ||= {}
design_lang = design_doc_parts.last.split(".").last
designs[folder]["language"] ||= LANGS[design_lang]
libs = ""
Dir["#{view_dir}/lib.#{design_lang}"].collect do |global_lib|
libs << open(global_lib).read
libs << "\n"
end
Dir["#{view_dir}/#{folder}/lib.#{design_lang}"].collect do |global_lib|
libs << open(global_lib).read
libs << "\n"
end
if design_doc_parts.last =~ /-map/
designs[folder]["views"]["#{view_name}-map"] ||= {}
designs[folder]["views"]["#{view_name}-map"]["map"] = read(design_doc, libs)
designs[folder]["views"]["#{view_name}-reduce"] ||= {}
designs[folder]["views"]["#{view_name}-reduce"]["map"] = read(design_doc, libs)
end
if design_doc_parts.last =~ /-reduce/
designs[folder]["views"]["#{view_name}-reduce"] ||= {}
designs[folder]["views"]["#{view_name}-reduce"]["reduce"] = read(design_doc, libs)
end
end
# cleanup empty maps and reduces
designs.each do |name, props|
props["views"].each do |view, funcs|
next unless view.include?("reduce")
props["views"].delete(view) unless funcs.keys.include?("reduce")
end
end
designs.each do |k,v|
create_or_update("_design/#{k}", v)
end
designs
end
private
def md5 string
Digest::MD5.hexdigest(string)
end
def read(file, libs=nil)
st = open(file).read
st.sub!(/\/\/include-lib/,libs) if libs
st
end
def create_or_update(id, fields)
existing = @db.get(id) rescue nil
if existing
updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
else
# puts "saving #{id}"
db.save(fields.merge({"_id" => id}))
end
if existing != updated
# puts "replacing #{id}"
db.save(updated)
end
end
end
end
__END__
# parse the file structure to load the public files, controllers, and views into a hash with the right shape for coucdb
couch = {}
couch["public"] = Dir["#{File.expand_path(File.dirname("."))}/public/**/*.*"].collect do |f|
{f.split("public/").last => open(f).read}
end
couch["controllers"] = {}
Dir["#{File.expand_path(File.dirname("."))}/app/controllers/**/*.*"].collect do |c|
path_parts = c.split("/")
controller_name = path_parts[path_parts.length - 2]
action_name = path_parts[path_parts.length - 1].split(".").first
couch["controllers"][controller_name] ||= {"actions" => {}}
couch["controllers"][controller_name]["actions"][action_name] = open(c).read
end
couch["designs"] = {}
Dir["#{File.expand_path(File.dirname("."))}/app/views/**/*.*"].collect do |design_doc|
design_doc_parts = design_doc.split('/')
pre_normalized_view_name = design_doc_parts.last.split("-")
view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
folder = design_doc.split("app/views").last.split("/")[1]
couch["designs"][folder] ||= {}
couch["designs"][folder]["views"] ||= {}
couch["designs"][folder]["language"] ||= LANGS[design_doc_parts.last.split(".").last]
if design_doc_parts.last =~ /-map/
couch["designs"][folder]["views"]["#{view_name}-map"] ||= {}
couch["designs"][folder]["views"]["#{view_name}-map"]["map"] = open(design_doc).read
couch["designs"][folder]["views"]["#{view_name}-reduce"] ||= {}
couch["designs"][folder]["views"]["#{view_name}-reduce"]["map"] = open(design_doc).read
end
if design_doc_parts.last =~ /-reduce/
couch["designs"][folder]["views"]["#{view_name}-reduce"] ||= {}
couch["designs"][folder]["views"]["#{view_name}-reduce"]["reduce"] = open(design_doc).read
end
end
# cleanup empty maps and reduces
couch["designs"].each do |name, props|
props["views"].delete("#{name}-reduce") unless props["views"]["#{name}-reduce"].keys.include?("reduce")
end
# parsing done, begin posting
# connect to couchdb
cr = CouchRest.new("http://localhost:5984")
@db = cr.database(DBNAME)
def create_or_update(id, fields)
existing = get(id)
if existing
updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
else
puts "saving #{id}"
save(fields.merge({"_id" => id}))
end
if existing == updated
puts "no change to #{id}. skipping..."
else
puts "replacing #{id}"
save(updated)
end
end
def get(id)
doc = handle_errors do
@db.get(id)
end
end
def save(doc)
handle_errors do
@db.save(doc)
end
end
def handle_errors(&block)
begin
yield
rescue Exception => e
# puts e.message
nil
end
end
if todo.include? "views"
puts "posting views into CouchDB"
couch["designs"].each do |k,v|
create_or_update("_design/#{k}", v)
end
puts
end
if todo.include? "controllers"
puts "posting controllers into CouchDB"
couch["controllers"].each do |k,v|
create_or_update("controller/#{k}", v)
end
puts
end
if todo.include? "public"
puts "posting public docs into CouchDB"
if couch["public"].empty?
puts "no docs in public"; exit
end
@content_types = {
"html" => "text/html",
"htm" => "text/html",
"png" => "image/png",
"css" => "text/css",
"js" => "test/javascript"
}
def md5 string
Digest::MD5.hexdigest(string)
end
@attachments = {}
@signatures = {}
couch["public"].each do |doc|
@signatures[doc.keys.first] = md5(doc.values.first)
@attachments[doc.keys.first] = {
"data" => doc.values.first,
"content_type" => @content_types[doc.keys.first.split('.').last]
}
end
doc = get("public")
unless doc
puts "creating public"
@db.save({"_id" => "public", "_attachments" => @attachments, "signatures" => @signatures})
exit
end
# remove deleted docs
to_be_removed = doc["signatures"].keys.select{|d| !couch["public"].collect{|p| p.keys.first}.include?(d) }
to_be_removed.each do |p|
puts "deleting #{p}"
doc["signatures"].delete(p)
doc["_attachments"].delete(p)
end
# update existing docs:
doc["signatures"].each do |path, sig|
if (@signatures[path] == sig)
puts "no change to #{path}. skipping..."
else
puts "replacing #{path}"
doc["signatures"][path] = md5(@attachments[path]["data"])
doc["_attachments"][path].delete("stub")
doc["_attachments"][path].delete("length")
doc["_attachments"][path]["data"] = @attachments[path]["data"]
doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
end
end
# add in new files
new_files = couch["public"].select{|d| !doc["signatures"].keys.include?( d.keys.first) }
new_files.each do |f|
puts "creating #{f}"
path = f.keys.first
content = f.values.first
doc["signatures"][path] = md5(content)
doc["_attachments"][path] = {
"data" => content,
"content_type" => @content_types[path.split('.').last]
}
end
begin
@db.save(doc)
rescue Exception => e
puts e.message
end
puts
end

View file

@ -112,11 +112,7 @@ when "push" # files to views
dviews.delete("#{view}-reduce") unless dviews["#{view}-reduce"]["reduce"]
end
# save them to the db
begin
view = db.get("_design/#{design}")
rescue RestClient::Request::RequestFailed
view = nil
end
view = db.get("_design/#{design}") rescue nil
if (view && view['views'] == dviews)
puts "no change to _design/#{design}. skipping..."
else

112
spec/file_manager_spec.rb Normal file
View file

@ -0,0 +1,112 @@
require File.dirname(__FILE__) + '/spec_helper'
describe CouchRest::FileManager do
before(:all) do
@cr = CouchRest.new(COUCHHOST)
@db = @cr.database(TESTDB)
@db.delete! rescue nil
@db = @cr.create_db(TESTDB) rescue nil
end
it "should initialize" do
@fm = CouchRest::FileManager.new(TESTDB)
@fm.should_not be_nil
end
it "should require a db name" do
lambda{CouchRest::FileManager.new}.should raise_error
end
it "should accept a db name" do
@fm = CouchRest::FileManager.new(TESTDB, 'http://localhost')
@fm.db.name.should == TESTDB
end
it "should default to localhost couchdb" do
@fm = CouchRest::FileManager.new(TESTDB)
@fm.db.host.should == 'http://localhost:5984'
end
end
describe CouchRest::FileManager, "pushing views" do
before(:all) do
@cr = CouchRest.new(COUCHHOST)
@db = @cr.database(TESTDB)
@db.delete! rescue nil
@db = @cr.create_db(TESTDB) rescue nil
@fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
@view_dir = File.dirname(__FILE__) + '/fixtures/views'
ds = @fm.push_views(@view_dir)
@design = @db.get("_design/test_view")
end
it "should create a design document for each folder" do
@design["views"].should_not be_nil
end
it "should push a map and reduce view" do
@design["views"]["test-map"].should_not be_nil
@design["views"]["test-reduce"].should_not be_nil
end
it "should push a map only view" do
@design["views"]["only-map"].should_not be_nil
@design["views"]["only-reduce"].should be_nil
end
it "should include library files" do
@design["views"]["only-map"]["map"].should include("globalLib")
@design["views"]["only-map"]["map"].should include("justThisView")
end
end
describe CouchRest::FileManager, "pushing a directory with id" do
before(:all) do
@cr = CouchRest.new(COUCHHOST)
@db = @cr.database(TESTDB)
@db.delete! rescue nil
@db = @cr.create_db(TESTDB) rescue nil
@fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
@push_dir = File.dirname(__FILE__) + '/fixtures/attachments'
ds = @fm.push_directory(@push_dir, 'attached')
end
it "should create a document for the folder" do
@db.get("attached")
end
it "should make attachments" do
doc = @db.get("attached")
doc["_attachments"]["test.html"].should_not be_nil
end
it "should set the content type" do
doc = @db.get("attached")
doc["_attachments"]["test.html"]["content_type"].should == "text/html"
end
end
describe CouchRest::FileManager, "pushing a directory without id" do
before(:all) do
@cr = CouchRest.new(COUCHHOST)
@db = @cr.database(TESTDB)
@db.delete! rescue nil
@db = @cr.create_db(TESTDB) rescue nil
@fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
@push_dir = File.dirname(__FILE__) + '/fixtures/attachments'
ds = @fm.push_directory(@push_dir)
end
it "should use the dirname" do
doc = @db.get("attachments")
doc["_attachments"]["test.html"].should_not be_nil
end
end
describe CouchRest::FileManager, "pushing a directory/ without id" do
before(:all) do
@cr = CouchRest.new(COUCHHOST)
@db = @cr.database(TESTDB)
@db.delete! rescue nil
@db = @cr.create_db(TESTDB) rescue nil
@fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
@push_dir = File.dirname(__FILE__) + '/fixtures/attachments/'
ds = @fm.push_directory(@push_dir)
end
it "should use the dirname" do
doc = @db.get("attachments")
doc["_attachments"]["test.html"].should_not be_nil
end
end

11
spec/fixtures/attachments/test.html vendored Normal file
View file

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html>
<head>
<title>Test</title>
</head>
<body>
<p>
Test
</p>
</body>
</html>

3
spec/fixtures/views/lib.js vendored Normal file
View file

@ -0,0 +1,3 @@
function globalLib() {
return "fixture";
};

3
spec/fixtures/views/test_view/lib.js vendored Normal file
View file

@ -0,0 +1,3 @@
function justThisView() {
return "fixture";
};

View file

@ -0,0 +1,4 @@
function(doc) {
//include-lib
emit(null, null);
};

View file

@ -0,0 +1,3 @@
function(doc) {
emit(null, null);
};

View file

@ -0,0 +1,3 @@
function(ks,vs,co) {
return vs.length;
};

View file

@ -1,4 +1,4 @@
require File.dirname(__FILE__) + '/../lib/couchrest'
require File.dirname(__FILE__) + '/../couchrest'
COUCHHOST = "http://localhost:5984"
TESTDB = 'couchrest-test'