From 541a3cac74a4a1d60b6817bb26385433b1d79eb1 Mon Sep 17 00:00:00 2001
From: Chris Anderson <jchris@grabb.it>
Date: Sat, 2 Aug 2008 00:03:54 -0700
Subject: [PATCH] brought file manager back in as a class

---
 couchrest.rb                                 |   1 +
 lib/file_manager.rb                          | 397 +++++++++++++++++++
 script/couchview                             |   6 +-
 spec/file_manager_spec.rb                    | 112 ++++++
 spec/fixtures/attachments/test.html          |  11 +
 spec/fixtures/views/lib.js                   |   3 +
 spec/fixtures/views/test_view/lib.js         |   3 +
 spec/fixtures/views/test_view/only-map.js    |   4 +
 spec/fixtures/views/test_view/test-map.js    |   3 +
 spec/fixtures/views/test_view/test-reduce.js |   3 +
 spec/spec_helper.rb                          |   2 +-
 11 files changed, 539 insertions(+), 6 deletions(-)
 create mode 100644 lib/file_manager.rb
 create mode 100644 spec/file_manager_spec.rb
 create mode 100644 spec/fixtures/attachments/test.html
 create mode 100644 spec/fixtures/views/lib.js
 create mode 100644 spec/fixtures/views/test_view/lib.js
 create mode 100644 spec/fixtures/views/test_view/only-map.js
 create mode 100644 spec/fixtures/views/test_view/test-map.js
 create mode 100644 spec/fixtures/views/test_view/test-reduce.js

diff --git a/couchrest.rb b/couchrest.rb
index 95c0aaf..108d71d 100644
--- a/couchrest.rb
+++ b/couchrest.rb
@@ -1,4 +1,5 @@
 require File.dirname(__FILE__) + '/lib/couchrest'
 require File.dirname(__FILE__) + '/lib/database'
 require File.dirname(__FILE__) + '/lib/pager'
+require File.dirname(__FILE__) + '/lib/file_manager'
 
diff --git a/lib/file_manager.rb b/lib/file_manager.rb
new file mode 100644
index 0000000..e4f037c
--- /dev/null
+++ b/lib/file_manager.rb
@@ -0,0 +1,397 @@
+require 'rubygems'
+require 'couchrest'
+require 'digest/md5'
+
+# todo = ARGV
+# todo = ["views", "public", "controllers"] if ARGV.include? "all"
+# 
+# 
+# PROJECT_ROOT = "#{File.dirname(__FILE__)}/.." unless defined?(PROJECT_ROOT)
+# DBNAME = JSON.load( open("#{PROJECT_ROOT}/config.json").read )["db"]
+
+
+class CouchRest
+  class FileManager
+    attr_reader :db
+    
+    LANGS = {"rb" => "ruby", "js" => "javascript"}
+    MIMES = {
+      "html"       => "text/html",
+      "htm"        => "text/html",
+      "png"        => "image/png",
+      "css"        => "text/css",
+      "js"         => "test/javascript"
+    }    
+    def initialize(dbname, host="http://localhost:5984")
+      @db = CouchRest.new(host).database(dbname)
+    end
+    
+    def push_directory(push_dir, docid=nil)
+      docid ||= push_dir.split('/').reverse.find{|part|!part.empty?}
+
+      pushfiles = Dir["#{push_dir}/**/*.*"].collect do |f|
+        {f.split("#{push_dir}/").last => open(f).read}
+      end
+
+      return if pushfiles.empty?
+
+      @attachments = {}
+      @signatures = {}
+      pushfiles.each do |file|
+        name = file.keys.first
+        value = file.values.first
+        @signatures[name] = md5(value)
+
+        @attachments[name] = {
+          "data" => value,
+          "content_type" => MIMES[name.split('.').last]
+        }
+      end
+
+      doc = @db.get(docid) rescue nil
+
+      unless doc
+        # puts "creating public"
+        @db.save({"_id" => docid, "_attachments" => @attachments, "signatures" => @signatures})
+        return
+      end
+
+      # remove deleted docs
+      to_be_removed = doc["signatures"].keys.select{|d| !couch["public"].collect{|p| p.keys.first}.include?(d) }
+
+      to_be_removed.each do |p|
+        # puts "deleting #{p}"
+        doc["signatures"].delete(p)
+        doc["_attachments"].delete(p)
+      end
+
+      # update existing docs:
+      doc["signatures"].each do |path, sig|
+        if (@signatures[path] == sig)
+          # puts "no change to #{path}. skipping..."
+        else
+          # puts "replacing #{path}"
+          doc["signatures"][path] = md5(@attachments[path]["data"])
+          doc["_attachments"][path].delete("stub")
+          doc["_attachments"][path].delete("length")    
+          doc["_attachments"][path]["data"] = @attachments[path]["data"]
+          doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
+
+        end
+      end
+
+      # add in new files
+      new_files = couch["public"].select{|d| !doc["signatures"].keys.include?( d.keys.first) } 
+
+      new_files.each do |f|
+        # puts "creating #{f}"
+        path = f.keys.first
+        content = f.values.first
+        doc["signatures"][path] = md5(content)
+
+        doc["_attachments"][path] = {
+          "data" => content,
+          "content_type" => @content_types[path.split('.').last]
+        }
+      end
+
+      begin
+        @db.save(doc)
+      rescue Exception => e
+        # puts e.message
+      end
+    end
+    
+    def push_views(view_dir)
+      designs = {}
+
+      Dir["#{view_dir}/**/*.*"].collect do |design_doc|
+        design_doc_parts = design_doc.split('/')
+        pre_normalized_view_name = design_doc_parts.last.split("-")
+        view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
+
+        folder = design_doc_parts[-2]
+
+        designs[folder] ||= {}
+        designs[folder]["views"] ||= {}
+        design_lang = design_doc_parts.last.split(".").last
+        designs[folder]["language"] ||= LANGS[design_lang]
+
+        libs = ""
+        Dir["#{view_dir}/lib.#{design_lang}"].collect do |global_lib|
+          libs << open(global_lib).read
+          libs << "\n"
+        end
+        Dir["#{view_dir}/#{folder}/lib.#{design_lang}"].collect do |global_lib|
+          libs << open(global_lib).read
+          libs << "\n"
+        end
+        if design_doc_parts.last =~ /-map/
+          designs[folder]["views"]["#{view_name}-map"] ||= {}
+
+          designs[folder]["views"]["#{view_name}-map"]["map"] = read(design_doc, libs)
+
+          designs[folder]["views"]["#{view_name}-reduce"] ||= {}
+          designs[folder]["views"]["#{view_name}-reduce"]["map"] = read(design_doc, libs)
+        end
+
+        if design_doc_parts.last =~ /-reduce/
+          designs[folder]["views"]["#{view_name}-reduce"] ||= {}
+
+          designs[folder]["views"]["#{view_name}-reduce"]["reduce"] = read(design_doc, libs)
+        end
+      end
+
+      # cleanup empty maps and reduces
+      designs.each do |name, props|
+        props["views"].each do |view, funcs|
+          next unless view.include?("reduce")
+          props["views"].delete(view) unless funcs.keys.include?("reduce")
+        end
+      end
+      
+      designs.each do |k,v|
+        create_or_update("_design/#{k}", v)
+      end
+      
+      designs
+    end
+    
+    
+    private
+    
+    def md5 string
+      Digest::MD5.hexdigest(string)
+    end
+    
+    def read(file, libs=nil)
+      st = open(file).read
+      st.sub!(/\/\/include-lib/,libs) if libs
+      st
+    end
+    
+    def create_or_update(id, fields)
+      existing = @db.get(id) rescue nil
+
+      if existing
+        updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
+      else
+        # puts "saving #{id}"
+        db.save(fields.merge({"_id" => id}))
+      end
+
+      if existing != updated
+        # puts "replacing #{id}"
+        db.save(updated)
+      end
+
+    end
+  end
+end
+
+__END__
+
+
+
+
+# parse the file structure to load the public files, controllers, and views into a hash with the right shape for coucdb
+couch = {}
+
+couch["public"] = Dir["#{File.expand_path(File.dirname("."))}/public/**/*.*"].collect do |f|
+  {f.split("public/").last => open(f).read}
+end
+
+couch["controllers"] = {}
+Dir["#{File.expand_path(File.dirname("."))}/app/controllers/**/*.*"].collect do |c|
+  path_parts = c.split("/")
+  
+  controller_name = path_parts[path_parts.length - 2]
+  action_name = path_parts[path_parts.length - 1].split(".").first
+
+  couch["controllers"][controller_name] ||= {"actions" => {}}
+  couch["controllers"][controller_name]["actions"][action_name] = open(c).read
+  
+end
+
+couch["designs"] = {}
+Dir["#{File.expand_path(File.dirname("."))}/app/views/**/*.*"].collect do |design_doc|
+  design_doc_parts = design_doc.split('/')
+  pre_normalized_view_name = design_doc_parts.last.split("-")
+  view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
+
+  folder = design_doc.split("app/views").last.split("/")[1]
+
+  couch["designs"][folder] ||= {}
+  couch["designs"][folder]["views"] ||= {}
+  couch["designs"][folder]["language"] ||= LANGS[design_doc_parts.last.split(".").last]
+  
+  if design_doc_parts.last =~ /-map/
+    couch["designs"][folder]["views"]["#{view_name}-map"] ||= {}
+
+    couch["designs"][folder]["views"]["#{view_name}-map"]["map"] = open(design_doc).read
+
+    couch["designs"][folder]["views"]["#{view_name}-reduce"] ||= {}
+    couch["designs"][folder]["views"]["#{view_name}-reduce"]["map"] = open(design_doc).read
+  end
+  
+  if design_doc_parts.last =~ /-reduce/
+    couch["designs"][folder]["views"]["#{view_name}-reduce"] ||= {}
+
+    couch["designs"][folder]["views"]["#{view_name}-reduce"]["reduce"] = open(design_doc).read
+  end
+end
+
+# cleanup empty maps and reduces
+couch["designs"].each do |name, props|
+  props["views"].delete("#{name}-reduce") unless props["views"]["#{name}-reduce"].keys.include?("reduce")
+end
+
+# parsing done, begin posting
+
+# connect to couchdb
+cr = CouchRest.new("http://localhost:5984")
+@db = cr.database(DBNAME)
+
+def create_or_update(id, fields)
+  existing = get(id)
+  
+  if existing
+    updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
+  else
+    puts "saving #{id}"
+    save(fields.merge({"_id" => id}))
+  end
+  
+  if existing == updated
+    puts "no change to #{id}. skipping..."
+  else
+    puts "replacing #{id}"
+    save(updated)
+  end
+
+end
+
+def get(id)
+  doc = handle_errors do
+    @db.get(id)
+  end
+end
+
+def save(doc)
+  handle_errors do
+    @db.save(doc)
+  end
+end
+
+def handle_errors(&block)
+  begin
+    yield
+  rescue Exception => e
+    # puts e.message
+    nil
+  end
+end
+
+
+if todo.include? "views"
+  puts "posting views into CouchDB"
+  couch["designs"].each do |k,v|
+    create_or_update("_design/#{k}", v)
+  end
+  puts
+end
+
+if todo.include? "controllers"
+  puts "posting controllers into CouchDB"
+  couch["controllers"].each do |k,v|
+    create_or_update("controller/#{k}", v)
+  end
+  puts
+end
+
+
+if todo.include? "public"
+  puts "posting public docs into CouchDB"
+
+  if couch["public"].empty?
+    puts "no docs in public"; exit 
+  end
+  
+  @content_types = {
+    "html"       => "text/html",
+    "htm"        => "text/html",
+    "png"        => "image/png",
+    "css"        => "text/css",
+    "js"         => "test/javascript"
+  }
+  
+  def md5 string
+    Digest::MD5.hexdigest(string)
+  end
+    
+  @attachments = {}
+  @signatures = {}
+  couch["public"].each do |doc|
+    @signatures[doc.keys.first] = md5(doc.values.first)
+    
+    @attachments[doc.keys.first] = {
+      "data" => doc.values.first,
+      "content_type" => @content_types[doc.keys.first.split('.').last]
+    }
+  end
+  
+  doc = get("public")
+  
+  unless doc
+    puts "creating public"
+    @db.save({"_id" => "public", "_attachments" => @attachments, "signatures" => @signatures})
+    exit
+  end
+  
+  # remove deleted docs
+  to_be_removed = doc["signatures"].keys.select{|d| !couch["public"].collect{|p| p.keys.first}.include?(d) }
+  
+  to_be_removed.each do |p|
+    puts "deleting #{p}"
+    doc["signatures"].delete(p)
+    doc["_attachments"].delete(p)
+  end
+  
+  # update existing docs:
+  doc["signatures"].each do |path, sig|
+    if (@signatures[path] == sig)
+      puts "no change to #{path}. skipping..."
+    else
+      puts "replacing #{path}"
+      doc["signatures"][path] = md5(@attachments[path]["data"])
+      doc["_attachments"][path].delete("stub")
+      doc["_attachments"][path].delete("length")    
+      doc["_attachments"][path]["data"] = @attachments[path]["data"]
+      doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
+      
+    end
+  end
+  
+  # add in new files
+  new_files = couch["public"].select{|d| !doc["signatures"].keys.include?( d.keys.first) } 
+  
+  new_files.each do |f|
+    puts "creating #{f}"
+    path = f.keys.first
+    content = f.values.first
+    doc["signatures"][path] = md5(content)
+    
+    doc["_attachments"][path] = {
+      "data" => content,
+      "content_type" => @content_types[path.split('.').last]
+    }
+  end
+  
+  begin
+    @db.save(doc)
+  rescue Exception => e
+    puts e.message
+  end
+  
+  puts
+end
\ No newline at end of file
diff --git a/script/couchview b/script/couchview
index a43c2fb..d2aea12 100755
--- a/script/couchview
+++ b/script/couchview
@@ -112,11 +112,7 @@ when "push" # files to views
       dviews.delete("#{view}-reduce") unless dviews["#{view}-reduce"]["reduce"]
     end
     # save them to the db
-    begin
-      view = db.get("_design/#{design}") 
-    rescue RestClient::Request::RequestFailed
-      view = nil
-    end
+    view = db.get("_design/#{design}") rescue nil
     if (view && view['views'] == dviews)
       puts "no change to _design/#{design}. skipping..."
     else
diff --git a/spec/file_manager_spec.rb b/spec/file_manager_spec.rb
new file mode 100644
index 0000000..ad4b01e
--- /dev/null
+++ b/spec/file_manager_spec.rb
@@ -0,0 +1,112 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe CouchRest::FileManager do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    @db = @cr.database(TESTDB)
+    @db.delete! rescue nil
+    @db = @cr.create_db(TESTDB) rescue nil
+  end
+  it "should initialize" do
+    @fm = CouchRest::FileManager.new(TESTDB)
+    @fm.should_not be_nil
+  end
+  it "should require a db name" do
+    lambda{CouchRest::FileManager.new}.should raise_error
+  end
+  it "should accept a db name" do
+    @fm = CouchRest::FileManager.new(TESTDB, 'http://localhost')
+    @fm.db.name.should == TESTDB
+  end
+  it "should default to localhost couchdb" do
+    @fm = CouchRest::FileManager.new(TESTDB)
+    @fm.db.host.should == 'http://localhost:5984'
+  end
+end
+
+describe CouchRest::FileManager, "pushing views" do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    @db = @cr.database(TESTDB)
+    @db.delete! rescue nil
+    @db = @cr.create_db(TESTDB) rescue nil
+    
+    @fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
+    @view_dir = File.dirname(__FILE__) + '/fixtures/views'
+    ds = @fm.push_views(@view_dir)
+    @design = @db.get("_design/test_view")
+  end
+  it "should create a design document for each folder" do
+    @design["views"].should_not be_nil
+  end
+  it "should push a map and reduce view" do
+    @design["views"]["test-map"].should_not be_nil
+    @design["views"]["test-reduce"].should_not be_nil
+  end
+  it "should push a map only view" do
+    @design["views"]["only-map"].should_not be_nil
+    @design["views"]["only-reduce"].should be_nil
+  end
+  it "should include library files" do
+    @design["views"]["only-map"]["map"].should include("globalLib")
+    @design["views"]["only-map"]["map"].should include("justThisView")
+  end
+end
+
+describe CouchRest::FileManager, "pushing a directory with id" do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    @db = @cr.database(TESTDB)
+    @db.delete! rescue nil
+    @db = @cr.create_db(TESTDB) rescue nil
+    
+    @fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
+    @push_dir = File.dirname(__FILE__) + '/fixtures/attachments'
+    ds = @fm.push_directory(@push_dir, 'attached')
+  end
+  it "should create a document for the folder" do
+    @db.get("attached")
+  end
+  it "should make attachments" do
+    doc = @db.get("attached")
+    doc["_attachments"]["test.html"].should_not be_nil
+  end
+  it "should set the content type" do
+    doc = @db.get("attached")
+    doc["_attachments"]["test.html"]["content_type"].should == "text/html"
+  end
+end
+
+describe CouchRest::FileManager, "pushing a directory without id" do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    @db = @cr.database(TESTDB)
+    @db.delete! rescue nil
+    @db = @cr.create_db(TESTDB) rescue nil
+    
+    @fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
+    @push_dir = File.dirname(__FILE__) + '/fixtures/attachments'
+    ds = @fm.push_directory(@push_dir)
+  end
+  it "should use the dirname" do
+    doc = @db.get("attachments")
+    doc["_attachments"]["test.html"].should_not be_nil
+  end
+end
+
+describe CouchRest::FileManager, "pushing a directory/ without id" do
+  before(:all) do
+    @cr = CouchRest.new(COUCHHOST)
+    @db = @cr.database(TESTDB)
+    @db.delete! rescue nil
+    @db = @cr.create_db(TESTDB) rescue nil
+    
+    @fm = CouchRest::FileManager.new(TESTDB, COUCHHOST)
+    @push_dir = File.dirname(__FILE__) + '/fixtures/attachments/'
+    ds = @fm.push_directory(@push_dir)
+  end
+  it "should use the dirname" do
+    doc = @db.get("attachments")
+    doc["_attachments"]["test.html"].should_not be_nil
+  end
+end
\ No newline at end of file
diff --git a/spec/fixtures/attachments/test.html b/spec/fixtures/attachments/test.html
new file mode 100644
index 0000000..d67d832
--- /dev/null
+++ b/spec/fixtures/attachments/test.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Test</title>
+  </head>
+  <body>
+    <p>
+      Test
+    </p>
+  </body>
+</html>
diff --git a/spec/fixtures/views/lib.js b/spec/fixtures/views/lib.js
new file mode 100644
index 0000000..12942d0
--- /dev/null
+++ b/spec/fixtures/views/lib.js
@@ -0,0 +1,3 @@
+function globalLib() {
+  return "fixture";
+};
\ No newline at end of file
diff --git a/spec/fixtures/views/test_view/lib.js b/spec/fixtures/views/test_view/lib.js
new file mode 100644
index 0000000..71f1a6f
--- /dev/null
+++ b/spec/fixtures/views/test_view/lib.js
@@ -0,0 +1,3 @@
+function justThisView() {
+  return "fixture";
+};
\ No newline at end of file
diff --git a/spec/fixtures/views/test_view/only-map.js b/spec/fixtures/views/test_view/only-map.js
new file mode 100644
index 0000000..c6b7a7c
--- /dev/null
+++ b/spec/fixtures/views/test_view/only-map.js
@@ -0,0 +1,4 @@
+function(doc) {
+  //include-lib
+  emit(null, null);
+};
\ No newline at end of file
diff --git a/spec/fixtures/views/test_view/test-map.js b/spec/fixtures/views/test_view/test-map.js
new file mode 100644
index 0000000..4bba433
--- /dev/null
+++ b/spec/fixtures/views/test_view/test-map.js
@@ -0,0 +1,3 @@
+function(doc) {
+  emit(null, null);
+};
\ No newline at end of file
diff --git a/spec/fixtures/views/test_view/test-reduce.js b/spec/fixtures/views/test_view/test-reduce.js
new file mode 100644
index 0000000..70c5bd9
--- /dev/null
+++ b/spec/fixtures/views/test_view/test-reduce.js
@@ -0,0 +1,3 @@
+function(ks,vs,co) {
+  return vs.length;
+};
\ No newline at end of file
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 22063b9..6369bb2 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../lib/couchrest'
+require File.dirname(__FILE__) + '/../couchrest'
 
 COUCHHOST = "http://localhost:5984"
 TESTDB = 'couchrest-test'
\ No newline at end of file