From 26de4acc5a24e0f005fb0b3920ad713e7defbea5 Mon Sep 17 00:00:00 2001
From: Chris Anderson <jchris@grabb.it>
Date: Thu, 11 Sep 2008 21:09:39 -0700
Subject: [PATCH] moge thng aournd

---
 lib/couchrest/commands.rb             |   5 +
 lib/couchrest/commands/generate.rb    |  71 ++++++++
 lib/couchrest/commands/push.rb        |  99 ++++++++++++
 lib/couchrest/helpers/file_manager.rb | 223 ++++++++++++++++++++++++++
 lib/couchrest/helpers/pager.rb        | 103 ++++++++++++
 lib/couchrest/helpers/streamer.rb     |  29 ++++
 lib/couchrest/monkeypatches.rb        |  22 +++
 7 files changed, 552 insertions(+)
 create mode 100644 lib/couchrest/commands.rb
 create mode 100644 lib/couchrest/commands/generate.rb
 create mode 100644 lib/couchrest/commands/push.rb
 create mode 100644 lib/couchrest/helpers/file_manager.rb
 create mode 100644 lib/couchrest/helpers/pager.rb
 create mode 100644 lib/couchrest/helpers/streamer.rb
 create mode 100644 lib/couchrest/monkeypatches.rb

diff --git a/lib/couchrest/commands.rb b/lib/couchrest/commands.rb
new file mode 100644
index 0000000..88369eb
--- /dev/null
+++ b/lib/couchrest/commands.rb
@@ -0,0 +1,5 @@
+require File.join(File.dirname(__FILE__), "..", "couchrest")
+  
+%w(push generate).each do |filename|
+  require File.join(File.dirname(__FILE__), "commands", filename)
+end
diff --git a/lib/couchrest/commands/generate.rb b/lib/couchrest/commands/generate.rb
new file mode 100644
index 0000000..bc46bcb
--- /dev/null
+++ b/lib/couchrest/commands/generate.rb
@@ -0,0 +1,71 @@
+require 'fileutils'
+
+class CouchRest
+  module Commands
+    module Generate
+
+      def self.run(options)
+        directory    = options[:directory]
+        design_names = options[:trailing_args]
+
+        FileUtils.mkdir_p(directory)
+        filename = File.join(directory, "lib.js")
+        self.write(filename, <<-FUNC)
+        // Put global functions here.
+        // Include in your views with
+        //
+        //   //include-lib
+        FUNC
+
+        design_names.each do |design_name|
+          subdirectory = File.join(directory, design_name)
+          FileUtils.mkdir_p(subdirectory)
+          filename = File.join(subdirectory, "sample-map.js")
+          self.write(filename, <<-FUNC)
+          function(doc) {
+            // Keys is first letter of _id
+            emit(doc._id[0], doc);
+          }
+          FUNC
+
+          filename = File.join(subdirectory, "sample-reduce.js")
+          self.write(filename, <<-FUNC)
+          function(keys, values) {
+            // Count the number of keys starting with this letter
+            return values.length;
+          }
+          FUNC
+
+          filename = File.join(subdirectory, "lib.js")
+          self.write(filename, <<-FUNC)
+          // Put functions specific to '#{design_name}' here.
+          // Include in your views with
+          //
+          //   //include-lib
+          FUNC
+        end
+      end
+
+      def self.help
+        helpstring = <<-GEN
+
+        Usage: couchview generate directory design1 design2 design3 ...
+
+        Couchview will create directories and example views for the design documents you specify.
+
+        GEN
+        helpstring.gsub(/^        /, '')
+      end
+
+      def self.write(filename, contents)
+        puts "Writing #{filename}"
+        File.open(filename, "w") do |f|
+          # Remove leading spaces
+          contents.gsub!(/^        (  )?/, '')
+          f.write contents
+        end
+      end
+
+    end
+  end
+end
diff --git a/lib/couchrest/commands/push.rb b/lib/couchrest/commands/push.rb
new file mode 100644
index 0000000..a9a2e95
--- /dev/null
+++ b/lib/couchrest/commands/push.rb
@@ -0,0 +1,99 @@
+class CouchRest
+
+  module Commands
+
+    module Push
+
+      def self.run(options)
+        directory = options[:directory]
+        database  = options[:trailing_args].first
+        
+        fm      = CouchRest::FileManager.new(database)
+        fm.loud = options[:loud]
+        puts "Pushing views from directory #{directory} to database #{fm.db}"
+        fm.push_views(directory)
+      end
+
+      def self.help
+        helpstring = <<-GEN
+
+        == Pushing views with Couchview ==
+
+        Usage: couchview push directory dbname
+
+        Couchview expects a specific filesystem layout for your CouchDB views (see
+        example below). It also supports advanced features like inlining of library
+        code (so you can keep DRY) as well as avoiding unnecessary document
+        modification.
+
+        Couchview also solves a problem with CouchDB's view API, which only provides
+        access to the final reduce side of any views which have both a map and a
+        reduce function defined. The intermediate map results are often useful for
+        development and production. CouchDB is smart enough to reuse map indexes for
+        functions duplicated across views within the same design document.
+
+        For views with a reduce function defined, Couchview creates both a reduce view
+        and a map-only view, so that you can browse and query the map side as well as
+        the reduction, with no performance penalty.
+
+        == Example ==
+
+        couchview push foo-project/bar-views baz-database
+
+        This will push the views defined in foo-project/bar-views into a database
+        called baz-database. Couchview expects the views to be defined in files with
+        names like:
+
+        foo-project/bar-views/my-design/viewname-map.js
+        foo-project/bar-views/my-design/viewname-reduce.js
+        foo-project/bar-views/my-design/noreduce-map.js
+
+        Pushed to => http://localhost:5984/baz-database/_design/my-design
+
+        And the design document:
+        {
+          "views" : {
+            "viewname-map" : {
+              "map" : "### contents of view-name-map.js ###"
+            },
+            "viewname-reduce" : {
+              "map" : "### contents of view-name-map.js ###",
+              "reduce" : "### contents of view-name-reduce.js ###"
+            },
+            "noreduce-map" : {
+              "map" : "### contents of noreduce-map.js ###"
+            }
+          }
+        }
+
+        Couchview will create a design document for each subdirectory of the views
+        directory specified on the command line.
+
+        == Library Inlining ==
+
+        Couchview can optionally inline library code into your views so you only have
+        to maintain it in one place. It looks for any files named lib.* in your
+        design-doc directory (for doc specific libs) and in the parent views directory
+        (for project global libs). These libraries are only inserted into views which
+        include the text
+
+        //include-lib
+
+        or
+
+        #include-lib
+
+        Couchview is a result of scratching my own itch. I'd be happy to make it more
+        general, so please contact me at jchris@grabb.it if you'd like to see anything
+        added or changed.
+
+        GEN
+        helpstring.gsub(/^        /, '')
+      end
+
+    end
+
+
+  end
+
+end
diff --git a/lib/couchrest/helpers/file_manager.rb b/lib/couchrest/helpers/file_manager.rb
new file mode 100644
index 0000000..60ee7a4
--- /dev/null
+++ b/lib/couchrest/helpers/file_manager.rb
@@ -0,0 +1,223 @@
+require 'digest/md5'
+
+class CouchRest
+  class FileManager
+    attr_reader :db
+    attr_accessor :loud
+    
+    LANGS = {"rb" => "ruby", "js" => "javascript"}
+    MIMES = {
+      "html"  => "text/html",
+      "htm"   => "text/html",
+      "png"   => "image/png",
+      "css"   => "text/css",
+      "js"    => "test/javascript"
+    }    
+    def initialize(dbname, host="http://localhost:5984")
+      @db = CouchRest.new(host).database(dbname)
+    end
+    
+    def push_directory(push_dir, docid=nil)
+      docid ||= push_dir.split('/').reverse.find{|part|!part.empty?}
+
+      pushfiles = Dir["#{push_dir}/**/*.*"].collect do |f|
+        {f.split("#{push_dir}/").last => open(f).read}
+      end
+
+      return if pushfiles.empty?
+
+      @attachments = {}
+      @signatures = {}
+      pushfiles.each do |file|
+        name = file.keys.first
+        value = file.values.first
+        @signatures[name] = md5(value)
+
+        @attachments[name] = {
+          "data" => value,
+          "content_type" => MIMES[name.split('.').last]
+        }
+      end
+
+      doc = @db.get(docid) rescue nil
+
+      unless doc
+        say "creating #{docid}"
+        @db.save({"_id" => docid, "_attachments" => @attachments, "signatures" => @signatures})
+        return
+      end
+
+      # remove deleted docs
+      to_be_removed = doc["signatures"].keys.select do |d| 
+        !pushfiles.collect{|p| p.keys.first}.include?(d) 
+      end
+
+      to_be_removed.each do |p|
+        say "deleting #{p}"
+        doc["signatures"].delete(p)
+        doc["_attachments"].delete(p)
+      end
+
+      # update existing docs:
+      doc["signatures"].each do |path, sig|
+        if (@signatures[path] == sig)
+          say "no change to #{path}. skipping..."
+        else
+          say "replacing #{path}"
+          doc["signatures"][path] = md5(@attachments[path]["data"])
+          doc["_attachments"][path].delete("stub")
+          doc["_attachments"][path].delete("length")    
+          doc["_attachments"][path]["data"] = @attachments[path]["data"]
+          doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} )
+
+        end
+      end
+
+      # add in new files
+      new_files = pushfiles.select{|d| !doc["signatures"].keys.include?( d.keys.first) } 
+
+      new_files.each do |f|
+        say "creating #{f}"
+        path = f.keys.first
+        content = f.values.first
+        doc["signatures"][path] = md5(content)
+
+        doc["_attachments"][path] = {
+          "data" => content,
+          "content_type" => MIMES[path.split('.').last]
+        }
+      end
+
+      begin
+        @db.save(doc)
+      rescue Exception => e
+        say e.message
+      end
+    end
+    
+    def push_views(view_dir)
+      designs = {}
+
+      Dir["#{view_dir}/**/*.*"].each do |design_doc|
+        design_doc_parts = design_doc.split('/')
+        next if /^lib\..*$/.match design_doc_parts.last
+        pre_normalized_view_name = design_doc_parts.last.split("-")
+        view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-")
+
+        folder = design_doc_parts[-2]
+
+        designs[folder] ||= {}
+        designs[folder]["views"] ||= {}
+        design_lang = design_doc_parts.last.split(".").last
+        designs[folder]["language"] ||= LANGS[design_lang]
+
+        libs = ""
+        Dir["#{view_dir}/lib.#{design_lang}"].collect do |global_lib|
+          libs << open(global_lib).read
+          libs << "\n"
+        end
+        Dir["#{view_dir}/#{folder}/lib.#{design_lang}"].collect do |global_lib|
+          libs << open(global_lib).read
+          libs << "\n"
+        end
+        if design_doc_parts.last =~ /-map/
+          designs[folder]["views"]["#{view_name}-map"] ||= {}
+
+          designs[folder]["views"]["#{view_name}-map"]["map"] = read(design_doc, libs)
+
+          designs[folder]["views"]["#{view_name}-reduce"] ||= {}
+          designs[folder]["views"]["#{view_name}-reduce"]["map"] = read(design_doc, libs)
+        end
+
+        if design_doc_parts.last =~ /-reduce/
+          designs[folder]["views"]["#{view_name}-reduce"] ||= {}
+
+          designs[folder]["views"]["#{view_name}-reduce"]["reduce"] = read(design_doc, libs)
+        end
+      end
+
+      # cleanup empty maps and reduces
+      designs.each do |name, props|
+        props["views"].each do |view, funcs|
+          next unless view.include?("reduce")
+          props["views"].delete(view) unless funcs.keys.include?("reduce")
+        end
+      end
+      
+      designs.each do |k,v|
+        create_or_update("_design/#{k}", v)
+      end
+      
+      designs
+    end
+    
+    def pull_views(view_dir)
+      prefix = "_design"
+      ds = db.documents(:startkey => '#{prefix}/', :endkey => '#{prefix}/ZZZZZZZZZ')
+      ds['rows'].collect{|r|r['id']}.each do |id|
+        puts directory = id.split('/').last
+        FileUtils.mkdir_p(File.join(view_dir,directory))
+        views = db.get(id)['views']
+
+        vgroups = views.keys.group_by{|k|k.sub(/\-(map|reduce)$/,'')}
+        vgroups.each do|g,vs|
+          mapname = vs.find {|v|views[v]["map"]}
+          if mapname
+            # save map
+            mapfunc = views[mapname]["map"]
+            mapfile = File.join(view_dir, directory, "#{g}-map.js") # todo support non-js views
+            File.open(mapfile,'w') do |f|
+              f.write mapfunc
+            end
+          end
+
+          reducename = vs.find {|v|views[v]["reduce"]}
+          if reducename
+            # save reduce
+            reducefunc = views[reducename]["reduce"]
+            reducefile = File.join(view_dir, directory, "#{g}-reduce.js") # todo support non-js views
+            File.open(reducefile,'w') do |f|
+              f.write reducefunc
+            end
+          end
+        end
+      end  
+      
+    end
+    
+    
+    private
+    
+    def say words
+      puts words if @loud
+    end
+    
+    def md5 string
+      Digest::MD5.hexdigest(string)
+    end
+    
+    def read(file, libs=nil)
+      st = open(file).read
+      st.sub!(/(\/\/|#)include-lib/,libs) if libs
+      st
+    end
+    
+    def create_or_update(id, fields)
+      existing = @db.get(id) rescue nil
+
+      if existing
+        updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]})
+        if existing != updated
+          say "replacing #{id}"
+          db.save(updated)
+        else
+          say "skipping #{id}"
+        end
+      else
+        say "creating #{id}"
+        db.save(fields.merge({"_id" => id}))
+      end
+
+    end
+  end
+end
diff --git a/lib/couchrest/helpers/pager.rb b/lib/couchrest/helpers/pager.rb
new file mode 100644
index 0000000..229ef8a
--- /dev/null
+++ b/lib/couchrest/helpers/pager.rb
@@ -0,0 +1,103 @@
+class CouchRest
+  class Pager
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    def all_docs(count=100, &block)
+      startkey = nil
+      oldend = nil
+      
+      while docrows = request_all_docs(count+1, startkey)        
+        startkey = docrows.last['key']
+        docrows.pop if docrows.length > count
+        if oldend == startkey
+          break
+        end
+        yield(docrows)
+        oldend = startkey
+      end
+    end
+    
+    def key_reduce(view, count, firstkey = nil, lastkey = nil, &block)
+      # start with no keys
+      startkey = firstkey
+      # lastprocessedkey = nil
+      keepgoing = true
+      
+      while keepgoing && viewrows = request_view(view, count, startkey)
+        startkey = viewrows.first['key']
+        endkey = viewrows.last['key']
+
+        if (startkey == endkey)
+          # we need to rerequest to get a bigger page
+          # so we know we have all the rows for that key
+          viewrows = @db.view(view, :key => startkey)['rows']
+          # we need to do an offset thing to find the next startkey
+          # otherwise we just get stuck
+          lastdocid = viewrows.last['id']
+          fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :count => 2)['rows']
+
+          newendkey = fornextloop.last['key']
+          if (newendkey == endkey)
+            keepgoing = false
+          else
+            startkey = newendkey
+          end
+          rows = viewrows
+        else
+          rows = []
+          for r in viewrows
+            if (lastkey && r['key'] == lastkey)
+              keepgoing = false
+              break
+            end
+            break if (r['key'] == endkey)
+            rows << r
+          end   
+          startkey = endkey
+        end
+        
+        key = :begin
+        values = []
+
+        rows.each do |r|
+          if key != r['key']
+            # we're on a new key, yield the old first and then reset
+            yield(key, values) if key != :begin
+            key = r['key']
+            values = []
+          end
+          # keep accumulating
+          values << r['value']
+        end
+        yield(key, values)
+
+      end
+    end
+
+    private
+    
+    def request_all_docs count, startkey = nil
+      opts = {}
+      opts[:count] = count if count
+      opts[:startkey] = startkey if startkey      
+      results = @db.documents(opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+    def request_view view, count = nil, startkey = nil, endkey = nil
+      opts = {}
+      opts[:count] = count if count
+      opts[:startkey] = startkey if startkey
+      opts[:endkey] = endkey if endkey
+      
+      results = @db.view(view, opts)
+      rows = results['rows']
+      rows unless rows.length == 0
+    end
+
+  end
+end
\ No newline at end of file
diff --git a/lib/couchrest/helpers/streamer.rb b/lib/couchrest/helpers/streamer.rb
new file mode 100644
index 0000000..c9133ee
--- /dev/null
+++ b/lib/couchrest/helpers/streamer.rb
@@ -0,0 +1,29 @@
+class CouchRest
+  class Streamer
+    attr_accessor :db
+    def initialize db
+      @db = db
+    end
+    
+    def view name, params = nil
+      urlst = /^_/.match(name) ? "#{@db.root}/#{name}" : "#{@db.root}/_view/#{name}"
+      url = CouchRest.paramify_url urlst, params
+      IO.popen("curl --silent #{url}") do |view|
+        view.gets # discard header
+        while row = parse_line(view.gets)
+          yield row
+        end
+      end
+    end
+    
+    private
+    
+    def parse_line line
+      return nil unless line
+      if /(\{.*\}),?/.match(line.chomp)
+        JSON.parse($1)
+      end
+    end
+    
+  end
+end
\ No newline at end of file
diff --git a/lib/couchrest/monkeypatches.rb b/lib/couchrest/monkeypatches.rb
new file mode 100644
index 0000000..b35dce2
--- /dev/null
+++ b/lib/couchrest/monkeypatches.rb
@@ -0,0 +1,22 @@
+
+# this file must be loaded after the JSON gem
+
+class Time
+  # this date format sorts lexicographically
+  # and is compatible with Javascript's new Date(time_string) constructor
+  # note that sorting will break if you store times from multiple timezones
+  # I like to add a ENV['TZ'] = 'UTC' to my apps
+
+  def to_json(options = nil)
+    %("#{strftime("%Y/%m/%d %H:%M:%S %z")}")
+  end
+
+  # this works to decode the outputted time format
+  # copied from ActiveSupport
+  # def self.parse string, fallback=nil
+  #   d = DateTime.parse(string).new_offset
+  #   self.utc(d.year, d.month, d.day, d.hour, d.min, d.sec)
+  # rescue
+  #   fallback
+  # end
+end
\ No newline at end of file