From ac07c15c2840a0c3d6aef1c49d0b7dfe556a45e6 Mon Sep 17 00:00:00 2001 From: Chris Anderson Date: Thu, 11 Sep 2008 21:09:24 -0700 Subject: [PATCH] move files around alot today --- lib/commands.rb | 5 - lib/commands/generate.rb | 71 ------------ lib/commands/push.rb | 99 ---------------- lib/couchrest.rb | 4 +- lib/helpers/file_manager.rb | 223 ------------------------------------ lib/helpers/pager.rb | 103 ----------------- lib/helpers/streamer.rb | 29 ----- lib/monkeypatches.rb | 22 ---- 8 files changed, 2 insertions(+), 554 deletions(-) delete mode 100644 lib/commands.rb delete mode 100644 lib/commands/generate.rb delete mode 100644 lib/commands/push.rb delete mode 100644 lib/helpers/file_manager.rb delete mode 100644 lib/helpers/pager.rb delete mode 100644 lib/helpers/streamer.rb delete mode 100644 lib/monkeypatches.rb diff --git a/lib/commands.rb b/lib/commands.rb deleted file mode 100644 index 88369eb..0000000 --- a/lib/commands.rb +++ /dev/null @@ -1,5 +0,0 @@ -require File.join(File.dirname(__FILE__), "..", "couchrest") - -%w(push generate).each do |filename| - require File.join(File.dirname(__FILE__), "commands", filename) -end diff --git a/lib/commands/generate.rb b/lib/commands/generate.rb deleted file mode 100644 index bc46bcb..0000000 --- a/lib/commands/generate.rb +++ /dev/null @@ -1,71 +0,0 @@ -require 'fileutils' - -class CouchRest - module Commands - module Generate - - def self.run(options) - directory = options[:directory] - design_names = options[:trailing_args] - - FileUtils.mkdir_p(directory) - filename = File.join(directory, "lib.js") - self.write(filename, <<-FUNC) - // Put global functions here. - // Include in your views with - // - // //include-lib - FUNC - - design_names.each do |design_name| - subdirectory = File.join(directory, design_name) - FileUtils.mkdir_p(subdirectory) - filename = File.join(subdirectory, "sample-map.js") - self.write(filename, <<-FUNC) - function(doc) { - // Keys is first letter of _id - emit(doc._id[0], doc); - } - FUNC - - filename = File.join(subdirectory, "sample-reduce.js") - self.write(filename, <<-FUNC) - function(keys, values) { - // Count the number of keys starting with this letter - return values.length; - } - FUNC - - filename = File.join(subdirectory, "lib.js") - self.write(filename, <<-FUNC) - // Put functions specific to '#{design_name}' here. - // Include in your views with - // - // //include-lib - FUNC - end - end - - def self.help - helpstring = <<-GEN - - Usage: couchview generate directory design1 design2 design3 ... - - Couchview will create directories and example views for the design documents you specify. - - GEN - helpstring.gsub(/^ /, '') - end - - def self.write(filename, contents) - puts "Writing #{filename}" - File.open(filename, "w") do |f| - # Remove leading spaces - contents.gsub!(/^ ( )?/, '') - f.write contents - end - end - - end - end -end diff --git a/lib/commands/push.rb b/lib/commands/push.rb deleted file mode 100644 index a9a2e95..0000000 --- a/lib/commands/push.rb +++ /dev/null @@ -1,99 +0,0 @@ -class CouchRest - - module Commands - - module Push - - def self.run(options) - directory = options[:directory] - database = options[:trailing_args].first - - fm = CouchRest::FileManager.new(database) - fm.loud = options[:loud] - puts "Pushing views from directory #{directory} to database #{fm.db}" - fm.push_views(directory) - end - - def self.help - helpstring = <<-GEN - - == Pushing views with Couchview == - - Usage: couchview push directory dbname - - Couchview expects a specific filesystem layout for your CouchDB views (see - example below). It also supports advanced features like inlining of library - code (so you can keep DRY) as well as avoiding unnecessary document - modification. - - Couchview also solves a problem with CouchDB's view API, which only provides - access to the final reduce side of any views which have both a map and a - reduce function defined. The intermediate map results are often useful for - development and production. CouchDB is smart enough to reuse map indexes for - functions duplicated across views within the same design document. - - For views with a reduce function defined, Couchview creates both a reduce view - and a map-only view, so that you can browse and query the map side as well as - the reduction, with no performance penalty. - - == Example == - - couchview push foo-project/bar-views baz-database - - This will push the views defined in foo-project/bar-views into a database - called baz-database. Couchview expects the views to be defined in files with - names like: - - foo-project/bar-views/my-design/viewname-map.js - foo-project/bar-views/my-design/viewname-reduce.js - foo-project/bar-views/my-design/noreduce-map.js - - Pushed to => http://localhost:5984/baz-database/_design/my-design - - And the design document: - { - "views" : { - "viewname-map" : { - "map" : "### contents of view-name-map.js ###" - }, - "viewname-reduce" : { - "map" : "### contents of view-name-map.js ###", - "reduce" : "### contents of view-name-reduce.js ###" - }, - "noreduce-map" : { - "map" : "### contents of noreduce-map.js ###" - } - } - } - - Couchview will create a design document for each subdirectory of the views - directory specified on the command line. - - == Library Inlining == - - Couchview can optionally inline library code into your views so you only have - to maintain it in one place. It looks for any files named lib.* in your - design-doc directory (for doc specific libs) and in the parent views directory - (for project global libs). These libraries are only inserted into views which - include the text - - //include-lib - - or - - #include-lib - - Couchview is a result of scratching my own itch. I'd be happy to make it more - general, so please contact me at jchris@grabb.it if you'd like to see anything - added or changed. - - GEN - helpstring.gsub(/^ /, '') - end - - end - - - end - -end diff --git a/lib/couchrest.rb b/lib/couchrest.rb index d3a5366..e63b520 100644 --- a/lib/couchrest.rb +++ b/lib/couchrest.rb @@ -6,8 +6,8 @@ $:.unshift File.expand_path(File.dirname(__FILE__)) require 'monkeypatches' -require 'lib/server' -require 'lib/database' +require 'couchrest/server' +require 'couchrest/database' module CouchRest diff --git a/lib/helpers/file_manager.rb b/lib/helpers/file_manager.rb deleted file mode 100644 index 60ee7a4..0000000 --- a/lib/helpers/file_manager.rb +++ /dev/null @@ -1,223 +0,0 @@ -require 'digest/md5' - -class CouchRest - class FileManager - attr_reader :db - attr_accessor :loud - - LANGS = {"rb" => "ruby", "js" => "javascript"} - MIMES = { - "html" => "text/html", - "htm" => "text/html", - "png" => "image/png", - "css" => "text/css", - "js" => "test/javascript" - } - def initialize(dbname, host="http://localhost:5984") - @db = CouchRest.new(host).database(dbname) - end - - def push_directory(push_dir, docid=nil) - docid ||= push_dir.split('/').reverse.find{|part|!part.empty?} - - pushfiles = Dir["#{push_dir}/**/*.*"].collect do |f| - {f.split("#{push_dir}/").last => open(f).read} - end - - return if pushfiles.empty? - - @attachments = {} - @signatures = {} - pushfiles.each do |file| - name = file.keys.first - value = file.values.first - @signatures[name] = md5(value) - - @attachments[name] = { - "data" => value, - "content_type" => MIMES[name.split('.').last] - } - end - - doc = @db.get(docid) rescue nil - - unless doc - say "creating #{docid}" - @db.save({"_id" => docid, "_attachments" => @attachments, "signatures" => @signatures}) - return - end - - # remove deleted docs - to_be_removed = doc["signatures"].keys.select do |d| - !pushfiles.collect{|p| p.keys.first}.include?(d) - end - - to_be_removed.each do |p| - say "deleting #{p}" - doc["signatures"].delete(p) - doc["_attachments"].delete(p) - end - - # update existing docs: - doc["signatures"].each do |path, sig| - if (@signatures[path] == sig) - say "no change to #{path}. skipping..." - else - say "replacing #{path}" - doc["signatures"][path] = md5(@attachments[path]["data"]) - doc["_attachments"][path].delete("stub") - doc["_attachments"][path].delete("length") - doc["_attachments"][path]["data"] = @attachments[path]["data"] - doc["_attachments"][path].merge!({"data" => @attachments[path]["data"]} ) - - end - end - - # add in new files - new_files = pushfiles.select{|d| !doc["signatures"].keys.include?( d.keys.first) } - - new_files.each do |f| - say "creating #{f}" - path = f.keys.first - content = f.values.first - doc["signatures"][path] = md5(content) - - doc["_attachments"][path] = { - "data" => content, - "content_type" => MIMES[path.split('.').last] - } - end - - begin - @db.save(doc) - rescue Exception => e - say e.message - end - end - - def push_views(view_dir) - designs = {} - - Dir["#{view_dir}/**/*.*"].each do |design_doc| - design_doc_parts = design_doc.split('/') - next if /^lib\..*$/.match design_doc_parts.last - pre_normalized_view_name = design_doc_parts.last.split("-") - view_name = pre_normalized_view_name[0..pre_normalized_view_name.length-2].join("-") - - folder = design_doc_parts[-2] - - designs[folder] ||= {} - designs[folder]["views"] ||= {} - design_lang = design_doc_parts.last.split(".").last - designs[folder]["language"] ||= LANGS[design_lang] - - libs = "" - Dir["#{view_dir}/lib.#{design_lang}"].collect do |global_lib| - libs << open(global_lib).read - libs << "\n" - end - Dir["#{view_dir}/#{folder}/lib.#{design_lang}"].collect do |global_lib| - libs << open(global_lib).read - libs << "\n" - end - if design_doc_parts.last =~ /-map/ - designs[folder]["views"]["#{view_name}-map"] ||= {} - - designs[folder]["views"]["#{view_name}-map"]["map"] = read(design_doc, libs) - - designs[folder]["views"]["#{view_name}-reduce"] ||= {} - designs[folder]["views"]["#{view_name}-reduce"]["map"] = read(design_doc, libs) - end - - if design_doc_parts.last =~ /-reduce/ - designs[folder]["views"]["#{view_name}-reduce"] ||= {} - - designs[folder]["views"]["#{view_name}-reduce"]["reduce"] = read(design_doc, libs) - end - end - - # cleanup empty maps and reduces - designs.each do |name, props| - props["views"].each do |view, funcs| - next unless view.include?("reduce") - props["views"].delete(view) unless funcs.keys.include?("reduce") - end - end - - designs.each do |k,v| - create_or_update("_design/#{k}", v) - end - - designs - end - - def pull_views(view_dir) - prefix = "_design" - ds = db.documents(:startkey => '#{prefix}/', :endkey => '#{prefix}/ZZZZZZZZZ') - ds['rows'].collect{|r|r['id']}.each do |id| - puts directory = id.split('/').last - FileUtils.mkdir_p(File.join(view_dir,directory)) - views = db.get(id)['views'] - - vgroups = views.keys.group_by{|k|k.sub(/\-(map|reduce)$/,'')} - vgroups.each do|g,vs| - mapname = vs.find {|v|views[v]["map"]} - if mapname - # save map - mapfunc = views[mapname]["map"] - mapfile = File.join(view_dir, directory, "#{g}-map.js") # todo support non-js views - File.open(mapfile,'w') do |f| - f.write mapfunc - end - end - - reducename = vs.find {|v|views[v]["reduce"]} - if reducename - # save reduce - reducefunc = views[reducename]["reduce"] - reducefile = File.join(view_dir, directory, "#{g}-reduce.js") # todo support non-js views - File.open(reducefile,'w') do |f| - f.write reducefunc - end - end - end - end - - end - - - private - - def say words - puts words if @loud - end - - def md5 string - Digest::MD5.hexdigest(string) - end - - def read(file, libs=nil) - st = open(file).read - st.sub!(/(\/\/|#)include-lib/,libs) if libs - st - end - - def create_or_update(id, fields) - existing = @db.get(id) rescue nil - - if existing - updated = fields.merge({"_id" => id, "_rev" => existing["_rev"]}) - if existing != updated - say "replacing #{id}" - db.save(updated) - else - say "skipping #{id}" - end - else - say "creating #{id}" - db.save(fields.merge({"_id" => id})) - end - - end - end -end diff --git a/lib/helpers/pager.rb b/lib/helpers/pager.rb deleted file mode 100644 index 229ef8a..0000000 --- a/lib/helpers/pager.rb +++ /dev/null @@ -1,103 +0,0 @@ -class CouchRest - class Pager - attr_accessor :db - def initialize db - @db = db - end - - def all_docs(count=100, &block) - startkey = nil - oldend = nil - - while docrows = request_all_docs(count+1, startkey) - startkey = docrows.last['key'] - docrows.pop if docrows.length > count - if oldend == startkey - break - end - yield(docrows) - oldend = startkey - end - end - - def key_reduce(view, count, firstkey = nil, lastkey = nil, &block) - # start with no keys - startkey = firstkey - # lastprocessedkey = nil - keepgoing = true - - while keepgoing && viewrows = request_view(view, count, startkey) - startkey = viewrows.first['key'] - endkey = viewrows.last['key'] - - if (startkey == endkey) - # we need to rerequest to get a bigger page - # so we know we have all the rows for that key - viewrows = @db.view(view, :key => startkey)['rows'] - # we need to do an offset thing to find the next startkey - # otherwise we just get stuck - lastdocid = viewrows.last['id'] - fornextloop = @db.view(view, :startkey => startkey, :startkey_docid => lastdocid, :count => 2)['rows'] - - newendkey = fornextloop.last['key'] - if (newendkey == endkey) - keepgoing = false - else - startkey = newendkey - end - rows = viewrows - else - rows = [] - for r in viewrows - if (lastkey && r['key'] == lastkey) - keepgoing = false - break - end - break if (r['key'] == endkey) - rows << r - end - startkey = endkey - end - - key = :begin - values = [] - - rows.each do |r| - if key != r['key'] - # we're on a new key, yield the old first and then reset - yield(key, values) if key != :begin - key = r['key'] - values = [] - end - # keep accumulating - values << r['value'] - end - yield(key, values) - - end - end - - private - - def request_all_docs count, startkey = nil - opts = {} - opts[:count] = count if count - opts[:startkey] = startkey if startkey - results = @db.documents(opts) - rows = results['rows'] - rows unless rows.length == 0 - end - - def request_view view, count = nil, startkey = nil, endkey = nil - opts = {} - opts[:count] = count if count - opts[:startkey] = startkey if startkey - opts[:endkey] = endkey if endkey - - results = @db.view(view, opts) - rows = results['rows'] - rows unless rows.length == 0 - end - - end -end \ No newline at end of file diff --git a/lib/helpers/streamer.rb b/lib/helpers/streamer.rb deleted file mode 100644 index c9133ee..0000000 --- a/lib/helpers/streamer.rb +++ /dev/null @@ -1,29 +0,0 @@ -class CouchRest - class Streamer - attr_accessor :db - def initialize db - @db = db - end - - def view name, params = nil - urlst = /^_/.match(name) ? "#{@db.root}/#{name}" : "#{@db.root}/_view/#{name}" - url = CouchRest.paramify_url urlst, params - IO.popen("curl --silent #{url}") do |view| - view.gets # discard header - while row = parse_line(view.gets) - yield row - end - end - end - - private - - def parse_line line - return nil unless line - if /(\{.*\}),?/.match(line.chomp) - JSON.parse($1) - end - end - - end -end \ No newline at end of file diff --git a/lib/monkeypatches.rb b/lib/monkeypatches.rb deleted file mode 100644 index b35dce2..0000000 --- a/lib/monkeypatches.rb +++ /dev/null @@ -1,22 +0,0 @@ - -# this file must be loaded after the JSON gem - -class Time - # this date format sorts lexicographically - # and is compatible with Javascript's new Date(time_string) constructor - # note that sorting will break if you store times from multiple timezones - # I like to add a ENV['TZ'] = 'UTC' to my apps - - def to_json(options = nil) - %("#{strftime("%Y/%m/%d %H:%M:%S %z")}") - end - - # this works to decode the outputted time format - # copied from ActiveSupport - # def self.parse string, fallback=nil - # d = DateTime.parse(string).new_offset - # self.utc(d.year, d.month, d.day, d.hour, d.min, d.sec) - # rescue - # fallback - # end -end \ No newline at end of file