diff --git a/AUTHORS b/AUTHOR similarity index 100% rename from AUTHORS rename to AUTHOR diff --git a/README.md b/README.md index b74060a..b0c6832 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,58 @@ Dependencies ============ +incomplete yet. + System ------ * ruby >= 1.9 (tested: 1.9.1, untested: 1.8 (maybe compatible)) -* tokyocabinet +* libdb >= 4 (tested: 4.7) +* C-Compiler ### Debian/Ubuntu: - # aptitude ruby1.9.1 ruby1.9.1-dev rubygems1.9.1 libtokyocabinet-dev libtokyotyrant-dev + # aptitude ruby1.9.1 ruby1.9.1-dev libdb4.7-dev rubygems1.9.1 -If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and +If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and gem1.9.1 instead gem. Change shebash in s2l.rb to #!/usr/bin/ruby1.9.1 -or - #!/usr/bin/env ruby1.9.1 +Ruby Gems +--------- + +* BDB >= 0.2.2 (patch needed - gem included) +* UUIDTools + +Install: (in syslog2logan-dir) + + # gem install bdb-0.2.2.gem uuidtools + Install ======= - # gem install syslog2logan + # gem build syslog2logan.gemspec + # gem install syslog2logan-*.gem + Usage ===== -First you should know, the database environments are in *this* directory, -where you call *s2l.rb*. You must use this directory for logan itself too! -Don't use this directory for anything else. - Start ----- -Simple on Ubuntu: +Simple: - # /var/lib/gems/1.9*/gems/syslog2logan-*/bin/s2l.rb + # ./s2l.rb -Deamonized: +Or deamonized: + + # sh -c 'nohup ./s2l.rb /dev/null 2>&1 &' & - # sh -c 'nohup PATHTO/s2l.rb /dev/null 2>&1 &' & Use it ------ @@ -57,7 +67,7 @@ You need these lines: source s_server { unix-stream( "/dev/log" max-connections(100)); - # internal(); # Statistics about dests. It's unimportant for LogAn. + # internal(); # Statistics about dests. You've any other dest than the server? file( "/proc/kmsg"); }; @@ -70,6 +80,9 @@ You need these lines: destination( d_server); }; +You should use your default source. + + ### rsyslog -I don't know. Please tell me, how to use. +I don't know. Please tell me, if you can. diff --git a/Rakefile b/Rakefile index 8a519d1..5c1b488 100644 --- a/Rakefile +++ b/Rakefile @@ -10,8 +10,8 @@ begin gem.email = "Denis.Knauf@gmail.com" gem.homepage = "http://github.com/DenisKnauf/syslog2logan" gem.authors = ["Denis Knauf"] - gem.files = %w[AUTHORS README.md VERSION bin/**/* lib/**/*.rb test/**/*.rb] - gem.require_paths = %w[bin] + gem.files = ["README.md", "VERSION", "bin/**/*", "lib/**/*.rb", "test/**/*.rb"] + gem.require_paths = ["bin"] gem.add_dependency 'sbdb' gem.add_dependency 'robustserver' gem.add_dependency 'select' diff --git a/VERSION b/VERSION index 092cda7..334bcc4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.3.0 +0.0.2.1 diff --git a/bin/s2l.rb b/bin/s2l.rb index e5fc371..5c07008 100755 --- a/bin/s2l.rb +++ b/bin/s2l.rb @@ -1,58 +1,143 @@ #!/usr/bin/ruby -$:.push File.join( File.dirname( $0), '..', 'lib') -require 'logger' -require 'json' require 'rubygems' +require 'sbdb' require 'uuidtools' require 'socket' require 'select' require 'robustserver' -require 'active_support' -require 'syslog2logan/rotate' -require 'syslog2logan/server' -$logger = Logger.new $stderr -$logger.formatter = proc { |severity, datetime, progname, msg| [severity, datetime, progname, msg.inspect].to_json+"\n" } +class S2L < Select::Server + attr_accessor :dbs + + def init p + super p + @dbs = p[:dbs] + end + + def event_new_client a + { :clientclass => S2L::Socket, :dbs => @dbs } + end +end module Kernel - def logger() $logger end + def debug( *p) logger :debug, *p end + def info( *p) logger :info, *p end + def warn( *p) logger :warn, *p end + def error( *p) logger :error, *p end + def fatal( *p) logger :fatal, *p end + + def logger l, *p + p = p.first if p.length == 1 + $stderr.puts [Time.now, l, p].inspect + end + private :logger +end + +class S2L::Socket < Select::Socket + def init opts + @dbs = opts[ :dbs] + super opts + end + + def event_line v + @dbs.emit v + end + alias emit event_line +end + +class Rotate + def initialize db, &e + @rdb, @env, @dbs = db, db.home, {} + self.hash = e || lambda {|k| + [k.timestamp.to_i/60/60/24].pack 'N' + } + end + + def hash= e + self.hash &e + end + + def hash &e + @hash_func = e if e + @hash_func + end + + def hashing k + @hash_func.call k + end + + def db_name id + h = hashing id + n = @rdb[ h] + if n + n = UUIDTools::UUID.parse_raw n + else + n = UUIDTools::UUID.timestamp_create + @rdb[ h] = n.raw + info :create => n.to_s + end + n + end + + def db n + @env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] + end + + def queue n + @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] + end + + def sync + @dbs.each{|n,db|db.sync} + @rdb.sync + end + + def close + @dbs.each{|n,db|db.close 0} + @rdb.close 0 + end + + def put v + id = UUIDTools::UUID.timestamp_create + s = [0x10, v].pack 'Na*' + n = db_name id + db( n)[ id.raw] = s + queue( n).push id.raw + end + alias emit put end class Main < RobustServer def initialize conf super - @logger = $logger @conf = conf - logger.info :open => S2L + info :open => S2L @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) + info :create => {:home => @conf[:home]} + Dir.mkdir @conf[:home] rescue Errno::EEXIST @sigs[:INT] = @sigs[:TERM] = method(:shutdown) - @sigs[:USR1] = method(:state) - end - - def state s = nil - logger.debug :server => @serv.class end def shutdown s = nil - logger.info :shutdown => [s, Signal[s]] + $stderr.puts [:signal, s, Signal[s]].inspect @serv.close exit 0 end def run - logger.info :open => @conf[:backend] - @conf[:backend][0].new( @conf[:backend][1]) do |backend| - logger.info :open => Rotate - @serv.dbs = Rotate.new &backend.to_proc - logger.info :run => @serv.class + info :open => SBDB::Env + SBDB::Env.new( @conf[:home], + log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, + flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| + info :open => Rotate + @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] + info :run => @serv @serv.run - logger.info :close => @conf[:backend] end end end -require 'syslog2logan/backend/tch' -Main.main :backend => [ Backend::TCH, {:dir => 'logs'}], :server => [ '', 1514], :retries => [1,1] # [10, 10] +Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10] -logger.info :halted +info :halted diff --git a/lib/file_queue.rb b/lib/file_queue.rb deleted file mode 100644 index 53a4f7a..0000000 --- a/lib/file_queue.rb +++ /dev/null @@ -1,48 +0,0 @@ -class File - def exclusive_lock - flock File::LOCK_EX - end - - def shared_lock - flock File::LOCK_SH - end - - def unblock - flock File::LOCK_UN - end -end - -class FileQueue - attr_reader :file, :size - alias to_io file - - def initialize file, size = 16 - @file = case file - when File then file - else File.open file, 'a+' - end - @size, @pack = size, "A#{size}" - end - - def push *a - f = @file - f.seek 0, IO::SEEK_END - f.exclusive_lock - f.write a.pack( @pack*a.length) - f.unblock - end - - def pop - f = @file - f.rewind - f.exclusive_lock - s = f.read( @size).unpack( 'L')[0] - f.rewind - f.write [s.succ].pack( 'L') - f.sync - f.shared_lock - f.pos = s - f.read( @size).unpack( 'L')[0] - f.unblock - end -end diff --git a/lib/in_tch.rb b/lib/in_tch.rb deleted file mode 100644 index b5e21c9..0000000 --- a/lib/in_tch.rb +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/ruby - -require 'logger' -require 'rubygems' -require 'uuidtools' -require 'socket' -require 'select' -require 'robustserver' -require 'active_support' -require 'syslog2logan/rotate' - -$logger = Logger.new $stderr - -class S2L < Select::Server - attr_accessor :dbs - - def init p - super p - @dbs = p[:dbs] - end - - def event_new_client a - logger.debug :connection => {:new => a} - { :clientclass => S2L::Socket, :dbs => @dbs } - end -end - -module Kernel - def logger() $logger end -end - -class S2L::Socket < Select::Socket - def init opts - @dbs = opts[ :dbs] - super opts - end - - def event_line v - logger.debug :line => v - @dbs.emit v - end - alias emit event_line -end - -class Main < RobustServer - def initialize conf - super - @logger = $logger - @conf = conf - logger.info :open => S2L - @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) - logger.info :create => {:home => @conf[:home]} - Dir.mkdir @conf[:home] rescue Errno::EEXIST - @sigs[:INT] = @sigs[:TERM] = method(:shutdown) - @sigs[:USR1] = method(:state) - end - - def state s = nil - logger.debug :server => @serv - end - - def shutdown s = nil - logger.info :shutdown => [s, Signal[s]] - @serv.close - exit 0 - end - - def run - logger.info :open => SBDB::Env - SBDB::Env.new( @conf[:home], - log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, - flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| - logger.info :open => Rotate - @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] - logger.info :run => @serv - @serv.run - end - end -end - -Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10] - -logger.info :halted diff --git a/lib/syslog2logan/backend/base.rb b/lib/syslog2logan/backend/base.rb deleted file mode 100644 index bf98610..0000000 --- a/lib/syslog2logan/backend/base.rb +++ /dev/null @@ -1,16 +0,0 @@ -module Backend -end - -class Backend::Base - def initialize opts = {} - if block_given? - yield self - else - self - end - end - - def to_proc - method :open - end -end diff --git a/lib/syslog2logan/backend/tch.rb b/lib/syslog2logan/backend/tch.rb deleted file mode 100644 index fbf1a9e..0000000 --- a/lib/syslog2logan/backend/tch.rb +++ /dev/null @@ -1,31 +0,0 @@ -require 'rufus/tokyo' -require 'syslog2logan/backend/base' - -class Backend::TCH < Backend::Base - attr_reader :dir - def initialize opts = {}, &e - @dir = opts[:dir] - Dir.mkdir @dir rescue Errno::EEXIST - @dbs = [] - if block_given? - begin - super opts, &e - ensure - close - end - else - super opts - end - end - - def close - @dbs.each &:close - end - - def open name - logger.info :open => name, :backend => self.class - db = Rufus::Tokyo::Cabinet.new File.join( @dir, name)+".tch" - @dbs.push db - db - end -end diff --git a/lib/syslog2logan/rotate.rb b/lib/syslog2logan/rotate.rb deleted file mode 100644 index 133027b..0000000 --- a/lib/syslog2logan/rotate.rb +++ /dev/null @@ -1,2 +0,0 @@ - -require 'syslog2logan/rotate/base.rb' diff --git a/lib/syslog2logan/rotate/base.rb b/lib/syslog2logan/rotate/base.rb deleted file mode 100644 index 703abfa..0000000 --- a/lib/syslog2logan/rotate/base.rb +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/ruby - -require 'logger' -require 'uuidtools' -require 'active_support/core_ext' - -class Rotate - # open_db_func: must returns a db-object with #[] and #[]=. - # #sync and #close are optional, for Rotate#sync, Rotate#close. - def initialize hash_func = nil, &open_db_func - @dbs = Hash.new {|h,k| h[k] = open_db_func.call(k) } - hash_func ||= lambda {|k| [k.timestamp.to_i/1.day].pack 'N' } - define_singleton_method :hashing, &hash_func - @rotate = @dbs['rotate'] - @queue = @dbs['queue'] - end - - def db_name id - h = hashing id - n = @rotate[ h] - if n - n = UUIDTools::UUID.parse_raw n - else - n = UUIDTools::UUID.timestamp_create - @rotate[ h] = n.raw - logger.info :create => n.to_s - end - n - end - - # Synchronize data to disc. - # Only avaible if db-backend provides #sync. - def sync - @dbs.each {|n, db| db.sync } - @rotate.sync - @queue.sync - end - - # Close databases. - # Only avaible if db-backend provides #close. - def close - @dbs.each {|n, db| db.close } - @rotate.close - @queue.close - end - - # Put new logline to databases. - # This will be written in a database with an UUID as name. - # If this db don't exist, it will be created via open_db_func (#initialize). - def put v - id = UUIDTools::UUID.timestamp_create - s = [0x10, v].pack 'Na*' - n = db_name id - @dbs[n][ id.raw] = s - @queue.push id.raw - end - alias emit put -end diff --git a/lib/syslog2logan/rotate/bdb.rb b/lib/syslog2logan/rotate/bdb.rb deleted file mode 100644 index 9ccd9b1..0000000 --- a/lib/syslog2logan/rotate/bdb.rb +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/ruby - -require 'logger' -require 'rubygems' -require 'sbdb' -require 'uuidtools' -require 'socket' -require 'select' -require 'robustserver' -require 'active_support' - -class Rotate::BDB - def initialize db, &e - @rdb, @env, @dbs = db, db.home, {} - self.hash = e || lambda {|k| - [k.timestamp.to_i/1.hour].pack 'N' - } - end - - def hash= e - self.hash &e - end - - def hash &e - @hash_func = e if e - @hash_func - end - - def hashing k - @hash_func.call k - end - - def db_name id - h = hashing id - n = @rdb[ h] - if n - n = UUIDTools::UUID.parse_raw n - else - n = UUIDTools::UUID.timestamp_create - @rdb[ h] = n.raw - logger.info :create => n.to_s - end - n - end - - def db n - @env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] - end - - def queue n - @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] - end - - def sync - @dbs.each {|n, db| db.sync } - @rdb.sync - end - - def close - @dbs.each {|n, db| db.close 0 } - @rdb.close 0 - end - - def put v - id = UUIDTools::UUID.timestamp_create - s = [0x10, v].pack 'Na*' - n = db_name id - db( n)[ id.raw] = s - queue( n).push id.raw - end - alias emit put -end - -class Main < RobustServer - def initialize conf - super - @logger = $logger - @conf = conf - logger.info :open => S2L - @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) - logger.info :create => {:home => @conf[:home]} - Dir.mkdir @conf[:home] rescue Errno::EEXIST - @sigs[:INT] = @sigs[:TERM] = method(:shutdown) - @sigs[:USR1] = method(:state) - end - - def state s = nil - logger.debug :server => @serv - end - - def shutdown s = nil - logger.info :shutdown => [s, Signal[s]] - @serv.close - exit 0 - end - - def run - logger.info :open => SBDB::Env - SBDB::Env.new( @conf[:home], - log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, - flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| - logger.info :open => Rotate - @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] - logger.info :run => @serv - @serv.run - end - end -end diff --git a/lib/syslog2logan/server.rb b/lib/syslog2logan/server.rb deleted file mode 100644 index 166d863..0000000 --- a/lib/syslog2logan/server.rb +++ /dev/null @@ -1,29 +0,0 @@ -require 'socket' -require 'select' - -class S2L < Select::Server - attr_accessor :dbs - - def init p - super p - @dbs = p[:dbs] - end - - def event_new_client a - logger.debug :connection => {:new => a} - { :clientclass => S2L::Socket, :dbs => @dbs } - end -end - -class S2L::Socket < Select::Socket - def init opts - @dbs = opts[ :dbs] - super opts - end - - def event_line v - logger.debug :line => v - @dbs.emit v - end - alias emit event_line -end