diff --git a/AUTHOR b/AUTHORS similarity index 100% rename from AUTHOR rename to AUTHORS diff --git a/README.md b/README.md index b0c6832..b74060a 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,48 @@ Dependencies ============ -incomplete yet. - System ------ * ruby >= 1.9 (tested: 1.9.1, untested: 1.8 (maybe compatible)) -* libdb >= 4 (tested: 4.7) -* C-Compiler +* tokyocabinet ### Debian/Ubuntu: - # aptitude ruby1.9.1 ruby1.9.1-dev libdb4.7-dev rubygems1.9.1 + # aptitude ruby1.9.1 ruby1.9.1-dev rubygems1.9.1 libtokyocabinet-dev libtokyotyrant-dev -If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and +If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and gem1.9.1 instead gem. Change shebash in s2l.rb to #!/usr/bin/ruby1.9.1 +or -Ruby Gems ---------- - -* BDB >= 0.2.2 (patch needed - gem included) -* UUIDTools - -Install: (in syslog2logan-dir) - - # gem install bdb-0.2.2.gem uuidtools - + #!/usr/bin/env ruby1.9.1 Install ======= - # gem build syslog2logan.gemspec - # gem install syslog2logan-*.gem - + # gem install syslog2logan Usage ===== +First you should know, the database environments are in *this* directory, +where you call *s2l.rb*. You must use this directory for logan itself too! +Don't use this directory for anything else. + Start ----- -Simple: +Simple on Ubuntu: - # ./s2l.rb + # /var/lib/gems/1.9*/gems/syslog2logan-*/bin/s2l.rb -Or deamonized: - - # sh -c 'nohup ./s2l.rb /dev/null 2>&1 &' & +Deamonized: + # sh -c 'nohup PATHTO/s2l.rb /dev/null 2>&1 &' & Use it ------ @@ -67,7 +57,7 @@ You need these lines: source s_server { unix-stream( "/dev/log" max-connections(100)); - # internal(); # Statistics about dests. You've any other dest than the server? + # internal(); # Statistics about dests. It's unimportant for LogAn. file( "/proc/kmsg"); }; @@ -80,9 +70,6 @@ You need these lines: destination( d_server); }; -You should use your default source. - - ### rsyslog -I don't know. Please tell me, if you can. +I don't know. Please tell me, how to use. diff --git a/Rakefile b/Rakefile index 5c1b488..8a519d1 100644 --- a/Rakefile +++ b/Rakefile @@ -10,8 +10,8 @@ begin gem.email = "Denis.Knauf@gmail.com" gem.homepage = "http://github.com/DenisKnauf/syslog2logan" gem.authors = ["Denis Knauf"] - gem.files = ["README.md", "VERSION", "bin/**/*", "lib/**/*.rb", "test/**/*.rb"] - gem.require_paths = ["bin"] + gem.files = %w[AUTHORS README.md VERSION bin/**/* lib/**/*.rb test/**/*.rb] + gem.require_paths = %w[bin] gem.add_dependency 'sbdb' gem.add_dependency 'robustserver' gem.add_dependency 'select' diff --git a/VERSION b/VERSION index 334bcc4..092cda7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.2.1 +0.0.3.0 diff --git a/bin/s2l.rb b/bin/s2l.rb index 5c07008..e5fc371 100755 --- a/bin/s2l.rb +++ b/bin/s2l.rb @@ -1,143 +1,58 @@ #!/usr/bin/ruby +$:.push File.join( File.dirname( $0), '..', 'lib') +require 'logger' +require 'json' require 'rubygems' -require 'sbdb' require 'uuidtools' require 'socket' require 'select' require 'robustserver' +require 'active_support' +require 'syslog2logan/rotate' +require 'syslog2logan/server' -class S2L < Select::Server - attr_accessor :dbs - - def init p - super p - @dbs = p[:dbs] - end - - def event_new_client a - { :clientclass => S2L::Socket, :dbs => @dbs } - end -end +$logger = Logger.new $stderr +$logger.formatter = proc { |severity, datetime, progname, msg| [severity, datetime, progname, msg.inspect].to_json+"\n" } module Kernel - def debug( *p) logger :debug, *p end - def info( *p) logger :info, *p end - def warn( *p) logger :warn, *p end - def error( *p) logger :error, *p end - def fatal( *p) logger :fatal, *p end - - def logger l, *p - p = p.first if p.length == 1 - $stderr.puts [Time.now, l, p].inspect - end - private :logger -end - -class S2L::Socket < Select::Socket - def init opts - @dbs = opts[ :dbs] - super opts - end - - def event_line v - @dbs.emit v - end - alias emit event_line -end - -class Rotate - def initialize db, &e - @rdb, @env, @dbs = db, db.home, {} - self.hash = e || lambda {|k| - [k.timestamp.to_i/60/60/24].pack 'N' - } - end - - def hash= e - self.hash &e - end - - def hash &e - @hash_func = e if e - @hash_func - end - - def hashing k - @hash_func.call k - end - - def db_name id - h = hashing id - n = @rdb[ h] - if n - n = UUIDTools::UUID.parse_raw n - else - n = UUIDTools::UUID.timestamp_create - @rdb[ h] = n.raw - info :create => n.to_s - end - n - end - - def db n - @env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] - end - - def queue n - @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] - end - - def sync - @dbs.each{|n,db|db.sync} - @rdb.sync - end - - def close - @dbs.each{|n,db|db.close 0} - @rdb.close 0 - end - - def put v - id = UUIDTools::UUID.timestamp_create - s = [0x10, v].pack 'Na*' - n = db_name id - db( n)[ id.raw] = s - queue( n).push id.raw - end - alias emit put + def logger() $logger end end class Main < RobustServer def initialize conf super + @logger = $logger @conf = conf - info :open => S2L + logger.info :open => S2L @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) - info :create => {:home => @conf[:home]} - Dir.mkdir @conf[:home] rescue Errno::EEXIST @sigs[:INT] = @sigs[:TERM] = method(:shutdown) + @sigs[:USR1] = method(:state) + end + + def state s = nil + logger.debug :server => @serv.class end def shutdown s = nil - $stderr.puts [:signal, s, Signal[s]].inspect + logger.info :shutdown => [s, Signal[s]] @serv.close exit 0 end def run - info :open => SBDB::Env - SBDB::Env.new( @conf[:home], - log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, - flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| - info :open => Rotate - @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] - info :run => @serv + logger.info :open => @conf[:backend] + @conf[:backend][0].new( @conf[:backend][1]) do |backend| + logger.info :open => Rotate + @serv.dbs = Rotate.new &backend.to_proc + logger.info :run => @serv.class @serv.run + logger.info :close => @conf[:backend] end end end -Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10] +require 'syslog2logan/backend/tch' +Main.main :backend => [ Backend::TCH, {:dir => 'logs'}], :server => [ '', 1514], :retries => [1,1] # [10, 10] -info :halted +logger.info :halted diff --git a/lib/file_queue.rb b/lib/file_queue.rb new file mode 100644 index 0000000..53a4f7a --- /dev/null +++ b/lib/file_queue.rb @@ -0,0 +1,48 @@ +class File + def exclusive_lock + flock File::LOCK_EX + end + + def shared_lock + flock File::LOCK_SH + end + + def unblock + flock File::LOCK_UN + end +end + +class FileQueue + attr_reader :file, :size + alias to_io file + + def initialize file, size = 16 + @file = case file + when File then file + else File.open file, 'a+' + end + @size, @pack = size, "A#{size}" + end + + def push *a + f = @file + f.seek 0, IO::SEEK_END + f.exclusive_lock + f.write a.pack( @pack*a.length) + f.unblock + end + + def pop + f = @file + f.rewind + f.exclusive_lock + s = f.read( @size).unpack( 'L')[0] + f.rewind + f.write [s.succ].pack( 'L') + f.sync + f.shared_lock + f.pos = s + f.read( @size).unpack( 'L')[0] + f.unblock + end +end diff --git a/lib/in_tch.rb b/lib/in_tch.rb new file mode 100644 index 0000000..b5e21c9 --- /dev/null +++ b/lib/in_tch.rb @@ -0,0 +1,83 @@ +#!/usr/bin/ruby + +require 'logger' +require 'rubygems' +require 'uuidtools' +require 'socket' +require 'select' +require 'robustserver' +require 'active_support' +require 'syslog2logan/rotate' + +$logger = Logger.new $stderr + +class S2L < Select::Server + attr_accessor :dbs + + def init p + super p + @dbs = p[:dbs] + end + + def event_new_client a + logger.debug :connection => {:new => a} + { :clientclass => S2L::Socket, :dbs => @dbs } + end +end + +module Kernel + def logger() $logger end +end + +class S2L::Socket < Select::Socket + def init opts + @dbs = opts[ :dbs] + super opts + end + + def event_line v + logger.debug :line => v + @dbs.emit v + end + alias emit event_line +end + +class Main < RobustServer + def initialize conf + super + @logger = $logger + @conf = conf + logger.info :open => S2L + @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) + logger.info :create => {:home => @conf[:home]} + Dir.mkdir @conf[:home] rescue Errno::EEXIST + @sigs[:INT] = @sigs[:TERM] = method(:shutdown) + @sigs[:USR1] = method(:state) + end + + def state s = nil + logger.debug :server => @serv + end + + def shutdown s = nil + logger.info :shutdown => [s, Signal[s]] + @serv.close + exit 0 + end + + def run + logger.info :open => SBDB::Env + SBDB::Env.new( @conf[:home], + log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, + flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| + logger.info :open => Rotate + @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] + logger.info :run => @serv + @serv.run + end + end +end + +Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10] + +logger.info :halted diff --git a/lib/syslog2logan/backend/base.rb b/lib/syslog2logan/backend/base.rb new file mode 100644 index 0000000..bf98610 --- /dev/null +++ b/lib/syslog2logan/backend/base.rb @@ -0,0 +1,16 @@ +module Backend +end + +class Backend::Base + def initialize opts = {} + if block_given? + yield self + else + self + end + end + + def to_proc + method :open + end +end diff --git a/lib/syslog2logan/backend/tch.rb b/lib/syslog2logan/backend/tch.rb new file mode 100644 index 0000000..fbf1a9e --- /dev/null +++ b/lib/syslog2logan/backend/tch.rb @@ -0,0 +1,31 @@ +require 'rufus/tokyo' +require 'syslog2logan/backend/base' + +class Backend::TCH < Backend::Base + attr_reader :dir + def initialize opts = {}, &e + @dir = opts[:dir] + Dir.mkdir @dir rescue Errno::EEXIST + @dbs = [] + if block_given? + begin + super opts, &e + ensure + close + end + else + super opts + end + end + + def close + @dbs.each &:close + end + + def open name + logger.info :open => name, :backend => self.class + db = Rufus::Tokyo::Cabinet.new File.join( @dir, name)+".tch" + @dbs.push db + db + end +end diff --git a/lib/syslog2logan/rotate.rb b/lib/syslog2logan/rotate.rb new file mode 100644 index 0000000..133027b --- /dev/null +++ b/lib/syslog2logan/rotate.rb @@ -0,0 +1,2 @@ + +require 'syslog2logan/rotate/base.rb' diff --git a/lib/syslog2logan/rotate/base.rb b/lib/syslog2logan/rotate/base.rb new file mode 100644 index 0000000..703abfa --- /dev/null +++ b/lib/syslog2logan/rotate/base.rb @@ -0,0 +1,58 @@ +#!/usr/bin/ruby + +require 'logger' +require 'uuidtools' +require 'active_support/core_ext' + +class Rotate + # open_db_func: must returns a db-object with #[] and #[]=. + # #sync and #close are optional, for Rotate#sync, Rotate#close. + def initialize hash_func = nil, &open_db_func + @dbs = Hash.new {|h,k| h[k] = open_db_func.call(k) } + hash_func ||= lambda {|k| [k.timestamp.to_i/1.day].pack 'N' } + define_singleton_method :hashing, &hash_func + @rotate = @dbs['rotate'] + @queue = @dbs['queue'] + end + + def db_name id + h = hashing id + n = @rotate[ h] + if n + n = UUIDTools::UUID.parse_raw n + else + n = UUIDTools::UUID.timestamp_create + @rotate[ h] = n.raw + logger.info :create => n.to_s + end + n + end + + # Synchronize data to disc. + # Only avaible if db-backend provides #sync. + def sync + @dbs.each {|n, db| db.sync } + @rotate.sync + @queue.sync + end + + # Close databases. + # Only avaible if db-backend provides #close. + def close + @dbs.each {|n, db| db.close } + @rotate.close + @queue.close + end + + # Put new logline to databases. + # This will be written in a database with an UUID as name. + # If this db don't exist, it will be created via open_db_func (#initialize). + def put v + id = UUIDTools::UUID.timestamp_create + s = [0x10, v].pack 'Na*' + n = db_name id + @dbs[n][ id.raw] = s + @queue.push id.raw + end + alias emit put +end diff --git a/lib/syslog2logan/rotate/bdb.rb b/lib/syslog2logan/rotate/bdb.rb new file mode 100644 index 0000000..9ccd9b1 --- /dev/null +++ b/lib/syslog2logan/rotate/bdb.rb @@ -0,0 +1,108 @@ +#!/usr/bin/ruby + +require 'logger' +require 'rubygems' +require 'sbdb' +require 'uuidtools' +require 'socket' +require 'select' +require 'robustserver' +require 'active_support' + +class Rotate::BDB + def initialize db, &e + @rdb, @env, @dbs = db, db.home, {} + self.hash = e || lambda {|k| + [k.timestamp.to_i/1.hour].pack 'N' + } + end + + def hash= e + self.hash &e + end + + def hash &e + @hash_func = e if e + @hash_func + end + + def hashing k + @hash_func.call k + end + + def db_name id + h = hashing id + n = @rdb[ h] + if n + n = UUIDTools::UUID.parse_raw n + else + n = UUIDTools::UUID.timestamp_create + @rdb[ h] = n.raw + logger.info :create => n.to_s + end + n + end + + def db n + @env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] + end + + def queue n + @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] + end + + def sync + @dbs.each {|n, db| db.sync } + @rdb.sync + end + + def close + @dbs.each {|n, db| db.close 0 } + @rdb.close 0 + end + + def put v + id = UUIDTools::UUID.timestamp_create + s = [0x10, v].pack 'Na*' + n = db_name id + db( n)[ id.raw] = s + queue( n).push id.raw + end + alias emit put +end + +class Main < RobustServer + def initialize conf + super + @logger = $logger + @conf = conf + logger.info :open => S2L + @serv = S2L.new :sock => TCPServer.new( *@conf[:server]) + logger.info :create => {:home => @conf[:home]} + Dir.mkdir @conf[:home] rescue Errno::EEXIST + @sigs[:INT] = @sigs[:TERM] = method(:shutdown) + @sigs[:USR1] = method(:state) + end + + def state s = nil + logger.debug :server => @serv + end + + def shutdown s = nil + logger.info :shutdown => [s, Signal[s]] + @serv.close + exit 0 + end + + def run + logger.info :open => SBDB::Env + SBDB::Env.new( @conf[:home], + log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, + flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv| + logger.info :open => Rotate + @serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT] + logger.info :run => @serv + @serv.run + end + end +end diff --git a/lib/syslog2logan/server.rb b/lib/syslog2logan/server.rb new file mode 100644 index 0000000..166d863 --- /dev/null +++ b/lib/syslog2logan/server.rb @@ -0,0 +1,29 @@ +require 'socket' +require 'select' + +class S2L < Select::Server + attr_accessor :dbs + + def init p + super p + @dbs = p[:dbs] + end + + def event_new_client a + logger.debug :connection => {:new => a} + { :clientclass => S2L::Socket, :dbs => @dbs } + end +end + +class S2L::Socket < Select::Socket + def init opts + @dbs = opts[ :dbs] + super opts + end + + def event_line v + logger.debug :line => v + @dbs.emit v + end + alias emit event_line +end