commit 87b2e1a0057409cb790a056ce9b712eb51c28767 Author: Denis Knauf Date: Sat Jan 30 14:58:28 2010 +0100 Initial diff --git a/README b/README new file mode 100644 index 0000000..5cac8b0 --- /dev/null +++ b/README @@ -0,0 +1,100 @@ +Contact +======= + +Denis Knauf + +* Denis dot Knauf at gmail dot com +* https://www.denkn.at +* jabber: DEac at jabber dot ccc dot de + +License +======= + +AGPL 3.0 + +Dependencies +============ + +System +------ + +(incomplete) + +* ruby >= 1.9 (tested: 1.9.1, untested: 1.8 (maybe compatible)) +* libdb >= 4 (tested: 4.7) +* C-Compiler + +Debian/Ubuntu: + + # aptitude ruby1.9.1 ruby1.9.1-dev libdb4.7-dev rubygems1.9.1 + +If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and +gem1.9.1 instead gem. +Change shebash in s2l.rb to + + #!/usr/bin/ruby1.9.1 + + +Ruby Gems +--------- + +* BDB >= 0.2.2 (patch needed - gem included) +* UUIDTools + +Install: (in syslog2logan-dir) + + # gem install bdb-0.2.2.gem uuidtools + + +Install +======= + +No install needed/possible yet. + + +Run +=== + +Simple: + + # ./s2l.rb + +Or deamonized: + + # sh -c 'nohup ./s2l.rb /dev/null 2>&1 &' & + + +Use it +====== + +Your Syslog-server should send everythin via tcp to port 1514. +UDP and TLS aren't possible yet. +If you want to use any of these, you can proxy it via a local syslog-ng. + +syslog-ng +--------- + +You need these lines: + + source s_server { + unix-stream( "/dev/log" max-connections(100)); + # internal(); # Statistics about dests. You've any other dest than the server? + file( "/proc/kmsg"); + }; + + destination d_server { + tcp( "SyslogServer.example.org" port (1514)); + }; + + log { + source( s_server); + destination( d_server); + }; + +You should use your default source. + + +rsyslog +------- + +I don't know. Please tell me, if you can. diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..c8a2c21 --- /dev/null +++ b/Rakefile @@ -0,0 +1,59 @@ +require 'rubygems' +require 'rake' + +begin + require 'jeweler' + Jeweler::Tasks.new do |gem| + gem.name = "syslog2logan" + gem.summary = %Q{Syslog-Server} + gem.description = %Q{Syslog-Server which logs to Berkeley Databases} + gem.email = "Denis.Knauf@gmail.com" + gem.homepage = "http://github.com/DenisKnauf/syslog2logan" + gem.authors = ["Denis Knauf"] + gem.files = ["README", "VERSION", "lib/**/*.rb", "test/**/*.rb"] + gem.require_paths = ["lib"] + gem.add_dependency 'sbdb' + gem.add_dependency 'select' + end + Jeweler::GemcutterTasks.new +rescue LoadError + puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler" +end + +require 'rake/testtask' +Rake::TestTask.new(:test) do |test| + test.libs << 'lib' << 'test' << 'ext' + test.pattern = 'test/**/*_test.rb' + test.verbose = true +end + +begin + require 'rcov/rcovtask' + Rcov::RcovTask.new do |test| + test.libs << 'test' + test.pattern = 'test/**/*_test.rb' + test.verbose = true + end +rescue LoadError + task :rcov do + abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov" + end +end + +task :test => :check_dependencies + +task :default => :test + +require 'rake/rdoctask' +Rake::RDocTask.new do |rdoc| + if File.exist?('VERSION') + version = File.read('VERSION') + else + version = "" + end + + rdoc.rdoc_dir = 'rdoc' + rdoc.title = "sbdb #{version}" + rdoc.rdoc_files.include('README*') + rdoc.rdoc_files.include('lib/**/*.rb') +end diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/bin/s2l.rb b/bin/s2l.rb new file mode 100755 index 0000000..0733327 --- /dev/null +++ b/bin/s2l.rb @@ -0,0 +1,161 @@ +#!/usr/bin/ruby + +require 'rubygems' +require 'sbdb' +require 'uuidtools' +require 'socket' +require 'select' + +class S2L < Select::Server + def init p + super p + @dbs = p[:dbs] + end + + def event_new_client a + { :clientclass => S2L::Socket, :dbs => @dbs } + end +end + +module Kernel + def debug( *p) logger :debug, *p end + def info( *p) logger :info, *p end + def warn( *p) logger :warn, *p end + def error( *p) logger :error, *p end + def fatal( *p) logger :fatal, *p end + + def logger l, *p + p = p.first if p.length == 1 + $stderr.puts [Time.now, l, p].inspect + end + private :logger +end + +class S2L::Socket < Select::Socket + def init opts + @dbs = opts[ :dbs] + super opts + end + + def event_line v + @dbs.emit v + end + alias emit event_line +end + +class Rotate + def initialize db, &e + @rdb, @env, @dbs = db, db.home, {} + self.hash = e || lambda {|k| + [UUIDTools::UUID.parse_raw( k).timestamp.to_i/60/60/24].pack 'N' + } + end + + def hash= e + self.hash &e + end + + def hash &e + @hash_func = e if e + @hash_func + end + + def hashing k + @hash_func.call k + end + + def db k + h = hashing k + db = @dbs[h] + unless db + n = @rdb[ h] + if n + n = UUIDTools::UUID.parse_raw n + else + n = UUIDTools::UUID.timestamp_create + @rdb[ h] = n.raw + end + info :open => n.to_s + db = @env.open SBDB::Btree, n.to_s, 'logs', Bdb::DB_CREATE | Bdb::DB_AUTO_COMMIT, nil + @dbs[h] = db + end + db + end + + def sync + @dbs.each{|n,db|db.sync} + @rdb.sync + end + + def close + @dbs.each{|n,db|db.close 0} + @rdb.close 0 + end + + def put v + id = UUIDTools::UUID.timestamp_create + s = [0x10, v].pack 'Na*' + db(id.raw)[ id.raw] = s + end + alias emit put +end + +class Retries + attr_accessor :max, :range + attr_reader :count, :last + + def initialize max = 10, range = 10 + @max, @range, @count, @last = max, range, 0, Time.now + end + + def retry? + @count = @last + @range > Time.now ? @count + 1 : 1 + @last = Time.now + @count < @max + end + + def run ex, &e + begin e.call *args + rescue ex + retries.retry? and retry + end + end +end + +$conf = { + :home => 'logs', + :server => [ '', 1514], + :retries => [10, 10] +} + +info :create => {:home => $conf[:home]} +Dir.mkdir $conf[:home] rescue Errno::EEXIST + +info :open => SBDB::Env +SBDB::Env.new( $conf[:home], SBDB::CREATE | SBDB::INIT_TXN | SBDB::INIT_LOCK | SBDB::INIT_LOG | SBDB::INIT_MPOOL | Bdb::DB_AUTO_COMMIT) do |dbenv| + info :open => SBDB::Btree + dbenv.open( SBDB::Btree, 'rotates.db', 'rotates', SBDB::CREATE | Bdb::DB_AUTO_COMMIT, nil) do |rdb| + info :open => Rotate + dbs = Rotate.new rdb + info :open => S2L + serv = S2L.new :sock => TCPServer.new( *$conf[:server]), :dbs => dbs + retries = Retries.new *$conf[:retries] + begin + info :run => serv + serv.run + info :shutdown => :stoped + rescue Interrupt + info :shutdown => :interrupted + rescue SignalException + info :shutdown => :signal + rescue Object + error :exception=>$!, :backtrace=>$!.backtrace + retries.retry? and retry + fatal "Too many retries (#{retries.count})" + info :shutdown => :fatal + end + info :close => rdb + end + info :close => dbenv +end +info :halted diff --git a/syslog2logan.gemspec b/syslog2logan.gemspec new file mode 100644 index 0000000..a951f7f --- /dev/null +++ b/syslog2logan.gemspec @@ -0,0 +1,40 @@ +# Generated by jeweler +# DO NOT EDIT THIS FILE DIRECTLY +# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command +# -*- encoding: utf-8 -*- + +Gem::Specification.new do |s| + s.name = %q{syslog2logan} + s.version = "0.0.1" + + s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= + s.authors = ["Denis Knauf"] + s.date = %q{2010-01-30} + s.default_executable = %q{s2l.rb} + s.description = %q{Syslog-Server which logs to Berkeley Databases} + s.email = %q{Denis.Knauf@gmail.com} + s.executables = ["s2l.rb"] + s.extra_rdoc_files = [ + "README" + ] + s.files = [ + "README", + "VERSION" + ] + s.homepage = %q{http://github.com/DenisKnauf/syslog2logan} + s.rdoc_options = ["--charset=UTF-8"] + s.require_paths = ["lib"] + s.rubygems_version = %q{1.3.5} + s.summary = %q{Syslog-Server} + + if s.respond_to? :specification_version then + current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION + s.specification_version = 3 + + if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then + else + end + else + end +end +