Compare commits
7 Commits
Author | SHA1 | Date |
---|---|---|
Denis Knauf | c919500ae3 | |
Denis Knauf | 1063201b64 | |
Denis Knauf | 3bdd2e6ceb | |
Denis Knauf | 1efdf76ffc | |
Denis Knauf | 896f641856 | |
Denis Knauf | 507951114e | |
Denis Knauf | 27f2157bf8 |
45
README.md
45
README.md
|
@ -1,58 +1,48 @@
|
|||
Dependencies
|
||||
============
|
||||
|
||||
incomplete yet.
|
||||
|
||||
System
|
||||
------
|
||||
|
||||
* ruby >= 1.9 (tested: 1.9.1, untested: 1.8 (maybe compatible))
|
||||
* libdb >= 4 (tested: 4.7)
|
||||
* C-Compiler
|
||||
* tokyocabinet
|
||||
|
||||
### Debian/Ubuntu:
|
||||
|
||||
# aptitude ruby1.9.1 ruby1.9.1-dev libdb4.7-dev rubygems1.9.1
|
||||
# aptitude ruby1.9.1 ruby1.9.1-dev rubygems1.9.1 libtokyocabinet-dev libtokyotyrant-dev
|
||||
|
||||
If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and
|
||||
If you've installed ruby1.8 (yet), you should run ruby1.9.1 instead ruby and
|
||||
gem1.9.1 instead gem.
|
||||
Change shebash in s2l.rb to
|
||||
|
||||
#!/usr/bin/ruby1.9.1
|
||||
|
||||
or
|
||||
|
||||
Ruby Gems
|
||||
---------
|
||||
|
||||
* BDB >= 0.2.2 (patch needed - gem included)
|
||||
* UUIDTools
|
||||
|
||||
Install: (in syslog2logan-dir)
|
||||
|
||||
# gem install bdb-0.2.2.gem uuidtools
|
||||
|
||||
#!/usr/bin/env ruby1.9.1
|
||||
|
||||
Install
|
||||
=======
|
||||
|
||||
# gem build syslog2logan.gemspec
|
||||
# gem install syslog2logan-*.gem
|
||||
|
||||
# gem install syslog2logan
|
||||
|
||||
Usage
|
||||
=====
|
||||
|
||||
First you should know, the database environments are in *this* directory,
|
||||
where you call *s2l.rb*. You must use this directory for logan itself too!
|
||||
Don't use this directory for anything else.
|
||||
|
||||
Start
|
||||
-----
|
||||
|
||||
Simple:
|
||||
Simple on Ubuntu:
|
||||
|
||||
# ./s2l.rb
|
||||
# /var/lib/gems/1.9*/gems/syslog2logan-*/bin/s2l.rb
|
||||
|
||||
Or deamonized:
|
||||
|
||||
# sh -c 'nohup ./s2l.rb </dev/null >/dev/null 2>&1 &' &
|
||||
Deamonized:
|
||||
|
||||
# sh -c 'nohup PATHTO/s2l.rb </dev/null >/dev/null 2>&1 &' &
|
||||
|
||||
Use it
|
||||
------
|
||||
|
@ -67,7 +57,7 @@ You need these lines:
|
|||
|
||||
source s_server {
|
||||
unix-stream( "/dev/log" max-connections(100));
|
||||
# internal(); # Statistics about dests. You've any other dest than the server?
|
||||
# internal(); # Statistics about dests. It's unimportant for LogAn.
|
||||
file( "/proc/kmsg");
|
||||
};
|
||||
|
||||
|
@ -80,9 +70,6 @@ You need these lines:
|
|||
destination( d_server);
|
||||
};
|
||||
|
||||
You should use your default source.
|
||||
|
||||
|
||||
### rsyslog
|
||||
|
||||
I don't know. Please tell me, if you can.
|
||||
I don't know. Please tell me, how to use.
|
||||
|
|
4
Rakefile
4
Rakefile
|
@ -10,8 +10,8 @@ begin
|
|||
gem.email = "Denis.Knauf@gmail.com"
|
||||
gem.homepage = "http://github.com/DenisKnauf/syslog2logan"
|
||||
gem.authors = ["Denis Knauf"]
|
||||
gem.files = ["README.md", "VERSION", "bin/**/*", "lib/**/*.rb", "test/**/*.rb"]
|
||||
gem.require_paths = ["bin"]
|
||||
gem.files = %w[AUTHORS README.md VERSION bin/**/* lib/**/*.rb test/**/*.rb]
|
||||
gem.require_paths = %w[bin]
|
||||
gem.add_dependency 'sbdb'
|
||||
gem.add_dependency 'robustserver'
|
||||
gem.add_dependency 'select'
|
||||
|
|
137
bin/s2l.rb
137
bin/s2l.rb
|
@ -1,143 +1,58 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
$:.push File.join( File.dirname( $0), '..', 'lib')
|
||||
require 'logger'
|
||||
require 'json'
|
||||
require 'rubygems'
|
||||
require 'sbdb'
|
||||
require 'uuidtools'
|
||||
require 'socket'
|
||||
require 'select'
|
||||
require 'robustserver'
|
||||
require 'active_support'
|
||||
require 'syslog2logan/rotate'
|
||||
require 'syslog2logan/server'
|
||||
|
||||
class S2L < Select::Server
|
||||
attr_accessor :dbs
|
||||
|
||||
def init p
|
||||
super p
|
||||
@dbs = p[:dbs]
|
||||
end
|
||||
|
||||
def event_new_client a
|
||||
{ :clientclass => S2L::Socket, :dbs => @dbs }
|
||||
end
|
||||
end
|
||||
$logger = Logger.new $stderr
|
||||
$logger.formatter = proc { |severity, datetime, progname, msg| [severity, datetime, progname, msg.inspect].to_json+"\n" }
|
||||
|
||||
module Kernel
|
||||
def debug( *p) logger :debug, *p end
|
||||
def info( *p) logger :info, *p end
|
||||
def warn( *p) logger :warn, *p end
|
||||
def error( *p) logger :error, *p end
|
||||
def fatal( *p) logger :fatal, *p end
|
||||
|
||||
def logger l, *p
|
||||
p = p.first if p.length == 1
|
||||
$stderr.puts [Time.now, l, p].inspect
|
||||
end
|
||||
private :logger
|
||||
end
|
||||
|
||||
class S2L::Socket < Select::Socket
|
||||
def init opts
|
||||
@dbs = opts[ :dbs]
|
||||
super opts
|
||||
end
|
||||
|
||||
def event_line v
|
||||
@dbs.emit v
|
||||
end
|
||||
alias emit event_line
|
||||
end
|
||||
|
||||
class Rotate
|
||||
def initialize db, &e
|
||||
@rdb, @env, @dbs = db, db.home, {}
|
||||
self.hash = e || lambda {|k|
|
||||
[k.timestamp.to_i/60/60/24].pack 'N'
|
||||
}
|
||||
end
|
||||
|
||||
def hash= e
|
||||
self.hash &e
|
||||
end
|
||||
|
||||
def hash &e
|
||||
@hash_func = e if e
|
||||
@hash_func
|
||||
end
|
||||
|
||||
def hashing k
|
||||
@hash_func.call k
|
||||
end
|
||||
|
||||
def db_name id
|
||||
h = hashing id
|
||||
n = @rdb[ h]
|
||||
if n
|
||||
n = UUIDTools::UUID.parse_raw n
|
||||
else
|
||||
n = UUIDTools::UUID.timestamp_create
|
||||
@rdb[ h] = n.raw
|
||||
info :create => n.to_s
|
||||
end
|
||||
n
|
||||
end
|
||||
|
||||
def db n
|
||||
@env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT]
|
||||
end
|
||||
|
||||
def queue n
|
||||
@env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16]
|
||||
end
|
||||
|
||||
def sync
|
||||
@dbs.each{|n,db|db.sync}
|
||||
@rdb.sync
|
||||
end
|
||||
|
||||
def close
|
||||
@dbs.each{|n,db|db.close 0}
|
||||
@rdb.close 0
|
||||
end
|
||||
|
||||
def put v
|
||||
id = UUIDTools::UUID.timestamp_create
|
||||
s = [0x10, v].pack 'Na*'
|
||||
n = db_name id
|
||||
db( n)[ id.raw] = s
|
||||
queue( n).push id.raw
|
||||
end
|
||||
alias emit put
|
||||
def logger() $logger end
|
||||
end
|
||||
|
||||
class Main < RobustServer
|
||||
def initialize conf
|
||||
super
|
||||
@logger = $logger
|
||||
@conf = conf
|
||||
info :open => S2L
|
||||
logger.info :open => S2L
|
||||
@serv = S2L.new :sock => TCPServer.new( *@conf[:server])
|
||||
info :create => {:home => @conf[:home]}
|
||||
Dir.mkdir @conf[:home] rescue Errno::EEXIST
|
||||
@sigs[:INT] = @sigs[:TERM] = method(:shutdown)
|
||||
@sigs[:USR1] = method(:state)
|
||||
end
|
||||
|
||||
def state s = nil
|
||||
logger.debug :server => @serv.class
|
||||
end
|
||||
|
||||
def shutdown s = nil
|
||||
$stderr.puts [:signal, s, Signal[s]].inspect
|
||||
logger.info :shutdown => [s, Signal[s]]
|
||||
@serv.close
|
||||
exit 0
|
||||
end
|
||||
|
||||
def run
|
||||
info :open => SBDB::Env
|
||||
SBDB::Env.new( @conf[:home],
|
||||
log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE,
|
||||
flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv|
|
||||
info :open => Rotate
|
||||
@serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT]
|
||||
info :run => @serv
|
||||
logger.info :open => @conf[:backend]
|
||||
@conf[:backend][0].new( @conf[:backend][1]) do |backend|
|
||||
logger.info :open => Rotate
|
||||
@serv.dbs = Rotate.new &backend.to_proc
|
||||
logger.info :run => @serv.class
|
||||
@serv.run
|
||||
logger.info :close => @conf[:backend]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10]
|
||||
require 'syslog2logan/backend/tch'
|
||||
Main.main :backend => [ Backend::TCH, {:dir => 'logs'}], :server => [ '', 1514], :retries => [1,1] # [10, 10]
|
||||
|
||||
info :halted
|
||||
logger.info :halted
|
||||
|
|
|
@ -0,0 +1,48 @@
|
|||
class File
|
||||
def exclusive_lock
|
||||
flock File::LOCK_EX
|
||||
end
|
||||
|
||||
def shared_lock
|
||||
flock File::LOCK_SH
|
||||
end
|
||||
|
||||
def unblock
|
||||
flock File::LOCK_UN
|
||||
end
|
||||
end
|
||||
|
||||
class FileQueue
|
||||
attr_reader :file, :size
|
||||
alias to_io file
|
||||
|
||||
def initialize file, size = 16
|
||||
@file = case file
|
||||
when File then file
|
||||
else File.open file, 'a+'
|
||||
end
|
||||
@size, @pack = size, "A#{size}"
|
||||
end
|
||||
|
||||
def push *a
|
||||
f = @file
|
||||
f.seek 0, IO::SEEK_END
|
||||
f.exclusive_lock
|
||||
f.write a.pack( @pack*a.length)
|
||||
f.unblock
|
||||
end
|
||||
|
||||
def pop
|
||||
f = @file
|
||||
f.rewind
|
||||
f.exclusive_lock
|
||||
s = f.read( @size).unpack( 'L')[0]
|
||||
f.rewind
|
||||
f.write [s.succ].pack( 'L')
|
||||
f.sync
|
||||
f.shared_lock
|
||||
f.pos = s
|
||||
f.read( @size).unpack( 'L')[0]
|
||||
f.unblock
|
||||
end
|
||||
end
|
|
@ -0,0 +1,83 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
require 'logger'
|
||||
require 'rubygems'
|
||||
require 'uuidtools'
|
||||
require 'socket'
|
||||
require 'select'
|
||||
require 'robustserver'
|
||||
require 'active_support'
|
||||
require 'syslog2logan/rotate'
|
||||
|
||||
$logger = Logger.new $stderr
|
||||
|
||||
class S2L < Select::Server
|
||||
attr_accessor :dbs
|
||||
|
||||
def init p
|
||||
super p
|
||||
@dbs = p[:dbs]
|
||||
end
|
||||
|
||||
def event_new_client a
|
||||
logger.debug :connection => {:new => a}
|
||||
{ :clientclass => S2L::Socket, :dbs => @dbs }
|
||||
end
|
||||
end
|
||||
|
||||
module Kernel
|
||||
def logger() $logger end
|
||||
end
|
||||
|
||||
class S2L::Socket < Select::Socket
|
||||
def init opts
|
||||
@dbs = opts[ :dbs]
|
||||
super opts
|
||||
end
|
||||
|
||||
def event_line v
|
||||
logger.debug :line => v
|
||||
@dbs.emit v
|
||||
end
|
||||
alias emit event_line
|
||||
end
|
||||
|
||||
class Main < RobustServer
|
||||
def initialize conf
|
||||
super
|
||||
@logger = $logger
|
||||
@conf = conf
|
||||
logger.info :open => S2L
|
||||
@serv = S2L.new :sock => TCPServer.new( *@conf[:server])
|
||||
logger.info :create => {:home => @conf[:home]}
|
||||
Dir.mkdir @conf[:home] rescue Errno::EEXIST
|
||||
@sigs[:INT] = @sigs[:TERM] = method(:shutdown)
|
||||
@sigs[:USR1] = method(:state)
|
||||
end
|
||||
|
||||
def state s = nil
|
||||
logger.debug :server => @serv
|
||||
end
|
||||
|
||||
def shutdown s = nil
|
||||
logger.info :shutdown => [s, Signal[s]]
|
||||
@serv.close
|
||||
exit 0
|
||||
end
|
||||
|
||||
def run
|
||||
logger.info :open => SBDB::Env
|
||||
SBDB::Env.new( @conf[:home],
|
||||
log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE,
|
||||
flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv|
|
||||
logger.info :open => Rotate
|
||||
@serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT]
|
||||
logger.info :run => @serv
|
||||
@serv.run
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Main.main :home => 'logs', :server => [ '', 1514], :retries => [1,1] # [10, 10]
|
||||
|
||||
logger.info :halted
|
|
@ -0,0 +1,16 @@
|
|||
module Backend
|
||||
end
|
||||
|
||||
class Backend::Base
|
||||
def initialize opts = {}
|
||||
if block_given?
|
||||
yield self
|
||||
else
|
||||
self
|
||||
end
|
||||
end
|
||||
|
||||
def to_proc
|
||||
method :open
|
||||
end
|
||||
end
|
|
@ -0,0 +1,31 @@
|
|||
require 'rufus/tokyo'
|
||||
require 'syslog2logan/backend/base'
|
||||
|
||||
class Backend::TCH < Backend::Base
|
||||
attr_reader :dir
|
||||
def initialize opts = {}, &e
|
||||
@dir = opts[:dir]
|
||||
Dir.mkdir @dir rescue Errno::EEXIST
|
||||
@dbs = []
|
||||
if block_given?
|
||||
begin
|
||||
super opts, &e
|
||||
ensure
|
||||
close
|
||||
end
|
||||
else
|
||||
super opts
|
||||
end
|
||||
end
|
||||
|
||||
def close
|
||||
@dbs.each &:close
|
||||
end
|
||||
|
||||
def open name
|
||||
logger.info :open => name, :backend => self.class
|
||||
db = Rufus::Tokyo::Cabinet.new File.join( @dir, name)+".tch"
|
||||
@dbs.push db
|
||||
db
|
||||
end
|
||||
end
|
|
@ -0,0 +1,2 @@
|
|||
|
||||
require 'syslog2logan/rotate/base.rb'
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
require 'logger'
|
||||
require 'uuidtools'
|
||||
require 'active_support/core_ext'
|
||||
|
||||
class Rotate
|
||||
# open_db_func: must returns a db-object with #[] and #[]=.
|
||||
# #sync and #close are optional, for Rotate#sync, Rotate#close.
|
||||
def initialize hash_func = nil, &open_db_func
|
||||
@dbs = Hash.new {|h,k| h[k] = open_db_func.call(k) }
|
||||
hash_func ||= lambda {|k| [k.timestamp.to_i/1.day].pack 'N' }
|
||||
define_singleton_method :hashing, &hash_func
|
||||
@rotate = @dbs['rotate']
|
||||
@queue = @dbs['queue']
|
||||
end
|
||||
|
||||
def db_name id
|
||||
h = hashing id
|
||||
n = @rotate[ h]
|
||||
if n
|
||||
n = UUIDTools::UUID.parse_raw n
|
||||
else
|
||||
n = UUIDTools::UUID.timestamp_create
|
||||
@rotate[ h] = n.raw
|
||||
logger.info :create => n.to_s
|
||||
end
|
||||
n
|
||||
end
|
||||
|
||||
# Synchronize data to disc.
|
||||
# Only avaible if db-backend provides #sync.
|
||||
def sync
|
||||
@dbs.each {|n, db| db.sync }
|
||||
@rotate.sync
|
||||
@queue.sync
|
||||
end
|
||||
|
||||
# Close databases.
|
||||
# Only avaible if db-backend provides #close.
|
||||
def close
|
||||
@dbs.each {|n, db| db.close }
|
||||
@rotate.close
|
||||
@queue.close
|
||||
end
|
||||
|
||||
# Put new logline to databases.
|
||||
# This will be written in a database with an UUID as name.
|
||||
# If this db don't exist, it will be created via open_db_func (#initialize).
|
||||
def put v
|
||||
id = UUIDTools::UUID.timestamp_create
|
||||
s = [0x10, v].pack 'Na*'
|
||||
n = db_name id
|
||||
@dbs[n][ id.raw] = s
|
||||
@queue.push id.raw
|
||||
end
|
||||
alias emit put
|
||||
end
|
|
@ -0,0 +1,108 @@
|
|||
#!/usr/bin/ruby
|
||||
|
||||
require 'logger'
|
||||
require 'rubygems'
|
||||
require 'sbdb'
|
||||
require 'uuidtools'
|
||||
require 'socket'
|
||||
require 'select'
|
||||
require 'robustserver'
|
||||
require 'active_support'
|
||||
|
||||
class Rotate::BDB
|
||||
def initialize db, &e
|
||||
@rdb, @env, @dbs = db, db.home, {}
|
||||
self.hash = e || lambda {|k|
|
||||
[k.timestamp.to_i/1.hour].pack 'N'
|
||||
}
|
||||
end
|
||||
|
||||
def hash= e
|
||||
self.hash &e
|
||||
end
|
||||
|
||||
def hash &e
|
||||
@hash_func = e if e
|
||||
@hash_func
|
||||
end
|
||||
|
||||
def hashing k
|
||||
@hash_func.call k
|
||||
end
|
||||
|
||||
def db_name id
|
||||
h = hashing id
|
||||
n = @rdb[ h]
|
||||
if n
|
||||
n = UUIDTools::UUID.parse_raw n
|
||||
else
|
||||
n = UUIDTools::UUID.timestamp_create
|
||||
@rdb[ h] = n.raw
|
||||
logger.info :create => n.to_s
|
||||
end
|
||||
n
|
||||
end
|
||||
|
||||
def db n
|
||||
@env[ n.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT]
|
||||
end
|
||||
|
||||
def queue n
|
||||
@env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16]
|
||||
end
|
||||
|
||||
def sync
|
||||
@dbs.each {|n, db| db.sync }
|
||||
@rdb.sync
|
||||
end
|
||||
|
||||
def close
|
||||
@dbs.each {|n, db| db.close 0 }
|
||||
@rdb.close 0
|
||||
end
|
||||
|
||||
def put v
|
||||
id = UUIDTools::UUID.timestamp_create
|
||||
s = [0x10, v].pack 'Na*'
|
||||
n = db_name id
|
||||
db( n)[ id.raw] = s
|
||||
queue( n).push id.raw
|
||||
end
|
||||
alias emit put
|
||||
end
|
||||
|
||||
class Main < RobustServer
|
||||
def initialize conf
|
||||
super
|
||||
@logger = $logger
|
||||
@conf = conf
|
||||
logger.info :open => S2L
|
||||
@serv = S2L.new :sock => TCPServer.new( *@conf[:server])
|
||||
logger.info :create => {:home => @conf[:home]}
|
||||
Dir.mkdir @conf[:home] rescue Errno::EEXIST
|
||||
@sigs[:INT] = @sigs[:TERM] = method(:shutdown)
|
||||
@sigs[:USR1] = method(:state)
|
||||
end
|
||||
|
||||
def state s = nil
|
||||
logger.debug :server => @serv
|
||||
end
|
||||
|
||||
def shutdown s = nil
|
||||
logger.info :shutdown => [s, Signal[s]]
|
||||
@serv.close
|
||||
exit 0
|
||||
end
|
||||
|
||||
def run
|
||||
logger.info :open => SBDB::Env
|
||||
SBDB::Env.new( @conf[:home],
|
||||
log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE,
|
||||
flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) do |dbenv|
|
||||
logger.info :open => Rotate
|
||||
@serv.dbs = Rotate.new dbenv[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | Bdb::DB_AUTO_COMMIT]
|
||||
logger.info :run => @serv
|
||||
@serv.run
|
||||
end
|
||||
end
|
||||
end
|
|
@ -0,0 +1,29 @@
|
|||
require 'socket'
|
||||
require 'select'
|
||||
|
||||
class S2L < Select::Server
|
||||
attr_accessor :dbs
|
||||
|
||||
def init p
|
||||
super p
|
||||
@dbs = p[:dbs]
|
||||
end
|
||||
|
||||
def event_new_client a
|
||||
logger.debug :connection => {:new => a}
|
||||
{ :clientclass => S2L::Socket, :dbs => @dbs }
|
||||
end
|
||||
end
|
||||
|
||||
class S2L::Socket < Select::Socket
|
||||
def init opts
|
||||
@dbs = opts[ :dbs]
|
||||
super opts
|
||||
end
|
||||
|
||||
def event_line v
|
||||
logger.debug :line => v
|
||||
@dbs.emit v
|
||||
end
|
||||
alias emit event_line
|
||||
end
|
Loading…
Reference in New Issue