From 492d9f6dffe0be60ce28e8746bc6a434ae1c0ee6 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Mon, 29 Mar 2010 22:15:06 +0200 Subject: [PATCH 01/45] Server: ArgumentError detected... --- lib/logan/inc/server.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index cacc9d8..0106e19 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -97,8 +97,9 @@ class LogAn::Inc::Server < ::Select::Server def event_cmd cmd sid, line = cmd.unpack 'Na*' - fileparser = @config[:fileparser][sid] - fileparser.event_line line, self if fileparser + fps = @config[:fileparser] + fp = fps[sid] + fp.event_line line, self if fileparser end end end From fb987eb4611386a6d5bae4858098a5d247ed405f Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Mon, 29 Mar 2010 22:36:57 +0200 Subject: [PATCH 02/45] the fileparser-problem --- lib/logan/cache.rb | 9 +++++---- lib/logan/inc/main.rb | 19 +++++++++++++++---- lib/logan/inc/server.rb | 3 +-- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 8c6087b..de71614 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -42,7 +42,7 @@ class LogAn::Cache def write_cache= type @type &= ~ (type ? 0 : 2) - define_singleton_method :[], method( type ? :oset : :dset) + define_singleton_method :[]=, method( type ? :oset : :dset) end #include Enumerable @@ -56,6 +56,7 @@ end class LogAn::AutoValueConvertHash include Enumerable + attr_reader :decode, :encode def initialize obj, encode = nil, each = nil, &decode @object, @encoder = obj, decode.nil? ? encode || Marshal.method( :dump) : nil, @@ -64,16 +65,16 @@ class LogAn::AutoValueConvertHash end def [] k - decode.call @object[k] + @decode.call @object[k] end def []= k, v - @object[k] = encode.call v + @object[k] = @encode.call v end def each *paras @each.call *paras do |k, v| - yield k, decode( v) + yield k, @decode.call( v) end end end diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index fe11de3..6d29a3e 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -10,11 +10,11 @@ require 'logan/cache' module LogAn::Inc class Main < RobustServer # Open Config. - def config env, db, type = nil, flags = nil + def config env, db, type = nil, flags = nil, &e $stderr.puts "Open Database \"sids.cnf\" #{db.inspect} (#{type.inspect})" type ||= 1+4 ret = env[ 'sids.cnf', db, :flags => flags || SBDB::RDONLY] - ret = LogAn::AutoValueConvertHash.new ret if type&4 > 0 + ret = LogAn::AutoValueConvertHash.new ret, &e if type&4 > 0 or e ret = LogAn::Cache.new ret, type&3 if type&3 > 0 ret end @@ -31,27 +31,38 @@ module LogAn::Inc def initialize conf super @conf = {} + # Copy config - changes possible conf.each {|key, val| @conf[key]= val } + # Default directories %w[logs etc].each {|key| @conf[key.to_sym] = key } + # Open Loglines-databases @logs = LogAn::Loglines.new @conf[:logs] + # Open config-databases Dir.mkdir @conf[:etc] rescue Errno::EEXIST @etc = SBDB::Env.new( @conf[:etc], log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) + # Set inc-config - stored in etc/inc.cnf - @conf[:inc] = {} - %w[hosts files fileparser].each {|key| @conf[:inc][key.to_sym] = config( @etc, key) } + begin + inc = @conf[:inc] = {} + %w[hosts files fileparser].each {|key| @conf[:inc][key.to_sym] = config( @etc, key) } + inc[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} + end @store = LogAn::Cache.new LogAn::AutoValueConvertHash.new( @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT]), 3 + # Prepare Inc-server - create server LogAn::Inc::FileParser::Base.logdb = @logs LogAn::Inc::FileParser::Base.store = @store @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:inc] + # Shutdown on signals @sigs[:INT] = @sigs[:TERM] = method( :shutdown) + rescue Object # It's better to close everything, because BDB doesn't like unexpected exits self.at_exit diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index 0106e19..a0c2e70 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -97,8 +97,7 @@ class LogAn::Inc::Server < ::Select::Server def event_cmd cmd sid, line = cmd.unpack 'Na*' - fps = @config[:fileparser] - fp = fps[sid] + fps = @config[:fileparser][sid] fp.event_line line, self if fileparser end end From 7a26a9dfad4d6f71cd81b27181a9bd92062205b0 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Mon, 29 Mar 2010 23:12:43 +0200 Subject: [PATCH 03/45] Cache: vars renamed --- lib/logan/cache.rb | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index de71614..f6ed504 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -10,16 +10,16 @@ class LogAn::Cache end def flush! - @data.each {|k,v| @obj[k] = v } + @data.each {|k,v| @source[k] = v } @data = {} end def dget k - @data[k] ||= @obj[k] + @data[k] ||= @source[k] end def oget k - @data[k] || @obj[k] + @data[k] || @source[k] end def dset k, v @@ -27,7 +27,7 @@ class LogAn::Cache end def oset k, v - @obj[k] = v + @source[k] = v end def type= type @@ -49,7 +49,7 @@ class LogAn::Cache #def each &e #return Enumerator.new self, :each unless e #flush! - #@obj.each &e + #@source.each &e #self #end end @@ -58,18 +58,18 @@ class LogAn::AutoValueConvertHash include Enumerable attr_reader :decode, :encode - def initialize obj, encode = nil, each = nil, &decode - @object, @encoder = obj, decode.nil? ? encode || Marshal.method( :dump) : nil, - @each = each || obj.method( :each) rescue NameError + def initialize source, encode = nil, each = nil, &decode + @source, @encoder = source, decode.nil? ? encode || Marshal.method( :dump) : nil, + @each = each || source.method( :each) rescue NameError @decode = decode || Marshal.method( :restore) end def [] k - @decode.call @object[k] + @decode.call @source[k] end def []= k, v - @object[k] = @encode.call v + @source[k] = @encode.call v end def each *paras From 3c86d43bcfa57bd0201d2593386dafd15997c8fb Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 00:32:50 +0200 Subject: [PATCH 04/45] stores (seeks) rewritten --- lib/logan/inc/command.rb | 18 ++++++++++------- lib/logan/inc/fileparser.rb | 9 ++------- lib/logan/inc/main.rb | 39 +++++++++++++++++++++++++------------ lib/logan/inc/server.rb | 2 +- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/lib/logan/inc/command.rb b/lib/logan/inc/command.rb index 29aab03..88adf69 100644 --- a/lib/logan/inc/command.rb +++ b/lib/logan/inc/command.rb @@ -14,24 +14,28 @@ module LogAn end class SID0 < Command - def initialize store, config, sid = 0 - @store, @config = store, config + class < flags || SBDB::RDONLY] ret = LogAn::AutoValueConvertHash.new ret, &e if type&4 > 0 or e ret = LogAn::Cache.new ret, type&3 if type&3 > 0 ret end + # Open Store. + def store env, db, type = nil, flags = nil, &e + $stderr.puts [:store, :open, "sids.cnf", db, type].inspect + cache env[ 'sids.cnf', db, :flags => flags || SBDB::CREATE | SBDB::AUTO_COMMIT], type, &e + end + + # Open Config. + def config env, db, type = nil, flags = nil, &e + $stderr.puts [:config, :open, "sids.cnf", db, type].inspect + cache env[ 'sids.cnf', db, :flags => flags || SBDB::RDONLY], type, &e + end + # Prepare Server. # # * conf: @@ -40,6 +49,7 @@ module LogAn::Inc # Open Loglines-databases @logs = LogAn::Loglines.new @conf[:logs] + LogAn::Inc::FileParser::Base.logdb = @logs # Open config-databases Dir.mkdir @conf[:etc] rescue Errno::EEXIST @@ -47,18 +57,23 @@ module LogAn::Inc log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE, flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL) - # Set inc-config - stored in etc/inc.cnf + # Open configs begin - inc = @conf[:inc] = {} - %w[hosts files fileparser].each {|key| @conf[:inc][key.to_sym] = config( @etc, key) } - inc[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} + configs = @conf[:configs] = {} + %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) } + configs[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} + end + + # Open seeks-database + begin + stores = @conf[:stores] = {} + stores[:seeks] = store( @etc, 'seeks', 3, + lambda {|val| val.pack( 'NN') }) {|val| (val||0.chr*8).unpack( 'NN') } + LogAn::Inc::FileParser::Base.store = LogAn::Inc::SID0.store = stores end - @store = LogAn::Cache.new LogAn::AutoValueConvertHash.new( @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT]), 3 # Prepare Inc-server - create server - LogAn::Inc::FileParser::Base.logdb = @logs - LogAn::Inc::FileParser::Base.store = @store - @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:inc] + @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:configs] # Shutdown on signals @sigs[:INT] = @sigs[:TERM] = method( :shutdown) diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index a0c2e70..ab50b16 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -98,7 +98,7 @@ class LogAn::Inc::Server < ::Select::Server def event_cmd cmd sid, line = cmd.unpack 'Na*' fps = @config[:fileparser][sid] - fp.event_line line, self if fileparser + fp.event_line line, self if fp end end end From 53bf167fd1d229bc9bcc833e96dfb983772c09c6 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 00:37:02 +0200 Subject: [PATCH 05/45] stores (seeks) rewritten --- lib/logan/inc/main.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 5718013..e92d191 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -67,8 +67,9 @@ module LogAn::Inc # Open seeks-database begin stores = @conf[:stores] = {} - stores[:seeks] = store( @etc, 'seeks', 3, - lambda {|val| val.pack( 'NN') }) {|val| (val||0.chr*8).unpack( 'NN') } + db = @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT] + db = LogAn::AutoValueConvertHash.new( db, lambda {|val| val.pack( 'NN') }) {|val| (val||0.chr*8).unpack( 'NN') } + stores[:seeks] = LogAn::Cache.new db LogAn::Inc::FileParser::Base.store = LogAn::Inc::SID0.store = stores end From 3f267e2ca2df5ebf4ae9c3dea9862bc051954d0d Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 01:15:03 +0200 Subject: [PATCH 06/45] sid0-problem --- lib/logan/inc/main.rb | 2 +- lib/logan/inc/server.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index e92d191..0242db0 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -9,7 +9,7 @@ require 'logan/cache' module LogAn::Inc class Main < RobustServer - def cache db, type, &e + def cache ret, type, &e type ||= 1+4 ret = LogAn::AutoValueConvertHash.new ret, &e if type&4 > 0 or e ret = LogAn::Cache.new ret, type&3 if type&3 > 0 diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index ab50b16..4f03e3b 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -92,12 +92,13 @@ class LogAn::Inc::Server < ::Select::Server def init opts super opts + @sid0 = LogAn::Inc::SID0.new @config = opts[:config] or raise( ArgumentError, "#{self.class} needs a Config!") end def event_cmd cmd sid, line = cmd.unpack 'Na*' - fps = @config[:fileparser][sid] + fp = sid == 0 ? @sid0 : @config[:fileparser][sid] fp.event_line line, self if fp end end From cc8f758f15856a5ed5f35c20c2ecd80ce3d6b80e Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 01:16:52 +0200 Subject: [PATCH 07/45] sid0-problem --- lib/logan/inc/command.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/command.rb b/lib/logan/inc/command.rb index 88adf69..50bd356 100644 --- a/lib/logan/inc/command.rb +++ b/lib/logan/inc/command.rb @@ -7,7 +7,7 @@ module LogAn @sid = sid end - def event_read line, sock + def event_line line, sock cmd, l = line.unpack 'na*' self[cmd].call( l, sock) if self[cmd] end From e901e7a99155957b9f65e527d2768fbff152ff87 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 01:18:09 +0200 Subject: [PATCH 08/45] fileparser-problem --- lib/logan/inc/main.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 0242db0..2905200 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -62,6 +62,7 @@ module LogAn::Inc configs = @conf[:configs] = {} %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) } configs[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} + LogAn::Inc::SID0.config = configs end # Open seeks-database From 3e9022a67730afd0173401d0d68c0d56433958eb Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 01:19:45 +0200 Subject: [PATCH 09/45] sid0-problem --- lib/logan/inc/command.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/logan/inc/command.rb b/lib/logan/inc/command.rb index 50bd356..f33044c 100644 --- a/lib/logan/inc/command.rb +++ b/lib/logan/inc/command.rb @@ -15,7 +15,10 @@ module LogAn class SID0 < Command class < Date: Tue, 30 Mar 2010 01:22:48 +0200 Subject: [PATCH 10/45] Cache#each added. --- lib/logan/cache.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index f6ed504..e079c91 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -45,13 +45,14 @@ class LogAn::Cache define_singleton_method :[]=, method( type ? :oset : :dset) end - #include Enumerable - #def each &e - #return Enumerator.new self, :each unless e - #flush! - #@source.each &e - #self - #end + include Enumerable + def each + return Enumerator.new self, :each unless e + (@source.keys + @data.keys).each do |key| + yield key, self[key] + end + self + end end class LogAn::AutoValueConvertHash From 694d2a22fa6209c52f4956688d1b628a5655e059 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 01:23:37 +0200 Subject: [PATCH 11/45] Cache#each added. --- lib/logan/cache.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index e079c91..31dfd8d 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -47,7 +47,7 @@ class LogAn::Cache include Enumerable def each - return Enumerator.new self, :each unless e + return Enumerator.new self, :each unless block_given? (@source.keys + @data.keys).each do |key| yield key, self[key] end From 3fa3ac495d0243f35757ba745e51bbf883f5e3f3 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 11:43:59 +0200 Subject: [PATCH 12/45] Cache#each -> AVCH#each_keys --- lib/logan/cache.rb | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 31dfd8d..4ab7e98 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -46,9 +46,10 @@ class LogAn::Cache end include Enumerable - def each + def each *paras return Enumerator.new self, :each unless block_given? - (@source.keys + @data.keys).each do |key| + flush! if @type&2 == 2 + @source.each_keys( *paras) do |key| yield key, self[key] end self @@ -74,8 +75,16 @@ class LogAn::AutoValueConvertHash end def each *paras + return Enumerator.new self, :each unless block_given? @each.call *paras do |k, v| yield k, @decode.call( v) end end + + def each_keys *paras + return Enumerator.new self, :each_keys unless block_given? + @each.call *paras do |k, v| + yield k + end + end end From c4d3e752239d29d50d13e3dc6d0b339b27e15082 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Tue, 30 Mar 2010 17:31:35 +0200 Subject: [PATCH 13/45] String-based config for files and hosts. Safebox-eval for fileparser --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 2905200..e849d4d 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -60,7 +60,7 @@ module LogAn::Inc # Open configs begin configs = @conf[:configs] = {} - %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) } + %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) {|l|l} } configs[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} LogAn::Inc::SID0.config = configs end From c8d68f41b1b38ec6035abc9fc3de41eb7e6bd4c0 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:31:00 +0200 Subject: [PATCH 14/45] bin/bos* removed. Status-informations every 5 seconds added --- bin/box.rb | 67 --------------------------------- bin/box2.rb | 87 ------------------------------------------- bin/box3.rb | 22 ----------- lib/logan/inc/main.rb | 12 +++++- 4 files changed, 11 insertions(+), 177 deletions(-) delete mode 100755 bin/box.rb delete mode 100755 bin/box2.rb delete mode 100755 bin/box3.rb diff --git a/bin/box.rb b/bin/box.rb deleted file mode 100755 index 7515402..0000000 --- a/bin/box.rb +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/ruby - -require 'thread' - -class Box - attr_reader :_, :emited - alias db emited - alias persistent emited - attr_accessor :emited - - def initialize db, _ - @_, @emited = _, db - end - - def emit k, v - @emited[k] = v - end - - def do code - instance_eval code, self.class.to_s, 0 - end -end - -require 'sbdb' - -class Persistent - include Enumerable - def initialize( db) @db, @cursor = db, db.cursor end - def emit( k, v) @db[k] = v end - alias push emit - alias put emit - alias []= emit - def get( k) @db[k] end - alias [] get - alias fetch get - def inspect() "#<%s:0x%016x>" % [ self.class, self.object_id ] end - def each &e - e ? @cursor.each( &e) : Enumerator.new( self, :each) - end - def to_hash - h = {} - each {|k, v| h[ k] = v } - h - end -end - -#Persistent.freeze - -r = nil -Dir.mkdir 'logs' rescue Errno::EEXIST -SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs| - db = Persistent.new logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE] - $stdout.print "(0)$ " - STDIN.each_with_index do |l, i| - r = Thread.new do - l.untaint - $SAFE = 4 - b = Box.new db, r - begin - b.do( l) - rescue Object - $! - end - end.value - $stdout.print "=> #{r.inspect}\n(#{i+1})$ " - end -end diff --git a/bin/box2.rb b/bin/box2.rb deleted file mode 100755 index dd02b06..0000000 --- a/bin/box2.rb +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/ruby - -require 'sbdb' - -module Sandbox - def self.run *paras, &exe - exe = paras.shift unless exe - box = paras.shift || Class - Thread.new do - $SAFE = 4 - this = box.new *paras - begin - [:value, this.instance_eval( exe, "Sandbox")] - rescue Object - [:exception, $!] - end - end.value - end - - def self.create_class *paras, &exe - exe = paras.shift unless exe - run Class, *paras do - eval exe - self - end - end - alias new_class create_class -end - -class Box - attr_reader :_, :db - - def initialize db, _ - @_, @db = _, db - end - - def put( key, val) @db[key] = val end - def get( key) @db[key] end -end - -class ClassBuilder -end - -class Emit - def initialize( db) @db = db end - def emit( key, val) @db[key] = val end - def inspect() "#<%s:0x%016x>" % [ self.class, self.object_id ] end -end - -class Persistent < Emit - include Enumerable - def initialize db, cursor - super db - @cursor = cursor - end - alias put emit - alias []= emit - def get( key) @db[key] end - alias [] get - alias fetch get - def each &exe - exe ? @cursor.each( &exe) : Enumerator.new( self, :each) - end - def to_hash - rh = {} - each {|key, val| rh[ key] = val } - rh - end -end - -_ = nil -Dir.mkdir 'logs' rescue Errno::EEXIST -SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs| - db = logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE] - db = Persistent.new db, db.cursor - $stdout.print "(0)$ " - STDIN.each_with_index do |line, i| - ret = Sandbox.run line, Box, db, _ - if :value == ret.first - _ = ret.last - $stdout.puts "=> #{ret.last.inspect}" - else - $stdout.puts ret.last.inspect - end - $stdout.print "(#{i+1})$ " - end -end diff --git a/bin/box3.rb b/bin/box3.rb deleted file mode 100755 index beae5a7..0000000 --- a/bin/box3.rb +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/ruby - -require 'sbdb' -require 'safebox' - -_ = nil -Dir.mkdir 'logs' rescue Errno::EEXIST -SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs| - db = logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE] - db = Safebox::Persistent.new db, db.cursor - $stdout.print "(0)$ " - STDIN.each_with_index do |line, i| - ret = Safebox.run line, Safebox::Box, db, _ - if :value == ret.first - _ = ret.last - $stdout.puts "=> #{ret.last.inspect}" - else - $stdout.puts ret.last.inspect - end - $stdout.print "(#{i+1})$ " - end -end diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index e849d4d..f5578ad 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -74,8 +74,18 @@ module LogAn::Inc LogAn::Inc::FileParser::Base.store = LogAn::Inc::SID0.store = stores end + # Select-framework + @select = LogAn::Inc::Select.new + status = lambda do + @select.at Time.now+5, &status + $stderr.puts "#{Time.now.strftime"%H:%M:%S"}|INFO|Statistic|#{@select.inspect}" + @conf[:stores].source.source.flush! + end + status.call + # Prepare Inc-server - create server - @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:configs] + @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:configs], :select => @select + $stderr.puts @serv.inspect # Shutdown on signals @sigs[:INT] = @sigs[:TERM] = method( :shutdown) From 0de563251623ad69d0b3ae84901c3001e2b2e4fe Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:36:00 +0200 Subject: [PATCH 15/45] Status-informations every 5 seconds added (fix) --- lib/logan/cache.rb | 2 +- lib/logan/inc/main.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 4ab7e98..2214d2d 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -58,7 +58,7 @@ end class LogAn::AutoValueConvertHash include Enumerable - attr_reader :decode, :encode + attr_reader :decode, :encode, :source def initialize source, encode = nil, each = nil, &decode @source, @encoder = source, decode.nil? ? encode || Marshal.method( :dump) : nil, diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index f5578ad..67396c3 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -79,7 +79,7 @@ module LogAn::Inc status = lambda do @select.at Time.now+5, &status $stderr.puts "#{Time.now.strftime"%H:%M:%S"}|INFO|Statistic|#{@select.inspect}" - @conf[:stores].source.source.flush! + @conf[:stores].each{|key, db| db.source.source.flush!} end status.call From a314bd8d9a4211dcc4fabd4d614b078348cb2029 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:37:34 +0200 Subject: [PATCH 16/45] Status-informations every 5 seconds added (flush cache, not db) --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 67396c3..a5533ad 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -79,7 +79,7 @@ module LogAn::Inc status = lambda do @select.at Time.now+5, &status $stderr.puts "#{Time.now.strftime"%H:%M:%S"}|INFO|Statistic|#{@select.inspect}" - @conf[:stores].each{|key, db| db.source.source.flush!} + @conf[:stores].each{|key, db| db.flush!} end status.call From adbf24bcdcae4de9b68d4b7e2652486b14ab8008 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:51:39 +0200 Subject: [PATCH 17/45] LogAn::Logging for logging added. --- lib/logan/inc/main.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index a5533ad..a46da51 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -7,6 +7,13 @@ require 'logan/inc' require 'logan/loglines' require 'logan/cache' +module LogAn::Logging + def log lvl, *txt + $stderr.puts( ([Time.now, lvl]+txt).inspect) + end + alias method_missing log +end + module LogAn::Inc class Main < RobustServer def cache ret, type, &e @@ -18,13 +25,13 @@ module LogAn::Inc # Open Store. def store env, db, type = nil, flags = nil, &e - $stderr.puts [:store, :open, "sids.cnf", db, type].inspect + LogAn::Logging.info :store, :open, "sids.cnf", db, type cache env[ 'sids.cnf', db, :flags => flags || SBDB::CREATE | SBDB::AUTO_COMMIT], type, &e end # Open Config. def config env, db, type = nil, flags = nil, &e - $stderr.puts [:config, :open, "sids.cnf", db, type].inspect + LogAn::Logging.info :config, :open, "sids.cnf", db, type cache env[ 'sids.cnf', db, :flags => flags || SBDB::RDONLY], type, &e end @@ -78,14 +85,14 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - $stderr.puts "#{Time.now.strftime"%H:%M:%S"}|INFO|Statistic|#{@select.inspect}" + LogAn::Logging.info @select @conf[:stores].each{|key, db| db.flush!} end status.call # Prepare Inc-server - create server @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:configs], :select => @select - $stderr.puts @serv.inspect + LogAn::Logging.debug @serv # Shutdown on signals @sigs[:INT] = @sigs[:TERM] = method( :shutdown) @@ -98,14 +105,14 @@ module LogAn::Inc # Will be called at exit. Will close all opened BDB::Env def at_exit - $stderr.puts :at_exit + LogAn::Logging.info :at_exit @logs and @logs.close @etc and @etc.close end # Shutdown Server cleanly. First shutdown TCPServer. def shutdown signal = nil - $stderr.puts [:signal, signal, Signal[signal]].inspect if signal + LogAn::Logging.info :signal, signal, Signal[signal]] if signal @serv.close exit 0 end From 3b8cae9ca40d640de747a9a503b7b199caa9add4 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:52:34 +0200 Subject: [PATCH 18/45] Logging-Usage fixed --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index a46da51..dd82c01 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -112,7 +112,7 @@ module LogAn::Inc # Shutdown Server cleanly. First shutdown TCPServer. def shutdown signal = nil - LogAn::Logging.info :signal, signal, Signal[signal]] if signal + LogAn::Logging.info :signal, signal, Signal[signal] if signal @serv.close exit 0 end From 19929f069beff5745667ac03083b419fc4c94090 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 31 Mar 2010 00:59:22 +0200 Subject: [PATCH 19/45] Logging extended --- lib/logan/inc/main.rb | 11 ++++++++--- lib/logan/inc/server.rb | 14 ++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index dd82c01..6c12915 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -8,10 +8,15 @@ require 'logan/loglines' require 'logan/cache' module LogAn::Logging - def log lvl, *txt - $stderr.puts( ([Time.now, lvl]+txt).inspect) + class << self + def log lvl, *txt + $stderr.puts( ([Time.now, lvl]+txt).inspect) + end + alias method_missing log + end + def method_missing *paras + self.class.log *paras end - alias method_missing log end module LogAn::Inc diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index 4f03e3b..0440ae3 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -53,13 +53,18 @@ class LogAn::Inc::Select <::Select end class LogAn::Inc::Socket <::Select::Socket + def initialize *p + super( *p) + LogAn::Logging.info :connected, self + end + def event_read sock = @sock, event = :read begin @linebuf += sock.readpartial( @bufsize) rescue EOFError self.event_eof sock - rescue Errno::EPIPE => e - self.event_errno e, sock, event + rescue Errno::EPIPE + self.event_errno $!, sock, event rescue IOError self.event_ioerror sock, event rescue Errno::ECONNRESET => e @@ -73,6 +78,11 @@ class LogAn::Inc::Socket <::Select::Socket event_cmd @linebuf.remove( l) end end + + def close + LogAn::Logging.info :disconnect, self + super + end end class LogAn::Inc::Server < ::Select::Server From 261d0ddc5f780cff4595bbd3c4b3a2f9d2182fac Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 13:45:43 +0200 Subject: [PATCH 20/45] LogAn::AutoValueConvertHash: encode-bug fixed. encode and decode now are methods. --- lib/logan/cache.rb | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 2214d2d..5fabe0d 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -58,26 +58,28 @@ end class LogAn::AutoValueConvertHash include Enumerable - attr_reader :decode, :encode, :source + attr_reader :source def initialize source, encode = nil, each = nil, &decode - @source, @encoder = source, decode.nil? ? encode || Marshal.method( :dump) : nil, - @each = each || source.method( :each) rescue NameError - @decode = decode || Marshal.method( :restore) + @source, @encode = source, decode.nil? ? encode || Marshal.method( :dump) : encode, + @each, @decode = each, decode || Marshal.method( :restore) + @each ||= source.method( :each) rescue NameError + define_method :encode, &@encode if @encode + define_method :decode, &@decode if @decode end def [] k - @decode.call @source[k] + decode @source[k] end def []= k, v - @source[k] = @encode.call v + @source[k] = encode v end def each *paras return Enumerator.new self, :each unless block_given? @each.call *paras do |k, v| - yield k, @decode.call( v) + yield k, decode( v) end end From ff2a4aa04058f1e264381e0a221f8f69081c94e1 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 13:51:25 +0200 Subject: [PATCH 21/45] fileparser: Safebox-usage fixed. --- lib/logan/inc/main.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 6c12915..eb9af42 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -72,8 +72,8 @@ module LogAn::Inc # Open configs begin configs = @conf[:configs] = {} - %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) {|l|l} } - configs[:fileparser] = config( @etc, 'fileparser') {|val| Safebox.run "[#{val}]"} + %w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) {|l| l } } + configs[:fileparser] = config @etc, 'fileparser', &Safebox.method( :eval) LogAn::Inc::SID0.config = configs end From 8989f3c559f36518a3f2e702459796347c77023a Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 13:56:40 +0200 Subject: [PATCH 22/45] define_method -> define_singleton_method --- lib/logan/cache.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 5fabe0d..d14f0a7 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -64,8 +64,8 @@ class LogAn::AutoValueConvertHash @source, @encode = source, decode.nil? ? encode || Marshal.method( :dump) : encode, @each, @decode = each, decode || Marshal.method( :restore) @each ||= source.method( :each) rescue NameError - define_method :encode, &@encode if @encode - define_method :decode, &@decode if @decode + define_singleton_method :encode, &@encode if @encode + define_singleton_method :decode, &@decode if @decode end def [] k From 61c013ea38213c8b90e8d727abef61ebe1a1385d Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:01:16 +0200 Subject: [PATCH 23/45] define_method -> define_singleton_method --- lib/logan/cache.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index d14f0a7..2e7d275 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -66,6 +66,7 @@ class LogAn::AutoValueConvertHash @each ||= source.method( :each) rescue NameError define_singleton_method :encode, &@encode if @encode define_singleton_method :decode, &@decode if @decode + LogAn::Logging.debug encode: @encode, decode: @decode end def [] k From 975769e03db2dc26aecc124b84d9849e7f9f1e72 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:05:29 +0200 Subject: [PATCH 24/45] encode/decode-problems --- lib/logan/cache.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 2e7d275..950725d 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -61,12 +61,12 @@ class LogAn::AutoValueConvertHash attr_reader :source def initialize source, encode = nil, each = nil, &decode - @source, @encode = source, decode.nil? ? encode || Marshal.method( :dump) : encode, + @source, @encode = source, encode || ( decode.nil? && Marshal.method( :dump) ) @each, @decode = each, decode || Marshal.method( :restore) @each ||= source.method( :each) rescue NameError define_singleton_method :encode, &@encode if @encode define_singleton_method :decode, &@decode if @decode - LogAn::Logging.debug encode: @encode, decode: @decode + LogAn::Logging.debug encode: @encode, decode: @decode, each: @each end def [] k From f370dcf52d8efa8ec512f7c1c807b430ed041482 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:07:52 +0200 Subject: [PATCH 25/45] SID0: flatten before pack --- lib/logan/inc/command.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/command.rb b/lib/logan/inc/command.rb index f33044c..ee009ed 100644 --- a/lib/logan/inc/command.rb +++ b/lib/logan/inc/command.rb @@ -38,7 +38,7 @@ module LogAn file = @@config[ :files][ sid] next unless file # command, SID, (inode, seek), file - pc = [1, sid, @@store[ :seeks][ sid], file].pack 'nNNNa*' + pc = [1, sid, @@store[ :seeks][ sid], file].flatten.pack 'nNNNa*' sock.write [pc.length, pc].pack( 'Na*') end end From 49b31f23339b0b8d3a30345251417dc10c324bbd Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:10:40 +0200 Subject: [PATCH 26/45] FileParser::*#event_line -> #event_read --- lib/logan/inc/command.rb | 2 +- lib/logan/inc/server.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/command.rb b/lib/logan/inc/command.rb index ee009ed..2525ce2 100644 --- a/lib/logan/inc/command.rb +++ b/lib/logan/inc/command.rb @@ -7,7 +7,7 @@ module LogAn @sid = sid end - def event_line line, sock + def event_read line, sock cmd, l = line.unpack 'na*' self[cmd].call( l, sock) if self[cmd] end diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index 0440ae3..40ef163 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -109,7 +109,7 @@ class LogAn::Inc::Server < ::Select::Server def event_cmd cmd sid, line = cmd.unpack 'Na*' fp = sid == 0 ? @sid0 : @config[:fileparser][sid] - fp.event_line line, self if fp + fp.event_read line, self if fp end end end From b53a39d991f3622866233e8c44b4f56270e2fa07 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:12:56 +0200 Subject: [PATCH 27/45] FileParser::Line: include not extend B --- lib/logan/inc/fileparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index af56e1c..d0f305b 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -18,7 +18,7 @@ module LogAn end class Line - extend Base + include Base attr_reader :sid, :delimiter, :buffer, :linebuffer def initialize sid, delimiter = nil From 56a9ae930a90d384197d0dbdb70258d7869d16a6 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:15:24 +0200 Subject: [PATCH 28/45] FileParser::Base: setter/getter for logdb and store --- lib/logan/inc/fileparser.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index d0f305b..6e4625d 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -4,7 +4,10 @@ module LogAn module FileParser module Base class < Date: Thu, 1 Apr 2010 14:32:30 +0200 Subject: [PATCH 29/45] FileParser::Base: emit fixed --- lib/logan/inc/fileparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index 6e4625d..7fbe5ab 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -10,7 +10,7 @@ module LogAn def store() @@store end end - def emit v + def emit line @@logdb.push @sid, line end From 4910930b2b9a2e071133438583649af34bc7422a Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:33:53 +0200 Subject: [PATCH 30/45] FileParser::Base: emit fixed --- lib/logan/inc/fileparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index 7fbe5ab..22c49fb 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -11,7 +11,7 @@ module LogAn end def emit line - @@logdb.push @sid, line + @@logdb.emit @sid, line end def seeks read From d3b1d12a129e9d7a19a56a46380dd369f760f140 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:35:24 +0200 Subject: [PATCH 31/45] FileParser::Base: emit fixed --- lib/logan/inc/fileparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index 22c49fb..718f126 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -11,7 +11,7 @@ module LogAn end def emit line - @@logdb.emit @sid, line + @@logdb.emit line, @sid end def seeks read From 9cee5add9e952896b170213af957d7cff604608c Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:38:59 +0200 Subject: [PATCH 32/45] Loglines: Queue enabled and DB-Name-cache --- lib/logan/loglines.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index cdfcdc2..be47f1d 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -64,7 +64,7 @@ module LogAn end def db name - @env[ name.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] + @dbs[name] ||= @env[ name.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] end def sync @@ -77,6 +77,7 @@ module LogAn dat = [sid || 0x10, val].pack 'Na*' name = db_name id db( name)[ id.raw] = dat + @queue.push id.raw end alias emit put From fa8f42940e5df61bd72f3bee683b58be72359650 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 14:42:39 +0200 Subject: [PATCH 33/45] Loglines: Counter; status every 5s with counter. --- lib/logan/inc/main.rb | 2 +- lib/logan/loglines.rb | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index eb9af42..33f65b6 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -90,7 +90,7 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - LogAn::Logging.info @select + LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv @conf[:stores].each{|key, db| db.flush!} end status.call diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index be47f1d..0e2be4c 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -6,7 +6,7 @@ require 'logan' module LogAn class Loglines - attr_reader :env, :rdb, :dbs + attr_reader :env, :rdb, :dbs, :counter def self.new *paras ret = obj = super( *paras) @@ -28,7 +28,7 @@ module LogAn end @rdb = @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] @queue = @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] - @dbs = {} + @dbs, @counter = {}, 0 self.hash_func = lambda {|k| [k.timestamp.to_i/60/60].pack 'N' # Hour-based rotation } @@ -77,6 +77,7 @@ module LogAn dat = [sid || 0x10, val].pack 'Na*' name = db_name id db( name)[ id.raw] = dat + @counter += 1 @queue.push id.raw end alias emit put From b0b9e6e4edf62e33d85c8dbe8ae8069899b6d64b Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 15:37:43 +0200 Subject: [PATCH 34/45] Status: smaller... --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 33f65b6..939cc89 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -90,7 +90,7 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv + LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv.clients @conf[:stores].each{|key, db| db.flush!} end status.call From 5926b94e68e2353957f228be8f911b8beed6e8a0 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 15:38:48 +0200 Subject: [PATCH 35/45] Status: smaller... --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 939cc89..f5db552 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -90,7 +90,7 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv.clients + LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv && @serv.clients @conf[:stores].each{|key, db| db.flush!} end status.call From 10f95500594c68cded727214eea7eaaf43ee615f Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 15:51:15 +0200 Subject: [PATCH 36/45] Status: smaller... --- lib/logan/inc/server.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/server.rb b/lib/logan/inc/server.rb index 40ef163..bb5b42e 100644 --- a/lib/logan/inc/server.rb +++ b/lib/logan/inc/server.rb @@ -86,7 +86,7 @@ class LogAn::Inc::Socket <::Select::Socket end class LogAn::Inc::Server < ::Select::Server - attr_reader :config + attr_reader :config, :clients def init opts super opts From f562f25d82921567d52a04f90b12b4d5f779bedb Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 15:57:18 +0200 Subject: [PATCH 37/45] Status: smarter... --- lib/logan/inc/main.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index f5db552..7e4b3ed 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -90,7 +90,7 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv && @serv.clients + LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv && @serv.clients.map{|cl|cl.sock.peeraddr} @conf[:stores].each{|key, db| db.flush!} end status.call From bc097f8532d1fe67ea0870d19ce817718d2b3ca0 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 1 Apr 2010 23:32:24 +0200 Subject: [PATCH 38/45] Status: smarter... --- lib/logan/inc/main.rb | 10 +++++++--- lib/logan/loglines.rb | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 7e4b3ed..c3a6a61 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -81,7 +81,9 @@ module LogAn::Inc begin stores = @conf[:stores] = {} db = @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT] - db = LogAn::AutoValueConvertHash.new( db, lambda {|val| val.pack( 'NN') }) {|val| (val||0.chr*8).unpack( 'NN') } + db = LogAn::AutoValueConvertHash.new( db, lambda {|val| val.pack( 'NN') }) {|val| + (val||0.chr*8).unpack( 'NN') + } stores[:seeks] = LogAn::Cache.new db LogAn::Inc::FileParser::Base.store = LogAn::Inc::SID0.store = stores end @@ -90,13 +92,15 @@ module LogAn::Inc @select = LogAn::Inc::Select.new status = lambda do @select.at Time.now+5, &status - LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv && @serv.clients.map{|cl|cl.sock.peeraddr} + LogAn::Logging.info :recv_lines => @logs.counter, + :connections => @serv && @serv.clients.map {|cl| cl.sid } @conf[:stores].each{|key, db| db.flush!} end status.call # Prepare Inc-server - create server - @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:configs], :select => @select + @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), + :config => @conf[:configs], :select => @select LogAn::Logging.debug @serv # Shutdown on signals diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index 0e2be4c..6611e7c 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -27,7 +27,8 @@ module LogAn else env end @rdb = @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] - @queue = @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] + @queue = @env[ "newids.queue", :type => SBDB::Queue, + :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] @dbs, @counter = {}, 0 self.hash_func = lambda {|k| [k.timestamp.to_i/60/60].pack 'N' # Hour-based rotation @@ -64,7 +65,8 @@ module LogAn end def db name - @dbs[name] ||= @env[ name.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] + @dbs[name] ||= @env[ name.to_s, :type => SBDB::Btree, + :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] end def sync From b7d6689225c670bc15758062fe548b4bf3d525c1 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Fri, 2 Apr 2010 00:37:48 +0200 Subject: [PATCH 39/45] Status: ready --- lib/logan/inc/main.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index c3a6a61..f19a5f8 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -93,8 +93,8 @@ module LogAn::Inc status = lambda do @select.at Time.now+5, &status LogAn::Logging.info :recv_lines => @logs.counter, - :connections => @serv && @serv.clients.map {|cl| cl.sid } - @conf[:stores].each{|key, db| db.flush!} + :connections => @serv && @serv.clients.map {|cl| cl.sock.peeraddr } + @conf[:stores].each {|key, db| db.flush!} end status.call From 7a244cd4653be5bc839cb2d9aebaaed11f248afa Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Fri, 2 Apr 2010 16:49:45 +0200 Subject: [PATCH 40/45] LogAn::Inc::Fileparser::Line#event_line added and modified #event_read --- lib/logan/inc/fileparser.rb | 27 +++++++++++++-------------- lib/logan/inc/main.rb | 19 ++++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/logan/inc/fileparser.rb b/lib/logan/inc/fileparser.rb index 718f126..9440e8b 100644 --- a/lib/logan/inc/fileparser.rb +++ b/lib/logan/inc/fileparser.rb @@ -30,12 +30,14 @@ module LogAn @buffer, @linebuffer = Select::Buffer.new( ''), Select::Buffer.new( '') end - def event_read str, sock + def event_read str, sock = nil @buffer += str - @buffer.each! @delimiter do |line| - emit line - seeks line.length - end + @buffer.each! @delimiter, &method( :event_line) + end + + def event_line line + emit line + seeks line.length end end @@ -45,15 +47,12 @@ module LogAn @multiline = multiline || /^\d\d-\d\d-\d\d:/ end - def event_read str, sock - @buffer += str - @buffer.each! @delimiter do |line| - if line =~ @multiline - emit @linebuffer.to_s - seeks @linebuffer.length - @linebuffer.replace line - else @linebuffer += line - end + def event_line line + if line =~ @multiline + emit @linebuffer.to_s + seeks @linebuffer.length + @linebuffer.replace line + else @linebuffer += line end end end diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index f19a5f8..6a92b84 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -51,9 +51,9 @@ module LogAn::Inc # : Server-Configuration. default { port: 1087 } def initialize conf super - @conf = {} # Copy config - changes possible + @conf = {} conf.each {|key, val| @conf[key]= val } # Default directories @@ -90,13 +90,7 @@ module LogAn::Inc # Select-framework @select = LogAn::Inc::Select.new - status = lambda do - @select.at Time.now+5, &status - LogAn::Logging.info :recv_lines => @logs.counter, - :connections => @serv && @serv.clients.map {|cl| cl.sock.peeraddr } - @conf[:stores].each {|key, db| db.flush!} - end - status.call + status # Init Status # Prepare Inc-server - create server @serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), @@ -112,6 +106,13 @@ module LogAn::Inc raise $! end + def status + LogAn::Logging.info :recv_lines => @logs.counter, + :connections => @serv && @serv.clients.map {|cl| cl.sock.peeraddr } + @conf[ :stores].each {|key, db| db.flush!} + @select.at Time.now+5, &status + end + # Will be called at exit. Will close all opened BDB::Env def at_exit LogAn::Logging.info :at_exit @@ -121,7 +122,7 @@ module LogAn::Inc # Shutdown Server cleanly. First shutdown TCPServer. def shutdown signal = nil - LogAn::Logging.info :signal, signal, Signal[signal] if signal + LogAn::Logging.info :signal, signal, Signal[ signal] if signal @serv.close exit 0 end From 338daed210f874bf1820a8287ac7b5b2c590883a Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Sat, 3 Apr 2010 13:04:47 +0200 Subject: [PATCH 41/45] status: fixed --- lib/logan/inc/main.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logan/inc/main.rb b/lib/logan/inc/main.rb index 6a92b84..0722934 100644 --- a/lib/logan/inc/main.rb +++ b/lib/logan/inc/main.rb @@ -90,6 +90,7 @@ module LogAn::Inc # Select-framework @select = LogAn::Inc::Select.new + @status = method :status status # Init Status # Prepare Inc-server - create server @@ -107,10 +108,10 @@ module LogAn::Inc end def status + @select.at Time.now+5, &@status LogAn::Logging.info :recv_lines => @logs.counter, :connections => @serv && @serv.clients.map {|cl| cl.sock.peeraddr } @conf[ :stores].each {|key, db| db.flush!} - @select.at Time.now+5, &status end # Will be called at exit. Will close all opened BDB::Env From 7aa766163a1a67f565289891b4bc4f77b2449eaa Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Sat, 3 Apr 2010 20:52:57 +0200 Subject: [PATCH 42/45] Initialize Analyse-Console --- bin/logansh | 12 +++++++++++ lib/logan/analyse.rb | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100755 bin/logansh create mode 100644 lib/logan/analyse.rb diff --git a/bin/logansh b/bin/logansh new file mode 100755 index 0000000..7c8cd72 --- /dev/null +++ b/bin/logansh @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby + +require 'logan/analyse' +require 'active_support' +require 'irb' + +$logan = LogAn::Analyse.new 'logs' +begin + IRB.start __FILE__ +ensure + $logan.close +end diff --git a/lib/logan/analyse.rb b/lib/logan/analyse.rb new file mode 100644 index 0000000..8c42404 --- /dev/null +++ b/lib/logan/analyse.rb @@ -0,0 +1,49 @@ + +require 'logan/loglines' +require 'time' + +class LogAn::Analyse + attr_reader :lines + + def close + @lines.close + end + + def initialize lines + @lines = String === lines ? LogAn::Loglines.new( lines) : lines + end + + def extremum val + val = case val + when String then Time.parse val + when Integer then Time.at val + when Time then val + else raise ArgumentError, "Unknwon type: #{val}", caller[ 1..-1] + end + end + + def timerange min, max = nil + exend = false + min, max, exend = min.min, min.max, min.exclude_end? if Range === min + Range.new extremum( min), extremum( max), exend + end + + def dbs min, max = nil, &exe + return Enumerator.new( self, :dbs, min, max) unless exe + range = timerange min, max + @lines.rdb.each &exe + end + + def search min, max = nil, &exe + dbs = @lines.dbs + range = timerange min, max + @lines.rdb.each do |time, db| + dbs[ UUIDTools::UUID.parse_raw( db)].each &exe if range === Time.at( *time.unpack( 'N')) + end + end + alias [] search + + def each min, max = nil, &exe + exe ? search( min, max, &exe) : Enumerator.new( self, :search, min, max) + end +end From a819b9b0e8ed5c1294c2c617039622efc1befb7f Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Sat, 3 Apr 2010 21:03:51 +0200 Subject: [PATCH 43/45] New Hash-function! UNIX-Timestamps as hash-values. Incompatible to older databases! --- lib/logan/loglines.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index 6611e7c..7a5e7dc 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -31,7 +31,8 @@ module LogAn :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] @dbs, @counter = {}, 0 self.hash_func = lambda {|k| - [k.timestamp.to_i/60/60].pack 'N' # Hour-based rotation + n = k.timestamp.to_i + [n - (n % 3600)].pack 'N' # Hour-based rotation } end From b76953f57d6b86e1ef19d5cd58bb17958dc91c12 Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Thu, 8 Apr 2010 13:13:43 +0200 Subject: [PATCH 44/45] AutoKeyConvertHash added --- lib/logan/analyse.rb | 4 +++- lib/logan/cache.rb | 41 +++++++++++++++++++++++++++++++++++++++++ lib/logan/loglines.rb | 10 +++++++--- 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/lib/logan/analyse.rb b/lib/logan/analyse.rb index 8c42404..cbf366b 100644 --- a/lib/logan/analyse.rb +++ b/lib/logan/analyse.rb @@ -31,7 +31,9 @@ class LogAn::Analyse def dbs min, max = nil, &exe return Enumerator.new( self, :dbs, min, max) unless exe range = timerange min, max - @lines.rdb.each &exe + @lines.rdb.each do |time, db| + exe.call db + end end def search min, max = nil, &exe diff --git a/lib/logan/cache.rb b/lib/logan/cache.rb index 950725d..bfd13b8 100644 --- a/lib/logan/cache.rb +++ b/lib/logan/cache.rb @@ -9,6 +9,10 @@ class LogAn::Cache @source, @data, self.type = source, data || {}, type end + def close + @source.close + end + def flush! @data.each {|k,v| @source[k] = v } @data = {} @@ -69,6 +73,10 @@ class LogAn::AutoValueConvertHash LogAn::Logging.debug encode: @encode, decode: @decode, each: @each end + def close + @source.close + end + def [] k decode @source[k] end @@ -91,3 +99,36 @@ class LogAn::AutoValueConvertHash end end end + +class LogAn::AutoKeyConvertHash + include Enumerable + attr_reader :source + + def initialize source, encode = nil, each = nil, &decode + @source, @encode = source, encode || ( decode.nil? && Marshal.method( :dump) ) + @each, @decode = each, decode || Marshal.method( :restore) + @each ||= source.method( :each) rescue NameError + define_singleton_method :encode, &@encode if @encode + define_singleton_method :decode, &@decode if @decode + LogAn::Logging.debug encode: @encode, decode: @decode, each: @each + end + + def close + @source.close + end + + def [] k + @source[ encode( k)] + end + + def []= k, v + @source[ encode( k)] = v + end + + def each *paras + return Enumerator.new self, :each unless block_given? + @each.call *paras do |k, v| + yield decode( k), v + end + end +end diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index 7a5e7dc..28bc18f 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -26,13 +26,17 @@ module LogAn flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL else env end - @rdb = @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT] + @rdb = AutoValueConvertHash.new( + AutoKeyConvertHash.new( + @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT], + lambda {|key| [key.to_i].pack 'N' }) {|key| Time.at key.unpack( 'N') }, + lambda {|val| String === val ? val : val.raw } {|val| val && UUIDTools::UUID.parse_raw( val) } @queue = @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] - @dbs, @counter = {}, 0 + @dbs, @counter = Cache.new, 0 self.hash_func = lambda {|k| n = k.timestamp.to_i - [n - (n % 3600)].pack 'N' # Hour-based rotation + n -= n % 3600 } end From 76a05892db33080392054751b9cfad641599f90e Mon Sep 17 00:00:00 2001 From: Denis Knauf Date: Wed, 14 Apr 2010 11:36:25 +0200 Subject: [PATCH 45/45] Inspector added (loganinc --inspector) --- bin/loganinc | 12 ++++++++++++ lib/logan/loglines.rb | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/bin/loganinc b/bin/loganinc index 94edc3d..18563dd 100755 --- a/bin/loganinc +++ b/bin/loganinc @@ -4,6 +4,7 @@ require 'logan' require 'logan/inc' opts = {} +opts[:inspector] = ARGV[0] == '--inspector' ? ARGV.shift : false opts[:server] = if ARGV[1] ARGV elsif ARGV[0] @@ -14,6 +15,17 @@ opts[:server][1] = opts[:server][1].to_i logan = LogAn::Inc::Main.new opts begin + logan.instance_eval do + @inspector_server = UNIXServer.new 'loganinc.inspector.sock' + Thread.new do + loop do + sock = @inspector_server.accept + sock.each_line do |line| + sock.puts eval( line).inspect + end + end + end + end if opts[:inspector] logan.main rescue Object logan.at_exit diff --git a/lib/logan/loglines.rb b/lib/logan/loglines.rb index 28bc18f..2f85d78 100644 --- a/lib/logan/loglines.rb +++ b/lib/logan/loglines.rb @@ -30,10 +30,10 @@ module LogAn AutoKeyConvertHash.new( @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT], lambda {|key| [key.to_i].pack 'N' }) {|key| Time.at key.unpack( 'N') }, - lambda {|val| String === val ? val : val.raw } {|val| val && UUIDTools::UUID.parse_raw( val) } + lambda {|val| String === val ? val : val.raw }) {|val| val && UUIDTools::UUID.parse_raw( val) } @queue = @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16] - @dbs, @counter = Cache.new, 0 + @dbs, @counter = {}, 0 self.hash_func = lambda {|k| n = k.timestamp.to_i n -= n % 3600