bin/box.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/ruby
+
+require 'thread'
+
+class Box
+	attr_reader :_, :emited
+	alias db emited
+	alias persistent emited
+	attr_accessor :emited
+
+	def initialize db, _
+		@_, @emited = _, db
+	end
+
+	def emit k, v
+		@emited[k] = v
+	end
+
+	def do code
+		instance_eval code, self.class.to_s, 0
+	end
+end
+
+require 'sbdb'
+
+class Persistent
+	include Enumerable
+	def initialize( db)  @db, @cursor = db, db.cursor  end
+	def emit( k, v)  @db[k] = v  end
+	alias push emit
+	alias put emit
+	alias []= emit
+	def get( k)  @db[k]  end
+	alias [] get
+	alias fetch get
+	def inspect() "#<%s:0x%016x>" % [ self.class, self.object_id ] end
+	def each &e
+		e ? @cursor.each( &e) : Enumerator.new( self, :each)
+	end
+	def to_hash
+		h = {}
+		each {|k, v| h[ k] = v }
+		h
+	end
+end
+
+#Persistent.freeze
+
+r = nil
+Dir.mkdir 'logs' rescue Errno::EEXIST
+SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs|
+	db = Persistent.new logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE]
+	$stdout.print "(0)$ "
+	STDIN.each_with_index do |l, i|
+		r = Thread.new do
+				l.untaint
+				$SAFE = 4
+				b = Box.new db, r
+				begin
+					b.do( l)
+				rescue Object
+					$!
+				end
+			end.value
+		$stdout.print "=> #{r.inspect}\n(#{i+1})$ "
+	end
+end

bin/box2.rb

@@ -0,0 +1,87 @@
+#!/usr/bin/ruby
+
+require 'sbdb'
+
+module Sandbox
+	def self.run *paras, &exe
+		exe = paras.shift  unless exe
+		box = paras.shift || Class
+		Thread.new do
+			$SAFE = 4
+			this = box.new *paras
+			begin
+				[:value, this.instance_eval( exe, "Sandbox")]
+			rescue Object
+				[:exception, $!]
+			end
+		end.value
+	end
+
+	def self.create_class *paras, &exe
+		exe = paras.shift  unless exe
+		run Class, *paras do
+			eval exe
+			self
+		end
+	end
+	alias new_class create_class
+end
+
+class Box
+	attr_reader :_, :db
+
+	def initialize db, _
+		@_, @db = _, db
+	end
+
+	def put( key, val)  @db[key] = val  end
+	def get( key)  @db[key]  end
+end
+
+class ClassBuilder
+end
+
+class Emit
+	def initialize( db)  @db = db  end
+	def emit( key, val)  @db[key] = val  end
+	def inspect()  "#<%s:0x%016x>" % [ self.class, self.object_id ]  end
+end
+
+class Persistent < Emit
+	include Enumerable
+	def initialize db, cursor
+		super db
+		@cursor = cursor
+	end
+	alias put emit
+	alias []= emit
+	def get( key)  @db[key]  end
+	alias [] get
+	alias fetch get
+	def each &exe
+		exe ? @cursor.each( &exe) : Enumerator.new( self, :each)
+	end
+	def to_hash
+		rh = {}
+		each {|key, val| rh[ key] = val }
+		rh
+	end
+end
+
+_ = nil
+Dir.mkdir 'logs' rescue Errno::EEXIST
+SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs|
+	db = logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE]
+	db = Persistent.new db, db.cursor
+	$stdout.print "(0)$ "
+	STDIN.each_with_index do |line, i|
+		ret = Sandbox.run line, Box, db, _
+		if :value == ret.first
+			_ = ret.last
+			$stdout.puts "=> #{ret.last.inspect}"
+		else
+			$stdout.puts ret.last.inspect
+		end
+		$stdout.print "(#{i+1})$ "
+	end
+end

bin/box3.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/ruby
+
+require 'sbdb'
+require 'safebox'
+
+_ = nil
+Dir.mkdir 'logs' rescue Errno::EEXIST
+SBDB::Env.new 'logs', SBDB::CREATE | SBDB::Env::INIT_TRANSACTION do |logs|
+	db = logs['test', :type => SBDB::Btree, :flags => SBDB::CREATE]
+	db = Safebox::Persistent.new db, db.cursor
+	$stdout.print "(0)$ "
+	STDIN.each_with_index do |line, i|
+		ret = Safebox.run line, Safebox::Box, db, _
+		if :value == ret.first
+			_ = ret.last
+			$stdout.puts "=> #{ret.last.inspect}"
+		else
+			$stdout.puts ret.last.inspect
+		end
+		$stdout.print "(#{i+1})$ "
+	end
+end

bin/loganinc

@@ -4,7 +4,6 @@ require 'logan'
 require 'logan/inc'
 
 opts = {}
-opts[:inspector] = ARGV[0] == '--inspector' ? ARGV.shift : false 
 opts[:server] = if ARGV[1]
 		ARGV
 	elsif ARGV[0]
@@ -15,17 +14,6 @@ opts[:server][1] = opts[:server][1].to_i
 
 logan = LogAn::Inc::Main.new opts
 begin
-	logan.instance_eval do
-		@inspector_server = UNIXServer.new 'loganinc.inspector.sock'
-		Thread.new do
-			loop do
-				sock = @inspector_server.accept
-				sock.each_line do |line|
-					sock.puts eval( line).inspect
-				end
-			end
-		end
-	end  if opts[:inspector]
 	logan.main
 rescue Object
 	logan.at_exit

bin/logansh

@@ -1,12 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'logan/analyse'
-require 'active_support'
-require 'irb'
-
-$logan = LogAn::Analyse.new 'logs'
-begin
-	IRB.start __FILE__
-ensure
-	$logan.close
-end

lib/logan/analyse.rb

@@ -1,51 +0,0 @@
-
-require 'logan/loglines'
-require 'time'
-
-class LogAn::Analyse
-	attr_reader :lines
-
-	def close
-		@lines.close
-	end
-
-	def initialize lines
-		@lines = String === lines ? LogAn::Loglines.new( lines) : lines
-	end
-
-	def extremum val
-		val = case val
-			when String  then Time.parse val
-			when Integer then Time.at val
-			when Time    then val
-			else raise ArgumentError, "Unknwon type: #{val}", caller[ 1..-1]
-			end
-	end
-
-	def timerange min, max = nil
-		exend = false
-		min, max, exend = min.min, min.max, min.exclude_end?  if Range === min
-		Range.new extremum( min), extremum( max), exend
-	end
-
-	def dbs min, max = nil, &exe
-		return Enumerator.new( self, :dbs, min, max)  unless exe
-		range = timerange min, max
-		@lines.rdb.each do |time, db|
-			exe.call db
-		end
-	end
-
-	def search min, max = nil, &exe
-		dbs = @lines.dbs
-		range = timerange min, max
-		@lines.rdb.each do |time, db|
-			dbs[ UUIDTools::UUID.parse_raw( db)].each &exe  if range === Time.at( *time.unpack( 'N'))
-		end
-	end
-	alias [] search
-
-	def each min, max = nil, &exe
-		exe ? search( min, max, &exe) : Enumerator.new( self, :search, min, max)
-	end
-end

lib/logan/cache.rb

@@ -9,21 +9,17 @@ class LogAn::Cache
 		@source, @data, self.type = source, data || {}, type
 	end
 
-	def close
-		@source.close
-	end
-
 	def flush!
-		@data.each {|k,v| @source[k] = v }
+		@data.each {|k,v| @obj[k] = v }
 		@data = {}
 	end
 
 	def dget k
-		@data[k] ||= @source[k]
+		@data[k] ||= @obj[k]
 	end
 
 	def oget k
-		@data[k] || @source[k]
+		@data[k] || @obj[k]
 	end
 
 	def dset k, v
@@ -31,7 +27,7 @@ class LogAn::Cache
 	end
 
 	def oset k, v
-		@source[k] = v
+		@obj[k] = v
 	end
 
 	def type= type
@@ -46,89 +42,38 @@ class LogAn::Cache
 
 	def write_cache= type
 		@type &= ~ (type ? 0 : 2)
-		define_singleton_method :[]=, method( type ? :oset : :dset)
+		define_singleton_method :[], method( type ? :oset : :dset)
 	end
 
-	include Enumerable
+	#include Enumerable
-	def each *paras
+	#def each &e
-		return Enumerator.new self, :each  unless block_given?
+		#return Enumerator.new self, :each  unless e
-		flush!  if @type&2 == 2
+		#flush!
-		@source.each_keys( *paras) do |key|
+		#@obj.each &e
-			yield key, self[key]
+		#self
-		end
+	#end
-		self
-	end
 end
 
 class LogAn::AutoValueConvertHash
 	include Enumerable
-	attr_reader :source
 
-	def initialize source, encode = nil, each = nil, &decode
+	def initialize obj, encode = nil, each = nil, &decode
-		@source, @encode = source, encode || ( decode.nil? && Marshal.method( :dump) )
+		@object, @encoder = obj, decode.nil? ? encode || Marshal.method( :dump) : nil,
-		@each, @decode = each, decode || Marshal.method( :restore)
+		@each = each || obj.method( :each)  rescue NameError
-		@each ||= source.method( :each)  rescue NameError
+		@decode = decode || Marshal.method( :restore)
-		define_singleton_method :encode, &@encode  if @encode
-		define_singleton_method :decode, &@decode  if @decode
-		LogAn::Logging.debug encode: @encode, decode: @decode, each: @each
-	end
-
-	def close
-		@source.close
 	end
 
 	def [] k
-		decode @source[k]
+		decode.call @object[k]
 	end
 
 	def []= k, v
-		@source[k] = encode v
+		@object[k] = encode.call v
 	end
 
 	def each *paras
-		return Enumerator.new self, :each  unless block_given?
 		@each.call *paras do |k, v|
 			yield k, decode( v)
 		end
 	end
-
-	def each_keys *paras
-		return Enumerator.new self, :each_keys  unless block_given?
-		@each.call *paras do |k, v|
-			yield k
-		end
-	end
-end
-
-class LogAn::AutoKeyConvertHash
-	include Enumerable
-	attr_reader :source
-
-	def initialize source, encode = nil, each = nil, &decode
-		@source, @encode = source, encode || ( decode.nil? && Marshal.method( :dump) )
-		@each, @decode = each, decode || Marshal.method( :restore)
-		@each ||= source.method( :each)  rescue NameError
-		define_singleton_method :encode, &@encode  if @encode
-		define_singleton_method :decode, &@decode  if @decode
-		LogAn::Logging.debug encode: @encode, decode: @decode, each: @each
-	end
-
-	def close
-		@source.close
-	end
-
-	def [] k
-		@source[ encode( k)]
-	end
-
-	def []= k, v
-		@source[ encode( k)] = v
-	end
-
-	def each *paras
-		return Enumerator.new self, :each  unless block_given?
-		@each.call *paras do |k, v|
-			yield decode( k), v
-		end
-	end
 end

lib/logan/inc/command.rb

@@ -14,31 +14,24 @@ module LogAn
 		end
 
 		class SID0 < Command
-			class <<self
+			def initialize store, config, sid = 0
-				def config=( db)  @@config = db  end
+				@store, @config = store, config
-				def config()  @@config  end
-				def store=( db)  @@store = db  end
-				def store()  @@store  end
-			end
-
-			def initialize sid = 0
-				super sid
 				self[9] = method :event_hostname
 				self[10] = method :event_filerotated
 			end
 
 			def event_filerotated line, sock
-				sid, inode, seek = line.unpack 'NNN'
+				sid, d = line.unpack 'Na8'
-				@@store[ :seeks][ sid] = [inode, seek]
+				@store[ :seeks, sid] = d
 			end
 
 			def event_hostname line, sock
-				@@config[ :hosts].each do |sid, host|
+				@config[ :hosts].each do |sid,host|
 					next  unless line == host
-					file = @@config[ :files][ sid]
+					file = @config[ :files, sid]
 					next  unless file
 					# command, SID, (inode, seek), file
-					pc = [1, sid, @@store[ :seeks][ sid], file].flatten.pack 'nNNNa*'
+					pc = [1, sid, @store[ :seeks, sid] || "\x00\x00\x00\x00\x00\x00\x00\x00", file].pack 'nNa8a*'
 					sock.write [pc.length, pc].pack( 'Na*')
 				end
 			end

lib/logan/inc/fileparser.rb

@@ -4,25 +4,27 @@ module LogAn
 		module FileParser
 			module Base
 				class <<self
-					def logdb=( var)  @@logdb = var  end
+					attr_accessor :logdb, :store
-					def logdb()  @@logdb  end
-					def store=( var)  @@store = var  end
-					def store()  @@store  end
 				end
 
-				def emit line
+				def emit v
-					@@logdb.emit line, @sid
+					@@logdb.push @sid, line
 				end
 
 				def seeks read
-					inode, seek = @@store[ :seeks][@sid]
+					inode, seek = (@@store[ :seeks, @sid] || "\0\0\0\0\0\0\0\0").unpack 'a4N'
-					@@store[ :seeks][@sid] = [inode, read + seek]
+					@@store[ :seeks, @sid] = [inode, read + seek].pack( 'a4N')
 				end
 			end
 
 			class Line
-				include Base
+				extend Base
 				attr_reader :sid, :delimiter, :buffer, :linebuffer
+				@@fileparser = []
+
+				def self.[] sid
+					@@fileparser[sid] ||= self.new sid
+				end
 
 				def initialize sid, delimiter = nil
 					@sid, @delimiter = sid, delimiter || "\n"
@@ -30,16 +32,14 @@ module LogAn
 					@buffer, @linebuffer = Select::Buffer.new( ''), Select::Buffer.new( '')
 				end
 
-				def event_read str, sock = nil
+				def event_read str, sock
 					@buffer += str
-					@buffer.each! @delimiter, &method( :event_line)
+					@buffer.each! @delimiter do |line|
-				end
-
-				def event_line line
 						emit line
 						seeks line.length
 					end
 				end
+			end
 
 			class Multiline < Line
 				def initialize sid, delimiter = nil, multiline = nil
@@ -47,7 +47,9 @@ module LogAn
 					@multiline = multiline || /^\d\d-\d\d-\d\d:/
 				end
 
-				def event_line line
+				def event_read str, sock
+					@buffer += str
+					@buffer.each! @delimiter do |line|
 						if line =~ @multiline
 							emit @linebuffer.to_s
 							seeks @linebuffer.length
@@ -59,3 +61,4 @@ module LogAn
 			end
 		end
 	end
+end

lib/logan/inc/main.rb

@@ -7,39 +7,18 @@ require 'logan/inc'
 require 'logan/loglines'
 require 'logan/cache'
 
-module LogAn::Logging
-	class  << self
-		def log lvl, *txt
-			$stderr.puts( ([Time.now, lvl]+txt).inspect)
-		end
-		alias method_missing log
-	end
-	def method_missing *paras
-		self.class.log *paras
-	end
-end
-
 module LogAn::Inc
 	class Main < RobustServer
-		def cache ret, type, &e
+		# Open Config.
+		def config env, db, type = nil, flags = nil
+			$stderr.puts "Open Database \"sids.cnf\" #{db.inspect} (#{type.inspect})"
 			type ||= 1+4
-			ret = LogAn::AutoValueConvertHash.new ret, &e  if type&4 > 0 or e
+			ret = env[ 'sids.cnf', db, :flags => flags || SBDB::RDONLY]
+			ret = LogAn::AutoValueConvertHash.new ret  if type&4 > 0
 			ret = LogAn::Cache.new ret, type&3  if type&3 > 0
 			ret
 		end
 
-		# Open Store.
-		def store env, db, type = nil, flags = nil, &e
-			LogAn::Logging.info :store, :open, "sids.cnf", db, type
-			cache env[ 'sids.cnf', db, :flags => flags || SBDB::CREATE | SBDB::AUTO_COMMIT], type, &e
-		end
-
-		# Open Config.
-		def config env, db, type = nil, flags = nil, &e
-			LogAn::Logging.info :config, :open, "sids.cnf", db, type
-			cache env[ 'sids.cnf', db, :flags => flags || SBDB::RDONLY], type, &e
-		end
-
 		# Prepare Server.
 		#
 		# * conf:
@@ -51,79 +30,44 @@ module LogAn::Inc
 		#    : Server-Configuration. default { port: 1087 }
 		def initialize conf
 			super
-
-			# Copy config - changes possible
 			@conf = {}
+			# Copy config - changes possible
 			conf.each {|key, val| @conf[key]= val }
-
 			# Default directories
 			%w[logs etc].each {|key| @conf[key.to_sym] = key }
-
 			# Open Loglines-databases
 			@logs = LogAn::Loglines.new @conf[:logs]
-			LogAn::Inc::FileParser::Base.logdb = @logs
-
 			# Open config-databases
 			Dir.mkdir @conf[:etc]  rescue Errno::EEXIST
 			@etc = SBDB::Env.new( @conf[:etc],
 					log_config: SBDB::Env::LOG_IN_MEMORY | SBDB::Env::LOG_AUTO_REMOVE,
 					flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL)
-
+			# Set inc-config - stored in etc/inc.cnf
-			# Open configs
+			@conf[:inc] = {}
-			begin
+			%w[hosts files fileparser].each {|key| @conf[:inc][key.to_sym] = config( @etc, key) }
-				configs = @conf[:configs] = {}
+			@store = LogAn::Cache.new LogAn::AutoValueConvertHash.new( @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT]), 3
-				%w[hosts files].each {|key| configs[key.to_sym] = config( @etc, key) {|l| l } }
-				configs[:fileparser] = config @etc, 'fileparser', &Safebox.method( :eval)
-				LogAn::Inc::SID0.config = configs
-			end
-
-			# Open seeks-database
-			begin
-				stores = @conf[:stores] = {}
-				db = @etc[ 'sids.store', 'seeks', SBDB::Recno, SBDB::CREATE | SBDB::AUTO_COMMIT]
-				db = LogAn::AutoValueConvertHash.new( db, lambda {|val| val.pack( 'NN') }) {|val|
-						(val||0.chr*8).unpack( 'NN')
-					}
-				stores[:seeks] = LogAn::Cache.new db
-				LogAn::Inc::FileParser::Base.store = LogAn::Inc::SID0.store = stores
-			end
-
-			# Select-framework
-			@select = LogAn::Inc::Select.new
-			@status = method :status
-			status  # Init Status
-
 			# Prepare Inc-server - create server
-			@serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]),
+			LogAn::Inc::FileParser::Base.logdb = @logs
-					:config => @conf[:configs], :select => @select
+			LogAn::Inc::FileParser::Base.store = @store
-			LogAn::Logging.debug @serv
+			@serv = LogAn::Inc::Server.new :sock => TCPServer.new( *@conf[:server]), :config => @conf[:inc]
-
 			# Shutdown on signals
 			@sigs[:INT] = @sigs[:TERM] = method( :shutdown)
-
 		rescue Object
 			# It's better to close everything, because BDB doesn't like unexpected exits
 			self.at_exit
 			raise $!
 		end
 
-		def status
-			@select.at Time.now+5, &@status
-			LogAn::Logging.info :recv_lines => @logs.counter,
-					:connections => @serv && @serv.clients.map {|cl| cl.sock.peeraddr }
-			@conf[ :stores].each {|key, db| db.flush!}
-		end
-
 		# Will be called at exit.  Will close all opened BDB::Env
 		def at_exit
-			LogAn::Logging.info :at_exit
+			$stderr.puts :at_exit
 			@logs and @logs.close
 			@etc and @etc.close
 		end
 
 		# Shutdown Server cleanly. First shutdown TCPServer.
 		def shutdown signal = nil
-			LogAn::Logging.info :signal, signal, Signal[ signal]  if signal
+			$stderr.puts [:signal, signal, Signal[signal]].inspect  if signal
 			@serv.close
 			exit 0
 		end

lib/logan/inc/server.rb

@@ -53,18 +53,13 @@ class LogAn::Inc::Select <::Select
 end
 
 class LogAn::Inc::Socket <::Select::Socket
-	def initialize *p
-		super( *p)
-		LogAn::Logging.info :connected, self
-	end
-
 	def event_read sock = @sock, event = :read
 		begin
 			@linebuf += sock.readpartial( @bufsize)
 		rescue EOFError
 			self.event_eof sock
-		rescue Errno::EPIPE
+		rescue Errno::EPIPE => e
-			self.event_errno $!, sock, event
+			self.event_errno e, sock, event
 		rescue IOError
 			self.event_ioerror sock, event
 		rescue Errno::ECONNRESET => e
@@ -78,15 +73,10 @@ class LogAn::Inc::Socket <::Select::Socket
 			event_cmd @linebuf.remove( l)
 		end
 	end
-
-	def close
-		LogAn::Logging.info :disconnect, self
-		super
-	end
 end
 
 class LogAn::Inc::Server < ::Select::Server
-	attr_reader :config, :clients
+	attr_reader :config
 
 	def init opts
 		super opts
@@ -102,14 +92,13 @@ class LogAn::Inc::Server < ::Select::Server
 
 		def init opts
 			super opts
-			@sid0 = LogAn::Inc::SID0.new
 			@config = opts[:config] or raise( ArgumentError, "#{self.class} needs a Config!")
 		end
 		
 		def event_cmd cmd
 			sid, line = cmd.unpack 'Na*'
-			fp = sid == 0 ? @sid0 : @config[:fileparser][sid]
+			fileparser = @config[:fileparser][sid]
-			fp.event_read line, self  if fp
+			fileparser.event_line line, self  if fileparser
 		end
 	end
 end

lib/logan/loglines.rb

@@ -6,7 +6,7 @@ require 'logan'
 
 module LogAn
 	class Loglines
-		attr_reader :env, :rdb, :dbs, :counter
+		attr_reader :env, :rdb, :dbs
 
 		def self.new *paras
 			ret = obj = super( *paras)
@@ -26,17 +26,11 @@ module LogAn
 						flags: SBDB::CREATE | SBDB::Env::INIT_TXN | Bdb::DB_INIT_MPOOL
 				else env
 				end
-			@rdb = AutoValueConvertHash.new(
+			@rdb = @env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT]
-					AutoKeyConvertHash.new(
+			@queue = @env[ "newids.queue", :type => SBDB::Queue, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16]
-						@env[ 'rotates.db', :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT],
+			@dbs = {}
-						lambda {|key| [key.to_i].pack 'N' }) {|key| Time.at key.unpack( 'N') },
-					lambda {|val| String === val ? val : val.raw }) {|val| val && UUIDTools::UUID.parse_raw( val) }
-			@queue = @env[ "newids.queue", :type => SBDB::Queue,
-					:flags => SBDB::CREATE | SBDB::AUTO_COMMIT, :re_len => 16]
-			@dbs, @counter = {}, 0
 			self.hash_func = lambda {|k|
-				n = k.timestamp.to_i
+				[k.timestamp.to_i/60/60].pack 'N'  # Hour-based rotation
-				n -= n % 3600
 			}
 		end
 
@@ -70,8 +64,7 @@ module LogAn
 		end
 
 		def db name
-			@dbs[name] ||= @env[ name.to_s, :type => SBDB::Btree,
+			@env[ name.to_s, :type => SBDB::Btree, :flags => SBDB::CREATE | SBDB::AUTO_COMMIT]
-					:flags => SBDB::CREATE | SBDB::AUTO_COMMIT]
 		end
 
 		def sync
@@ -84,8 +77,6 @@ module LogAn
 			dat = [sid || 0x10, val].pack 'Na*'
 			name = db_name id
 			db( name)[ id.raw] = dat
-			@counter += 1
-			@queue.push id.raw
 		end
 		alias emit put