Added back in basic benchmark support. To run:

ruby benchmark/benchmark.rb --adapter mysql --num 1000
  ruby benchmark/benchmark.rb --adapter mysql --num 1000 --to-csv /tmp/results.csv
  ruby benchmark/benchmark.rb --adapter mysql --num 1000 --to-html /tmp/results.html
This commit is contained in:
Zach Dennis 2010-04-08 23:12:28 -04:00
parent 369a5e0e64
commit 5c287f1042
18 changed files with 509 additions and 22 deletions

32
benchmarks/README Normal file
View file

@ -0,0 +1,32 @@
To run the benchmarks, from within the benchmarks run:
ruby benchmark.rb [options]
The following options are supported:
--adapter [String] The database adapter to use. IE: mysql, postgresql, oracle
--do-not-delete By default all records in the benchmark tables will be deleted at the end of the benchmark. This flag indicates not to delete the benchmark data.
--num [Integer] The number of objects to benchmark. (Required!)
--table-type [String] The table type to test. This can be used multiple times. By default it is all table types.
--to-csv [String] Print results in a CSV file format
--to-html [String] Print results in HTML format (String filename must be supplied)
See "ruby benchmark.rb -h" for the complete listing of options.
EXAMPLES
--------
To output to html format:
ruby benchmark.rb --adapter=mysql --to-html=results.html
To output to csv format:
ruby benchmark.rb --adapter=mysql --to-csv=results.csv
LIMITATIONS
-----------
Currently MySQL is the only supported adapter to benchmark.
AUTHOR
------
Zach Dennis
zach.dennis@gmail.com
http://www.continuousthinking.com

64
benchmarks/benchmark.rb Normal file
View file

@ -0,0 +1,64 @@
require "pathname"
this_dir = Pathname.new File.dirname(__FILE__)
require this_dir.join('boot')
# Parse the options passed in via the command line
options = BenchmarkOptionParser.parse( ARGV )
# The support directory where we use to load our connections and models for the
# benchmarks.
SUPPORT_DIR = this_dir.join('../test')
# Load the database adapter
adapter = options.adapter
# load the library
LIB_DIR = this_dir.join("../lib")
require LIB_DIR.join("ar-extensions/import/#{adapter}")
ActiveRecord::Base.logger = Logger.new("log/test.log")
ActiveRecord::Base.logger.level = Logger::DEBUG
ActiveRecord::Base.configurations["test"] = YAML.load(SUPPORT_DIR.join("database.yml").open)[adapter]
ActiveRecord::Base.establish_connection "test"
ActiveSupport::Notifications.subscribe(/active_record.sql/) do |event, _, _, _, hsh|
ActiveRecord::Base.logger.info hsh[:sql]
end
adapter_schema = SUPPORT_DIR.join("schema/#{adapter}_schema.rb")
require adapter_schema if File.exists?(adapter_schema)
Dir[this_dir.join("models/*.rb")].each{ |file| require file }
# Load databse specific benchmarks
require File.join( File.dirname( __FILE__ ), 'lib', "#{adapter}_benchmark" )
# TODO implement method/table-type selection
table_types = nil
if options.benchmark_all_types
table_types = [ "all" ]
else
table_types = options.table_types.keys
end
puts
letter = options.adapter[0].chr
clazz_str = letter.upcase + options.adapter[1..-1].downcase
clazz = Object.const_get( clazz_str + "Benchmark" )
benchmarks = []
options.number_of_objects.each do |num|
benchmarks << (benchmark = clazz.new)
benchmark.send( "benchmark", table_types, num )
end
options.outputs.each do |output|
format = output.format.downcase
output_module = Object.const_get( "OutputTo#{format.upcase}" )
benchmarks.each do |benchmark|
output_module.output_results( output.filename, benchmark.results )
end
end
puts
puts "Done with benchmark!"

18
benchmarks/boot.rb Normal file
View file

@ -0,0 +1,18 @@
begin ; require 'rubygems' ; rescue LoadError ; end
require 'active_record' # ActiveRecord loads the Benchmark library automatically
require 'active_record/version'
require 'fastercsv'
require 'fileutils'
require 'logger'
# Files are loaded alphabetically. If this is a problem then manually specify the files
# that need to be loaded here.
Dir[ File.join( File.dirname( __FILE__ ), 'lib', '*.rb' ) ].sort.each{ |f| require f }
ActiveRecord::Base.logger = Logger.new STDOUT

137
benchmarks/lib/base.rb Normal file
View file

@ -0,0 +1,137 @@
class BenchmarkBase
attr_reader :results
# The main benchmark method dispatcher. This dispatches the benchmarks
# to actual benchmark_xxxx methods.
#
# == PARAMETERS
# * table_types - an array of table types to benchmark
# * num - the number of record insertions to test
def benchmark( table_types, num )
array_of_cols_and_vals = build_array_of_cols_and_vals( num )
table_types.each do |table_type|
self.send( "benchmark_#{table_type}", array_of_cols_and_vals )
end
end
# Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
# actual benchmark.
#
# == PARAMETERS
# * description - the description of the block that is getting benchmarked
# * blk - the block of code to benchmark
#
# == RETURNS
# An OpenStruct object with the following attributes:
# * description - the description of the benchmark ran
# * tms - a Benchmark::Tms containing the results of the benchmark
def bm( description, &blk )
tms = nil
puts "Benchmarking #{description}"
Benchmark.bm { |x| tms = x.report { blk.call } }
delete_all
failed = false
OpenStruct.new :description=>description, :tms=>tms, :failed=>failed
end
# Given a model class (ie: Topic), and an array of columns and value sets
# this will perform all of the benchmarks necessary for this library.
#
# == PARAMETERS
# * model_clazz - the model class to benchmark (ie: Topic)
# * array_of_cols_and_vals - an array of column identifiers and value sets
#
# == RETURNS
# returns true
def bm_model( model_clazz, array_of_cols_and_vals )
puts
puts "------ Benchmarking #{model_clazz.name} -------"
cols,vals = array_of_cols_and_vals
num_inserts = vals.size
# add a new result group for this particular benchmark
group = []
@results << group
description = "#{model_clazz.name}.create (#{num_inserts} records)"
group << bm( description ) {
vals.each do |values|
model_clazz.create create_hash_for_cols_and_vals( cols, values )
end }
description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations"
group << bm( description ) { model_clazz.import cols, vals, :validate=>true }
description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations"
group << bm( description ) { model_clazz.import cols, vals, :validate=>false }
models = []
array_of_attrs = []
vals.each do |arr|
array_of_attrs << (attrs={})
arr.each_with_index { |value, i| attrs[cols[i]] = value }
end
array_of_attrs.each{ |attrs| models << model_clazz.new(attrs) }
description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations"
group << bm( description ) { model_clazz.import models, :validate=>true }
description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations"
group << bm( description ) { model_clazz.import models, :validate=>false }
true
end
# Returns a two element array composing of an array of columns and an array of
# value sets given the passed +num+.
#
# === What is a value set?
# A value set is an array of arrays. Each child array represents an array of value sets
# for a given row of data.
#
# For example, say we wanted to represent an insertion of two records:
# column_names = [ 'id', 'name', 'description' ]
# record1 = [ 1, 'John Doe', 'A plumber' ]
# record2 = [ 2, 'John Smith', 'A painter' ]
# value_set [ record1, record2 ]
#
# == PARAMETER
# * num - the number of records to create
def build_array_of_cols_and_vals( num )
cols = [ :my_name, :description ]
value_sets = []
num.times { |i| value_sets << [ "My Name #{i}", "My Description #{i}" ] }
[ cols, value_sets ]
end
# Returns a hash of column identifier to value mappings giving the passed in
# value array.
#
# Example:
# cols = [ 'id', 'name', 'description' ]
# values = [ 1, 'John Doe', 'A plumber' ]
# hsh = create_hash_for_cols_and_vals( cols, values )
# # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' }
def create_hash_for_cols_and_vals( cols, vals )
h = {}
cols.zip( vals ){ |col,val| h[col] = val }
h
end
# Deletes all records from all ActiveRecord subclasses
def delete_all
ActiveRecord::Base.send( :subclasses ).each do |subclass|
subclass.delete_all if subclass.respond_to? :delete_all
end
end
def initialize # :nodoc:
@results = []
end
end

View file

@ -0,0 +1,103 @@
require 'optparse'
require 'ostruct'
#
# == PARAMETERS
# * a - database adapter. ie: mysql, postgresql, oracle, etc.
# * n - number of objects to test with. ie: 1, 100, 1000, etc.
# * t - the table types to test. ie: myisam, innodb, memory, temporary, etc.
#
module BenchmarkOptionParser
BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options."
def self.print_banner
puts BANNER
end
def self.print_banner!
print_banner
exit
end
def self.print_options( options )
puts "Benchmarking the following options:"
puts " Database adapter: #{options.adapter}"
puts " Number of objects: #{options.number_of_objects}"
puts " Table types:"
print_valid_table_types( options, :prefix=>" " )
end
# TODO IMPLEMENT THIS
def self.print_valid_table_types( options, hsh={:prefix=>''} )
if options.table_types.keys.size > 0
options.table_types.keys.sort.each{ |type| puts hsh[:prefix].to_s + type.to_s }
else
puts 'No table types defined.'
end
end
def self.parse( args )
options = OpenStruct.new(
:table_types => {},
:delete_on_finish => true,
:number_of_objects => [],
:outputs => [] )
opts = OptionParser.new do |opts|
opts.banner = BANNER
# parse the database adapter
opts.on( "a", "--adapter [String]",
"The database adapter to use. IE: mysql, postgresql, oracle" ) do |arg|
options.adapter = arg
end
# parse do_not_delete flag
opts.on( "d", "--do-not-delete",
"By default all records in the benchmark tables will be deleted at the end of the benchmark. " +
"This flag indicates not to delete the benchmark data." ) do |arg|
options.delete_on_finish = false
end
# parse the number of row objects to test
opts.on( "n", "--num [Integer]",
"The number of objects to benchmark." ) do |arg|
options.number_of_objects << arg.to_i
end
# parse the table types to test
opts.on( "t", "--table-type [String]",
"The table type to test. This can be used multiple times." ) do |arg|
if arg =~ /^all$/
options.table_types['all'] = options.benchmark_all_types = true
else
options.table_types[arg] = true
end
end
# print results in CSV format
opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
options.outputs << OpenStruct.new( :format=>'csv', :filename=>filename)
end
# print results in HTML format
opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
options.outputs << OpenStruct.new( :format=>'html', :filename=>filename )
end
end #end opt.parse!
begin
opts.parse!( args )
if options.table_types.size == 0
options.table_types['all'] = options.benchmark_all_types = true
end
rescue Exception => ex
print_banner!
end
print_options( options )
options
end
end

15
benchmarks/lib/float.rb Normal file
View file

@ -0,0 +1,15 @@
# Taken from http://www.programmingishard.com/posts/show/128
# Posted by rbates
class Float
def round_to(x)
(self * 10**x).round.to_f / 10**x
end
def ceil_to(x)
(self * 10**x).ceil.to_f / 10**x
end
def floor_to(x)
(self * 10**x).floor.to_f / 10**x
end
end

View file

@ -0,0 +1,22 @@
class MysqlBenchmark < BenchmarkBase
def benchmark_all( array_of_cols_and_vals )
methods = self.methods.find_all { |m| m =~ /benchmark_/ }
methods.delete_if{ |m| m =~ /benchmark_(all|model)/ }
methods.each { |method| self.send( method, array_of_cols_and_vals ) }
end
def benchmark_myisam( array_of_cols_and_vals )
bm_model( TestMyISAM, array_of_cols_and_vals )
end
def benchmark_innodb( array_of_cols_and_vals )
bm_model( TestInnoDb, array_of_cols_and_vals )
end
def benchmark_memory( array_of_cols_and_vals )
bm_model( TestMemory, array_of_cols_and_vals )
end
end

View file

@ -0,0 +1,18 @@
require 'fastercsv'
module OutputToCSV
def self.output_results( filename, results )
FasterCSV.open( filename, 'w' ) do |csv|
# Iterate over each result set, which contains many results
results.each do |result_set|
columns, times = [], []
result_set.each do |result|
columns << result.description
times << result.tms.real
end
csv << columns
csv << times
end
end
end
end

View file

@ -0,0 +1,69 @@
require 'erb'
module OutputToHTML
TEMPLATE_HEADER =<<"EOT"
<div>
All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
the very first column (which is always the default ActiveRecord::Base.create method.
</div>
EOT
TEMPLATE =<<"EOT"
<style>
td#benchmarkTitle {
border: 1px solid black;
padding: 2px;
font-size: 0.8em;
background-color: black;
color: white;
}
td#benchmarkCell {
border: 1px solid black;
padding: 2px;
font-size: 0.8em;
}
</style>
<table>
<tr>
<% columns.each do |col| %>
<td id="benchmarkTitle"><%= col %></td>
<% end %>
</tr>
<tr>
<% times.each do |time| %>
<td id="benchmarkCell"><%= time %></td>
<% end %>
</tr>
<tr><td>&nbsp;</td></tr>
</table>
EOT
def self.output_results( filename, results )
html = ''
results.each do |result_set|
columns, times = [], []
result_set.each do |result|
columns << result.description
if result.failed
times << "failed"
else
time = result.tms.real.round_to( 3 )
speedup = ( result_set.first.tms.real / result.tms.real ).round
if result == result_set.first
times << "#{time}"
else
times << "#{time} (#{speedup}x speedup)"
end
end
end
template = ERB.new( TEMPLATE, 0, "%<>")
html << template.result( binding )
end
File.open( filename, 'w' ){ |file| file.write( TEMPLATE_HEADER + html ) }
end
end

View file

@ -0,0 +1,3 @@
class TestInnoDb < ActiveRecord::Base
set_table_name 'test_innodb'
end

View file

@ -0,0 +1,3 @@
class TestMemory < ActiveRecord::Base
set_table_name 'test_memory'
end

View file

@ -0,0 +1,3 @@
class TestMyISAM < ActiveRecord::Base
set_table_name 'test_myisam'
end

View file

@ -0,0 +1,16 @@
ActiveRecord::Schema.define do
create_table :test_myisam, :options=>'ENGINE=MyISAM', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
create_table :test_innodb, :options=>'ENGINE=InnoDb', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
create_table :test_memory, :options=>'ENGINE=Memory', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
end

View file

@ -1,3 +1,5 @@
require "active_record/connection_adapters/postgresql_adapter"
module ActiveRecord # :nodoc:
module ConnectionAdapters # :nodoc:
class PostgreSQLAdapter # :nodoc:

View file

@ -3,7 +3,6 @@ require "ostruct"
module ActiveRecord::Extensions::ConnectionAdapters ; end
module ActiveRecord::Extensions::Import #:nodoc:
module ImportSupport #:nodoc:
def supports_import? #:nodoc:
true
@ -15,7 +14,6 @@ module ActiveRecord::Extensions::Import #:nodoc:
true
end
end
end
class ActiveRecord::Base
@ -299,7 +297,6 @@ class ActiveRecord::Base
private
def add_special_rails_stamps( column_names, array_of_attributes, options )
AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk|

View file

@ -1,3 +1,4 @@
require "pathname"
require "active_record"
require "active_record/version"
@ -9,5 +10,6 @@ module ActiveRecord::Extensions
end
end
require "ar-extensions/import"
require "ar-extensions/active_record/adapters/abstract_adapter"
this_dir = Pathname.new File.dirname(__FILE__)
require this_dir.join("../import")
require this_dir.join("../active_record/adapters/abstract_adapter")

View file

@ -1,4 +1,2 @@
require "active_record/connection_adapters/postgresql_adapter"
require File.join File.dirname(__FILE__), "base"
ActiveRecord::Extensions.require_adapter "postgresql"

View file

@ -1,20 +1,5 @@
ActiveRecord::Schema.define do
create_table :test_myisam, :options=>'ENGINE=MyISAM', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
create_table :test_innodb, :options=>'ENGINE=InnoDb', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
create_table :test_memory, :options=>'ENGINE=Memory', :force=>true do |t|
t.column :my_name, :string, :null=>false
t.column :description, :string
end
create_table :books, :options=>'ENGINE=MyISAM', :force=>true do |t|
t.column :title, :string, :null=>false
t.column :publisher, :string, :null=>false, :default => 'Default Publisher'