From 5c5bd5f716aee4823f6d600e3c9865ac5492d96c Mon Sep 17 00:00:00 2001 From: Zach Dennis Date: Fri, 12 Mar 2010 00:04:15 -0500 Subject: [PATCH] Implemented tests for #get_value_insert_sets. --- lib/ar-extensions.rb | 1 + .../adapters/abstract_adapter.rb | 146 ++++++++++++++++++ test/active_record/connection_adapter_test.rb | 53 +++++++ test/import_test.rb | 29 +--- 4 files changed, 201 insertions(+), 28 deletions(-) create mode 100644 lib/ar-extensions/active_record/adapters/abstract_adapter.rb create mode 100644 test/active_record/connection_adapter_test.rb diff --git a/lib/ar-extensions.rb b/lib/ar-extensions.rb index 20c10b4..8d00dd1 100644 --- a/lib/ar-extensions.rb +++ b/lib/ar-extensions.rb @@ -5,3 +5,4 @@ module ActiveRecord::Extensions end require "ar-extensions/import" +require "ar-extensions/active_record/adapters/abstract_adapter" diff --git a/lib/ar-extensions/active_record/adapters/abstract_adapter.rb b/lib/ar-extensions/active_record/adapters/abstract_adapter.rb new file mode 100644 index 0000000..a088ccb --- /dev/null +++ b/lib/ar-extensions/active_record/adapters/abstract_adapter.rb @@ -0,0 +1,146 @@ +module ActiveRecord # :nodoc: + module ConnectionAdapters # :nodoc: + class AbstractAdapter # :nodoc: + NO_MAX_PACKET = 0 + QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from. + + def next_value_for_sequence(sequence_name) + %{#{sequence_name}.nextval} + end + + # +sql+ can be a single string or an array. If it is an array all + # elements that are in position >= 1 will be appended to the final SQL. + def insert_many( sql, values, *args ) # :nodoc: + # the number of inserts default + number_of_inserts = 0 + + base_sql,post_sql = if sql.is_a?( String ) + [ sql, '' ] + elsif sql.is_a?( Array ) + [ sql.shift, sql.join( ' ' ) ] + end + + sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size + + # the number of bytes the requested insert statement values will take up + values_in_bytes = self.class.sum_sizes( *values ) + + # the number of bytes (commas) it will take to comma separate our values + comma_separated_bytes = values.size-1 + + # the total number of bytes required if this statement is one statement + total_bytes = sql_size + values_in_bytes + comma_separated_bytes + + max = max_allowed_packet + + # if we can insert it all as one statement + if NO_MAX_PACKET == max or total_bytes < max + number_of_inserts += 1 + sql2insert = base_sql + values.join( ',' ) + post_sql + insert( sql2insert, *args ) + else + value_sets = self.class.get_insert_value_sets( values, sql_size, max ) + value_sets.each do |values| + number_of_inserts += 1 + sql2insert = base_sql + values.join( ',' ) + post_sql + insert( sql2insert, *args ) + end + end + + number_of_inserts + end + + def pre_sql_statements(options) + sql = [] + sql << options[:pre_sql] if options[:pre_sql] + sql << options[:command] if options[:command] + sql << "IGNORE" if options[:ignore] + + #add keywords like IGNORE or DELAYED + if options[:keywords].is_a?(Array) + sql.concat(options[:keywords]) + elsif options[:keywords] + sql << options[:keywords].to_s + end + + sql + end + + # Synchronizes the passed in ActiveRecord instances with the records in + # the database by calling +reload+ on each instance. + def after_import_synchronize( instances ) + instances.each { |e| e.reload } + end + + # Returns an array of post SQL statements given the passed in options. + def post_sql_statements( table_name, options ) # :nodoc: + post_sql_statements = [] + if options[:on_duplicate_key_update] + post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] ) + end + + #custom user post_sql + post_sql_statements << options[:post_sql] if options[:post_sql] + + #with rollup + post_sql_statements << rollup_sql if options[:rollup] + + post_sql_statements + end + + + # Generates the INSERT statement used in insert multiple value sets. + def multiple_value_sets_insert_sql(table_name, column_names, options) # :nodoc: + "INSERT #{options[:ignore] ? 'IGNORE ':''}INTO #{table_name} (#{column_names.join(',')}) VALUES " + end + + # Returns SQL the VALUES for an INSERT statement given the passed in +columns+ + # and +array_of_attributes+. + def values_sql_for_column_names_and_attributes( columns, array_of_attributes ) # :nodoc: + values = [] + array_of_attributes.each do |arr| + my_values = [] + arr.each_with_index do |val,j| + my_values << quote( val, columns[j] ) + end + values << my_values + end + values_arr = values.map{ |arr| '(' + arr.join( ',' ) + ')' } + end + + # Returns the sum of the sizes of the passed in objects. This should + # probably be moved outside this class, but to where? + def self.sum_sizes( *objects ) # :nodoc: + objects.inject( 0 ){|sum,o| sum += o.size } + end + + # Returns the maximum number of bytes that the server will allow + # in a single packet + def max_allowed_packet + NO_MAX_PACKET + end + + def self.get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc: + value_sets = [] + arr, current_arr_values_size, current_size = [], 0, 0 + values.each_with_index do |val,i| + comma_bytes = arr.size + sql_size_thus_far = sql_size + current_size + val.size + comma_bytes + if NO_MAX_PACKET == max_bytes or sql_size_thus_far <= max_bytes + current_size += val.size + arr << val + else + value_sets << arr + arr = [ val ] + current_size = val.size + end + + # if we're on the last iteration push whatever we have in arr to value_sets + value_sets << arr if i == (values.size-1) + end + [ *value_sets ] + end + + end + end +end diff --git a/test/active_record/connection_adapter_test.rb b/test/active_record/connection_adapter_test.rb new file mode 100644 index 0000000..c10cc0e --- /dev/null +++ b/test/active_record/connection_adapter_test.rb @@ -0,0 +1,53 @@ +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') + +describe "ActiveRecord::ConnectionADapter::AbstractAdapter" do + context "#get_insert_value_sets - computing insert value sets" do + let(:adapter){ ActiveRecord::ConnectionAdapters::AbstractAdapter } + let(:base_sql){ "INSERT INTO atable (a,b,c)" } + let(:values){ [ "(1,2,3)", "(2,3,4)", "(3,4,5)" ] } + + context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do + it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 33 + assert_equal 3, value_sets.size + end + end + + context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do + it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 40 + puts value_sets.inspect + assert_equal 3, value_sets.size + end + end + + context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do + it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 41 + assert_equal 2, value_sets.size + end + end + + context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do + it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 48 + assert_equal 2, value_sets.size + end + end + + context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do + it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 49 + assert_equal 1, value_sets.size + end + end + + context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do + it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do + value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 999999 + assert_equal 1, value_sets.size + end + end + end + +end diff --git a/test/import_test.rb b/test/import_test.rb index 1fac348..e516e29 100644 --- a/test/import_test.rb +++ b/test/import_test.rb @@ -215,32 +215,5 @@ describe "#import" do assert_equal "superx", Group.first.order end end - - - # - # describe "computing insert value sets" do - # context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 3 value sets of 7 bytes a piece" - # end - # - # context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 3 value sets of 7 bytes a piece" - # end - # - # context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 2 value sets of 7 bytes a piece" - # end - # - # context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 2 value sets of 7 bytes a piece" - # end - # - # context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 1 value sets of 7 bytes a piece" - # end - # - # context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do - # it "should return 3 value sets when given 1 value sets of 7 bytes a piece" - # end - # end + end \ No newline at end of file