diff --git a/Rakefile b/Rakefile index ff42901..1a50839 100644 --- a/Rakefile +++ b/Rakefile @@ -36,7 +36,7 @@ namespace :display do end task :default => ["display:notice"] -ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool) +ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool mysqlspatial mysql2spatial spatialite postgis) ADAPTERS.each do |adapter| namespace :test do desc "Runs #{adapter} database tests." diff --git a/VERSION b/VERSION index d156ab4..f8112eb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.10 \ No newline at end of file +0.2.11 \ No newline at end of file diff --git a/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb b/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb index 7f34e2f..74828b7 100644 --- a/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +++ b/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb @@ -1,6 +1,6 @@ require "active_record/connection_adapters/mysql2_adapter" -require "activerecord-import/adapters/mysql_adapter" +require "activerecord-import/adapters/mysql2_adapter" class ActiveRecord::ConnectionAdapters::Mysql2Adapter - include ActiveRecord::Import::MysqlAdapter + include ActiveRecord::Import::Mysql2Adapter end diff --git a/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb b/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb index f0940c8..c50b8ca 100644 --- a/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +++ b/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb @@ -1,7 +1,7 @@ require "active_record/connection_adapters/sqlite3_adapter" require "activerecord-import/adapters/sqlite3_adapter" -class ActiveRecord::ConnectionAdapters::Sqlite3Adapter - include ActiveRecord::Import::Sqlite3Adapter +class ActiveRecord::ConnectionAdapters::SQLite3Adapter + include ActiveRecord::Import::SQLite3Adapter end diff --git a/lib/activerecord-import/adapters/abstract_adapter.rb b/lib/activerecord-import/adapters/abstract_adapter.rb index 5f38001..3fcc353 100644 --- a/lib/activerecord-import/adapters/abstract_adapter.rb +++ b/lib/activerecord-import/adapters/abstract_adapter.rb @@ -34,7 +34,7 @@ module ActiveRecord::Import::AbstractAdapter # elements that are in position >= 1 will be appended to the final SQL. def insert_many( sql, values, *args ) # :nodoc: # the number of inserts default - number_of_inserts = 0 + number_of_inserts, last_inserted_id = 0, nil base_sql,post_sql = if sql.is_a?( String ) [ sql, '' ] @@ -59,17 +59,17 @@ module ActiveRecord::Import::AbstractAdapter if NO_MAX_PACKET == max or total_bytes < max number_of_inserts += 1 sql2insert = base_sql + values.join( ',' ) + post_sql - insert( sql2insert, *args ) + last_inserted_id = insert( sql2insert, *args ) else value_sets = self.class.get_insert_value_sets( values, sql_size, max ) value_sets.each do |values| number_of_inserts += 1 sql2insert = base_sql + values.join( ',' ) + post_sql - insert( sql2insert, *args ) + last_inserted_id = insert( sql2insert, *args ) end end - number_of_inserts + [number_of_inserts, last_inserted_id] end def pre_sql_statements(options) diff --git a/lib/activerecord-import/adapters/mysql2_adapter.rb b/lib/activerecord-import/adapters/mysql2_adapter.rb new file mode 100644 index 0000000..71e6a64 --- /dev/null +++ b/lib/activerecord-import/adapters/mysql2_adapter.rb @@ -0,0 +1,5 @@ +require File.dirname(__FILE__) + "/mysql_adapter" + +module ActiveRecord::Import::Mysql2Adapter + include ActiveRecord::Import::MysqlAdapter +end \ No newline at end of file diff --git a/lib/activerecord-import/adapters/sqlite3_adapter.rb b/lib/activerecord-import/adapters/sqlite3_adapter.rb index 0d298a4..2fb673d 100644 --- a/lib/activerecord-import/adapters/sqlite3_adapter.rb +++ b/lib/activerecord-import/adapters/sqlite3_adapter.rb @@ -1,4 +1,4 @@ -module ActiveRecord::Import::Sqlite3Adapter +module ActiveRecord::Import::SQLite3Adapter def next_value_for_sequence(sequence_name) %{nextval('#{sequence_name}')} end diff --git a/lib/activerecord-import/base.rb b/lib/activerecord-import/base.rb index 8aeb27c..8498357 100644 --- a/lib/activerecord-import/base.rb +++ b/lib/activerecord-import/base.rb @@ -5,10 +5,20 @@ require "active_record/version" module ActiveRecord::Import AdapterPath = File.join File.expand_path(File.dirname(__FILE__)), "/active_record/adapters" + def self.base_adapter(adapter) + case adapter + when 'mysqlspatial' then 'mysql' + when 'mysql2spatial' then 'mysql2' + when 'spatialite' then 'sqlite3' + when 'postgis' then 'postgresql' + else adapter + end + end + # Loads the import functionality for a specific database adapter def self.require_adapter(adapter) require File.join(AdapterPath,"/abstract_adapter") - require File.join(AdapterPath,"/#{adapter}_adapter") + require File.join(AdapterPath,"/#{base_adapter(adapter)}_adapter") end # Loads the import functionality for the passed in ActiveRecord connection diff --git a/lib/activerecord-import/import.rb b/lib/activerecord-import/import.rb index a596a4c..ecb634a 100644 --- a/lib/activerecord-import/import.rb +++ b/lib/activerecord-import/import.rb @@ -3,7 +3,7 @@ require "ostruct" module ActiveRecord::Import::ConnectionAdapters ; end module ActiveRecord::Import #:nodoc: - class Result < Struct.new(:failed_instances, :num_inserts) + class Result < Struct.new(:failed_instances, :num_inserts, :last_inserted_id) end module ImportSupport #:nodoc: @@ -137,8 +137,8 @@ class ActiveRecord::Base # # # Example synchronizing unsaved/new instances in memory by using a uniqued imported field # posts = [BlogPost.new(:title => "Foo"), BlogPost.new(:title => "Bar")] - # BlogPost.import posts, :synchronize => posts - # puts posts.first.new_record? # => false + # BlogPost.import posts, :synchronize => posts, :synchronize_keys => [:title] + # puts posts.first.persisted? # => true # # == On Duplicate Key Update (MySQL only) # @@ -162,9 +162,10 @@ class ActiveRecord::Base # BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title } # # = Returns - # This returns an object which responds to +failed_instances+ and +num_inserts+. + # This returns an object which responds to +failed_instances+, +num_inserts+, +last_inserted_id+. # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed. - # * num_inserts - the number of insert statements it took to import the data + # * num_inserts - the number of insert statements it took to import the data. + # * last_inserted_id - the last inserted id. Should be the id of the latest inserted row. def import( *args ) options = { :validate=>true, :timestamps=>true } options.merge!( args.pop ) if args.last.is_a? Hash @@ -191,7 +192,7 @@ class ActiveRecord::Base end # supports empty array elsif args.last.is_a?( Array ) and args.last.empty? - return ActiveRecord::Import::Result.new([], 0) if args.last.empty? + return ActiveRecord::Import::Result.new([], 0, nil) if args.last.empty? # supports 2-element array and array elsif args.size == 2 and args.first.is_a?( Array ) and args.last.is_a?( Array ) column_names, array_of_attributes = args @@ -218,8 +219,8 @@ class ActiveRecord::Base return_obj = if is_validating import_with_validations( column_names, array_of_attributes, options ) else - num_inserts = import_without_validations_or_callbacks( column_names, array_of_attributes, options ) - ActiveRecord::Import::Result.new([], num_inserts) + [num_inserts, last_inserted_id] = import_without_validations_or_callbacks( column_names, array_of_attributes, options ) + ActiveRecord::Import::Result.new([], num_inserts, last_inserted_id) end if options[:synchronize] @@ -261,12 +262,12 @@ class ActiveRecord::Base end array_of_attributes.compact! - num_inserts = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any? - 0 + num_inserts, last_inserted_id = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any? + [0, nil] else import_without_validations_or_callbacks( column_names, array_of_attributes, options ) end - ActiveRecord::Import::Result.new(failed_instances, num_inserts) + ActiveRecord::Import::Result.new(failed_instances, num_inserts, last_inserted_id) end # Imports the passed in +column_names+ and +array_of_attributes+ @@ -276,6 +277,14 @@ class ActiveRecord::Base # information on +column_names+, +array_of_attributes_ and # +options+. def import_without_validations_or_callbacks( column_names, array_of_attributes, options={} ) + number_inserted, last_inserted_id = 0, nil + scope_columns, scope_values = scope_attributes.to_a.transpose + + unless scope_columns.blank? + column_names.concat scope_columns + array_of_attributes.each { |a| a.concat scope_values } + end + columns = column_names.each_with_index.map do |name, i| column = columns_hash[name.to_s] @@ -298,11 +307,11 @@ class ActiveRecord::Base post_sql_statements = connection.post_sql_statements( quoted_table_name, options ) # perform the inserts - number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, + number_inserted, last_inserted_id = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, values_sql, "#{self.class.name} Create Many Without Validations Or Callbacks" ) end - number_inserted + [number_inserted, last_inserted_id] end private @@ -310,14 +319,22 @@ class ActiveRecord::Base # Returns SQL the VALUES for an INSERT statement given the passed in +columns+ # and +array_of_attributes+. def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc: + # connection gets called a *lot* in this high intensity loop. + # Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports) + connection_memo = connection array_of_attributes.map do |arr| my_values = arr.each_with_index.map do |val,j| column = columns[j] - if val.nil? && !sequence_name.blank? && column.name == primary_key - connection.next_value_for_sequence(sequence_name) + # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly + if val.nil? && column.name == primary_key && !sequence_name.blank? + connection_memo.next_value_for_sequence(sequence_name) else - connection.quote(column.type_cast(val), column) + if serialized_attributes.include?(column.name) + connection_memo.quote(serialized_attributes[column.name].dump(val), column) + else + connection_memo.quote(val, column) + end end end "(#{my_values.join(',')})" diff --git a/lib/activerecord-import/synchronize.rb b/lib/activerecord-import/synchronize.rb index 60408e5..a5286d7 100644 --- a/lib/activerecord-import/synchronize.rb +++ b/lib/activerecord-import/synchronize.rb @@ -43,6 +43,10 @@ module ActiveRecord # :nodoc: instance.clear_aggregation_cache instance.clear_association_cache instance.instance_variable_set '@attributes', matched_instance.attributes + # Since the instance now accurately reflects the record in + # the database, ensure that instance.persisted? is true. + instance.instance_variable_set '@new_record', false + instance.instance_variable_set '@destroyed', false end end end @@ -52,4 +56,4 @@ module ActiveRecord # :nodoc: self.class.synchronize(instances, key) end end -end \ No newline at end of file +end diff --git a/test/active_record/connection_adapter_test.rb b/test/active_record/connection_adapter_test.rb index ad3234c..5b81e7d 100644 --- a/test/active_record/connection_adapter_test.rb +++ b/test/active_record/connection_adapter_test.rb @@ -50,3 +50,13 @@ describe "ActiveRecord::ConnectionAdapter::AbstractAdapter" do end end + +describe "ActiveRecord::Import DB-specific adapter class" do + context "when ActiveRecord::Import is in use" do + it "should appear in the AR connection adapter class's ancestors" do + connection = ActiveRecord::Base.connection + import_class_name = 'ActiveRecord::Import::' + connection.class.name.demodulize + assert_includes connection.class.ancestors, import_class_name.constantize + end + end +end \ No newline at end of file diff --git a/test/adapters/mysql2spatial.rb b/test/adapters/mysql2spatial.rb new file mode 100644 index 0000000..3bc9f6c --- /dev/null +++ b/test/adapters/mysql2spatial.rb @@ -0,0 +1 @@ +ENV["ARE_DB"] = "mysql2spatial" \ No newline at end of file diff --git a/test/adapters/mysqlspatial.rb b/test/adapters/mysqlspatial.rb new file mode 100644 index 0000000..03316d1 --- /dev/null +++ b/test/adapters/mysqlspatial.rb @@ -0,0 +1 @@ +ENV["ARE_DB"] = "mysqlspatial" \ No newline at end of file diff --git a/test/adapters/postgis.rb b/test/adapters/postgis.rb new file mode 100644 index 0000000..0902039 --- /dev/null +++ b/test/adapters/postgis.rb @@ -0,0 +1 @@ +ENV["ARE_DB"] = "postgis" \ No newline at end of file diff --git a/test/adapters/spatialite.rb b/test/adapters/spatialite.rb new file mode 100644 index 0000000..1ff27f2 --- /dev/null +++ b/test/adapters/spatialite.rb @@ -0,0 +1 @@ +ENV["ARE_DB"] = "spatialite" \ No newline at end of file diff --git a/test/database.yml.sample b/test/database.yml.sample index 6a9a364..9bd8221 100644 --- a/test/database.yml.sample +++ b/test/database.yml.sample @@ -13,6 +13,12 @@ mysql2: <<: *common adapter: mysql2 +mysqlspatial: + <<: *mysql + +mysqlspatial2: + <<: *mysql2 + seamless_database_pool: <<: *common adapter: seamless_database_pool @@ -26,6 +32,9 @@ postgresql: adapter: postgresql min_messages: warning +postgis: + <<: *postgresql + oracle: <<: *common adapter: oracle @@ -38,3 +47,6 @@ sqlite: sqlite3: adapter: sqlite3 database: test.db + +spatialite: + <<: *sqlite3 diff --git a/test/import_test.rb b/test/import_test.rb index 982154e..cc06891 100644 --- a/test/import_test.rb +++ b/test/import_test.rb @@ -134,8 +134,18 @@ describe "#import" do let(:new_topics) { Build(3, :topics) } it "reloads data for existing in-memory instances" do - Topic.import(new_topics, :synchronize => new_topics, :synchronize_key => [:title] ) - assert new_topics.all?(&:new_record?), "Records should have been reloaded" + Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] ) + assert new_topics.all?(&:persisted?), "Records should have been reloaded" + end + end + + context "synchronizing on destroyed records with explicit conditions" do + let(:new_topics) { Generate(3, :topics) } + + it "reloads data for existing in-memory instances" do + new_topics.each &:destroy + Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] ) + assert new_topics.all?(&:persisted?), "Records should have been reloaded" end end end @@ -293,4 +303,18 @@ describe "#import" do assert_equal "2010/05/14".to_date, Topic.last.last_read.to_date end end -end \ No newline at end of file + + context "importing through an association scope" do + [ true, false ].each do |b| + context "when validation is " + (b ? "enabled" : "disabled") do + it "should automatically set the foreign key column" do + books = [[ "David Chelimsky", "The RSpec Book" ], [ "Chad Fowler", "Rails Recipes" ]] + topic = Factory.create :topic + topic.books.import [ :author_name, :title ], books, :validate => b + assert_equal 2, topic.books.count + assert topic.books.all? { |b| b.topic_id == topic.id } + end + end + end + end +end diff --git a/test/mysqlspatial/import_test.rb b/test/mysqlspatial/import_test.rb new file mode 100644 index 0000000..feaff67 --- /dev/null +++ b/test/mysqlspatial/import_test.rb @@ -0,0 +1,6 @@ +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') + +require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') +require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') + +should_support_mysql_import_functionality \ No newline at end of file diff --git a/test/mysqlspatial2/import_test.rb b/test/mysqlspatial2/import_test.rb new file mode 100644 index 0000000..feaff67 --- /dev/null +++ b/test/mysqlspatial2/import_test.rb @@ -0,0 +1,6 @@ +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') + +require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') +require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') + +should_support_mysql_import_functionality \ No newline at end of file diff --git a/test/postgis/import_test.rb b/test/postgis/import_test.rb new file mode 100644 index 0000000..436fc83 --- /dev/null +++ b/test/postgis/import_test.rb @@ -0,0 +1,4 @@ +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') +require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') + +should_support_postgresql_import_functionality \ No newline at end of file diff --git a/test/postgresql/import_test.rb b/test/postgresql/import_test.rb index 45e97ef..436fc83 100644 --- a/test/postgresql/import_test.rb +++ b/test/postgresql/import_test.rb @@ -1,20 +1,4 @@ -require File.expand_path('../../test_helper', __FILE__) +require File.expand_path(File.dirname(__FILE__) + '/../test_helper') +require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') -describe "#supports_imports?" do - it "should support import" do - assert ActiveRecord::Base.supports_import? - end -end - -describe "#import" do - it "should import with a single insert" do - # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics - assert_difference "Topic.count", +10 do - result = Topic.import Build(3, :topics) - assert_equal 1, result.num_inserts - - result = Topic.import Build(7, :topics) - assert_equal 1, result.num_inserts - end - end -end +should_support_postgresql_import_functionality \ No newline at end of file diff --git a/test/support/postgresql/import_examples.rb b/test/support/postgresql/import_examples.rb new file mode 100644 index 0000000..fe7b61f --- /dev/null +++ b/test/support/postgresql/import_examples.rb @@ -0,0 +1,21 @@ +# encoding: UTF-8 +def should_support_postgresql_import_functionality + describe "#supports_imports?" do + it "should support import" do + assert ActiveRecord::Base.supports_import? + end + end + + describe "#import" do + it "should import with a single insert" do + # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics + assert_difference "Topic.count", +10 do + result = Topic.import Build(3, :topics) + assert_equal 1, result.num_inserts + + result = Topic.import Build(7, :topics) + assert_equal 1, result.num_inserts + end + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 73f2802..a28e5b4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -44,3 +44,8 @@ adapter_schema = test_dir.join("schema/#{adapter}_schema.rb") require adapter_schema if File.exists?(adapter_schema) Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file } + +# Prevent this deprecation warning from breaking the tests. +module Rake::DeprecatedObjectDSL + remove_method :import +end