import_many, import_*: returns last_inserted_id

[#47]

Rakefile

@@ -36,7 +36,7 @@ namespace :display do
 end
 task :default => ["display:notice"]
 
-ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool)
+ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool mysqlspatial mysql2spatial spatialite postgis)
 ADAPTERS.each do |adapter|
   namespace :test do
     desc "Runs #{adapter} database tests."

VERSION

@@ -1 +1 @@
-0.2.10
+0.2.11

lib/activerecord-import/active_record/adapters/mysql2_adapter.rb

@@ -1,6 +1,6 @@
 require "active_record/connection_adapters/mysql2_adapter"
-require "activerecord-import/adapters/mysql_adapter"
+require "activerecord-import/adapters/mysql2_adapter"
 
 class ActiveRecord::ConnectionAdapters::Mysql2Adapter
-  include ActiveRecord::Import::MysqlAdapter
+  include ActiveRecord::Import::Mysql2Adapter
 end

lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb

@@ -1,7 +1,7 @@
 require "active_record/connection_adapters/sqlite3_adapter"
 require "activerecord-import/adapters/sqlite3_adapter"
 
-class ActiveRecord::ConnectionAdapters::Sqlite3Adapter
+class ActiveRecord::ConnectionAdapters::SQLite3Adapter
-  include ActiveRecord::Import::Sqlite3Adapter
+  include ActiveRecord::Import::SQLite3Adapter
 end
 

lib/activerecord-import/adapters/abstract_adapter.rb

@@ -34,7 +34,7 @@ module ActiveRecord::Import::AbstractAdapter
     # elements that are in position >= 1 will be appended to the final SQL.
     def insert_many( sql, values, *args ) # :nodoc:
       # the number of inserts default
-      number_of_inserts = 0
+      number_of_inserts, last_inserted_id = 0, nil
     
       base_sql,post_sql = if sql.is_a?( String )
         [ sql, '' ]
@@ -59,17 +59,17 @@ module ActiveRecord::Import::AbstractAdapter
       if NO_MAX_PACKET == max or total_bytes < max
         number_of_inserts += 1
         sql2insert = base_sql + values.join( ',' ) + post_sql
-        insert( sql2insert, *args )
+        last_inserted_id = insert( sql2insert, *args )
       else
         value_sets = self.class.get_insert_value_sets( values, sql_size, max )
         value_sets.each do |values|
           number_of_inserts += 1
           sql2insert = base_sql + values.join( ',' ) + post_sql
-          insert( sql2insert, *args )
+          last_inserted_id = insert( sql2insert, *args )
         end
       end
 
-      number_of_inserts
+      [number_of_inserts, last_inserted_id]
     end
 
     def pre_sql_statements(options)

lib/activerecord-import/adapters/mysql2_adapter.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + "/mysql_adapter"
+
+module ActiveRecord::Import::Mysql2Adapter
+  include ActiveRecord::Import::MysqlAdapter
+end

lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -1,4 +1,4 @@
-module ActiveRecord::Import::Sqlite3Adapter
+module ActiveRecord::Import::SQLite3Adapter
   def next_value_for_sequence(sequence_name)
     %{nextval('#{sequence_name}')}
   end

lib/activerecord-import/base.rb

@@ -5,10 +5,20 @@ require "active_record/version"
 module ActiveRecord::Import
   AdapterPath = File.join File.expand_path(File.dirname(__FILE__)), "/active_record/adapters"
 
+  def self.base_adapter(adapter)
+    case adapter
+    when 'mysqlspatial' then 'mysql'
+    when 'mysql2spatial' then 'mysql2'
+    when 'spatialite' then 'sqlite3'
+    when 'postgis' then 'postgresql'
+    else adapter
+    end
+  end
+  
   # Loads the import functionality for a specific database adapter
   def self.require_adapter(adapter)
     require File.join(AdapterPath,"/abstract_adapter")
-    require File.join(AdapterPath,"/#{adapter}_adapter")
+    require File.join(AdapterPath,"/#{base_adapter(adapter)}_adapter")
   end
 
   # Loads the import functionality for the passed in ActiveRecord connection

lib/activerecord-import/import.rb

@@ -3,7 +3,7 @@ require "ostruct"
 module ActiveRecord::Import::ConnectionAdapters ; end
 
 module ActiveRecord::Import #:nodoc:
-  class Result < Struct.new(:failed_instances, :num_inserts)
+  class Result < Struct.new(:failed_instances, :num_inserts, :last_inserted_id)
   end
 
   module ImportSupport #:nodoc:
@@ -137,8 +137,8 @@ class ActiveRecord::Base
     #
     #  # Example synchronizing unsaved/new instances in memory by using a uniqued imported field
     #  posts = [BlogPost.new(:title => "Foo"), BlogPost.new(:title => "Bar")]
-    #  BlogPost.import posts, :synchronize => posts
+    #  BlogPost.import posts, :synchronize => posts, :synchronize_keys => [:title]
-    #  puts posts.first.new_record? # => false
+    #  puts posts.first.persisted? # => true
     #
     # == On Duplicate Key Update (MySQL only)
     #
@@ -162,9 +162,10 @@ class ActiveRecord::Base
     #   BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title } 
     #  
     # = Returns
-    # This returns an object which responds to +failed_instances+ and +num_inserts+.
+    # This returns an object which responds to +failed_instances+, +num_inserts+, +last_inserted_id+.
     # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
-    # * num_inserts - the number of insert statements it took to import the data
+    # * num_inserts - the number of insert statements it took to import the data.
+    # * last_inserted_id - the last inserted id. Should be the id of the latest inserted row.
     def import( *args )
       options = { :validate=>true, :timestamps=>true }
       options.merge!( args.pop ) if args.last.is_a? Hash
@@ -191,7 +192,7 @@ class ActiveRecord::Base
         end
         # supports empty array
       elsif args.last.is_a?( Array ) and args.last.empty?
-        return ActiveRecord::Import::Result.new([], 0) if args.last.empty?
+        return ActiveRecord::Import::Result.new([], 0, nil) if args.last.empty?
         # supports 2-element array and array
       elsif args.size == 2 and args.first.is_a?( Array ) and args.last.is_a?( Array )
         column_names, array_of_attributes = args
@@ -218,8 +219,8 @@ class ActiveRecord::Base
       return_obj = if is_validating
         import_with_validations( column_names, array_of_attributes, options )
       else
-        num_inserts = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
+        [num_inserts, last_inserted_id] = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
-        ActiveRecord::Import::Result.new([], num_inserts)
+        ActiveRecord::Import::Result.new([], num_inserts, last_inserted_id)
       end
 
       if options[:synchronize]
@@ -261,12 +262,12 @@ class ActiveRecord::Base
       end
       array_of_attributes.compact!
 
-      num_inserts = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any?
+      num_inserts, last_inserted_id = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any?
-                      0
+                      [0, nil]
                     else
                       import_without_validations_or_callbacks( column_names, array_of_attributes, options )
                     end
-      ActiveRecord::Import::Result.new(failed_instances, num_inserts)
+      ActiveRecord::Import::Result.new(failed_instances, num_inserts, last_inserted_id)
     end
     
     # Imports the passed in +column_names+ and +array_of_attributes+
@@ -276,6 +277,14 @@ class ActiveRecord::Base
     # information on +column_names+, +array_of_attributes_ and
     # +options+.
     def import_without_validations_or_callbacks( column_names, array_of_attributes, options={} )
+      number_inserted, last_inserted_id = 0, nil
+      scope_columns, scope_values = scope_attributes.to_a.transpose
+
+      unless scope_columns.blank?
+        column_names.concat scope_columns
+        array_of_attributes.each { |a| a.concat scope_values }
+      end
+
       columns = column_names.each_with_index.map do |name, i|
         column = columns_hash[name.to_s]
 
@@ -298,11 +307,11 @@ class ActiveRecord::Base
         post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
         
         # perform the inserts
-        number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, 
+        number_inserted, last_inserted_id = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, 
                                                   values_sql,
                                                   "#{self.class.name} Create Many Without Validations Or Callbacks" )
       end
-      number_inserted
+      [number_inserted, last_inserted_id]
     end
 
     private
@@ -310,14 +319,22 @@ class ActiveRecord::Base
     # Returns SQL the VALUES for an INSERT statement given the passed in +columns+
     # and +array_of_attributes+.
     def values_sql_for_columns_and_attributes(columns, array_of_attributes)   # :nodoc:
+      # connection gets called a *lot* in this high intensity loop.
+      # Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports)
+      connection_memo = connection
       array_of_attributes.map do |arr|
         my_values = arr.each_with_index.map do |val,j|
           column = columns[j]
 
-          if val.nil? && !sequence_name.blank? && column.name == primary_key
+          # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
-             connection.next_value_for_sequence(sequence_name)
+          if val.nil? && column.name == primary_key && !sequence_name.blank?
+             connection_memo.next_value_for_sequence(sequence_name)
           else
-            connection.quote(column.type_cast(val), column)
+            if serialized_attributes.include?(column.name)
+              connection_memo.quote(serialized_attributes[column.name].dump(val), column)
+            else
+              connection_memo.quote(val, column)
+            end
           end
         end
         "(#{my_values.join(',')})"

lib/activerecord-import/synchronize.rb

@@ -43,6 +43,10 @@ module ActiveRecord # :nodoc:
           instance.clear_aggregation_cache
           instance.clear_association_cache
           instance.instance_variable_set '@attributes', matched_instance.attributes
+          # Since the instance now accurately reflects the record in
+          # the database, ensure that instance.persisted? is true.
+          instance.instance_variable_set '@new_record', false
+          instance.instance_variable_set '@destroyed', false
         end
       end
     end

test/active_record/connection_adapter_test.rb

@@ -50,3 +50,13 @@ describe "ActiveRecord::ConnectionAdapter::AbstractAdapter" do
   end
   
 end
+
+describe "ActiveRecord::Import DB-specific adapter class" do
+  context "when ActiveRecord::Import is in use" do
+    it "should appear in the AR connection adapter class's ancestors" do
+      connection = ActiveRecord::Base.connection
+      import_class_name = 'ActiveRecord::Import::' + connection.class.name.demodulize
+      assert_includes connection.class.ancestors, import_class_name.constantize
+    end
+  end
+end

test/adapters/mysql2spatial.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "mysql2spatial"

test/adapters/mysqlspatial.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "mysqlspatial"

test/adapters/postgis.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "postgis"

test/adapters/spatialite.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "spatialite"

test/database.yml.sample

@@ -13,6 +13,12 @@ mysql2:
   <<: *common
   adapter: mysql2
 
+mysqlspatial:
+  <<: *mysql
+
+mysqlspatial2:
+  <<: *mysql2
+
 seamless_database_pool:
   <<: *common
   adapter: seamless_database_pool
@@ -26,6 +32,9 @@ postgresql:
   adapter: postgresql
   min_messages: warning
 
+postgis:
+  <<: *postgresql
+
 oracle:
   <<: *common
   adapter: oracle
@@ -38,3 +47,6 @@ sqlite:
 sqlite3:
   adapter: sqlite3
   database: test.db
+
+spatialite:
+  <<: *sqlite3

test/import_test.rb

@@ -134,8 +134,18 @@ describe "#import" do
       let(:new_topics) { Build(3, :topics) }
 
       it "reloads data for existing in-memory instances" do
-        Topic.import(new_topics, :synchronize => new_topics, :synchronize_key => [:title] )
+        Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] )
-        assert new_topics.all?(&:new_record?), "Records should have been reloaded"
+        assert new_topics.all?(&:persisted?), "Records should have been reloaded"
+      end      
+    end
+
+    context "synchronizing on destroyed records with explicit conditions" do
+      let(:new_topics) { Generate(3, :topics) }
+
+      it "reloads data for existing in-memory instances" do
+        new_topics.each &:destroy
+        Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] )
+        assert new_topics.all?(&:persisted?), "Records should have been reloaded"
       end      
     end
   end
@@ -293,4 +303,18 @@ describe "#import" do
       assert_equal "2010/05/14".to_date, Topic.last.last_read.to_date
     end
   end
+
+  context "importing through an association scope" do
+    [ true, false ].each do |b|
+      context "when validation is " + (b ? "enabled" : "disabled") do
+        it "should automatically set the foreign key column" do
+          books = [[ "David Chelimsky", "The RSpec Book" ], [ "Chad Fowler", "Rails Recipes" ]]
+          topic = Factory.create :topic
+          topic.books.import [ :author_name, :title ], books, :validate => b
+          assert_equal 2, topic.books.count
+          assert topic.books.all? { |b| b.topic_id == topic.id }
+        end
+      end
+    end
+  end
 end

test/mysqlspatial/import_test.rb

@@ -0,0 +1,6 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions')
+require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples')
+
+should_support_mysql_import_functionality

test/mysqlspatial2/import_test.rb

@@ -0,0 +1,6 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+
+require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions')
+require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples')
+
+should_support_mysql_import_functionality

test/postgis/import_test.rb

@@ -0,0 +1,4 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
+
+should_support_postgresql_import_functionality

test/postgresql/import_test.rb

@@ -1,20 +1,4 @@
-require File.expand_path('../../test_helper', __FILE__)
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
 
-describe "#supports_imports?" do
+should_support_postgresql_import_functionality
-  it "should support import" do
-    assert ActiveRecord::Base.supports_import?
-  end
-end
-
-describe "#import" do
-  it "should import with a single insert" do
-    # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics
-    assert_difference "Topic.count", +10 do
-      result = Topic.import Build(3, :topics)
-      assert_equal 1, result.num_inserts
-    
-      result = Topic.import Build(7, :topics)
-      assert_equal 1, result.num_inserts
-    end
-  end
-end

test/support/postgresql/import_examples.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+def should_support_postgresql_import_functionality
+  describe "#supports_imports?" do
+    it "should support import" do
+      assert ActiveRecord::Base.supports_import?
+    end
+  end
+
+  describe "#import" do
+    it "should import with a single insert" do
+      # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics
+      assert_difference "Topic.count", +10 do
+        result = Topic.import Build(3, :topics)
+        assert_equal 1, result.num_inserts
+
+        result = Topic.import Build(7, :topics)
+        assert_equal 1, result.num_inserts
+      end
+    end
+  end
+end

test/test_helper.rb

@@ -44,3 +44,8 @@ adapter_schema = test_dir.join("schema/#{adapter}_schema.rb")
 require adapter_schema if File.exists?(adapter_schema)
 
 Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file }
+
+# Prevent this deprecation warning from breaking the tests.
+module Rake::DeprecatedObjectDSL
+  remove_method :import
+end