Compare commits
56 commits
Author | SHA1 | Date | |
---|---|---|---|
038a29682e | |||
1c675d7c31 | |||
6697e2b4db | |||
eec20355e5 | |||
5f0e220038 | |||
d0c1055b65 | |||
9fd19a36bd | |||
413e35fedd | |||
31ff20a332 | |||
58cac8bfb0 | |||
082a95df0e | |||
1ae5122642 | |||
ec2059e802 | |||
a4f17e6d45 | |||
394e0b51c3 | |||
ee5604d4d5 | |||
43bd628e05 | |||
22473145d1 | |||
c08d31b528 | |||
b03af0cd3e | |||
6182b944ef | |||
0b03543612 | |||
b4bc1466f1 | |||
75a63d77d1 | |||
0e14c5f643 | |||
190fcd32ff | |||
1d9b30cf83 | |||
23a07371dd | |||
151c2f271f | |||
7628e0a9d8 | |||
de048bb61e | |||
9ee02d87ff | |||
cc0185abb3 | |||
2bb1150e00 | |||
f12d72b0b1 | |||
a1a183e43d | |||
b500ebe5a1 | |||
71a393d000 | |||
e12f9c4951 | |||
eac7d5f949 | |||
f74bd2fbbd | |||
b36259af29 | |||
e6cafdb23a | |||
3dc99c357d | |||
8544641260 | |||
41952485a4 | |||
b5c550860c | |||
a5a9928b26 | |||
7fa043cf31 | |||
c9a4313f8b | |||
b2c71336ca | |||
2539a71574 | |||
b5261d0225 | |||
a41050c2b5 | |||
08c36d33e4 | |||
8d20594f12 |
8
Gemfile
8
Gemfile
|
@ -1,6 +1,6 @@
|
|||
source :gemcutter
|
||||
|
||||
gem "activerecord", "~> 3.0pre"
|
||||
gem "activerecord", "~> 3.0"
|
||||
|
||||
group :development do
|
||||
gem "rake"
|
||||
|
@ -11,9 +11,10 @@ group :test do
|
|||
# Database Adapters
|
||||
platforms :ruby do
|
||||
gem "mysql", "~> 2.8.1"
|
||||
gem "mysql2", "~> 0.2.4"
|
||||
gem "mysql2", "~> 0.3.0"
|
||||
gem "pg", "~> 0.9"
|
||||
gem "sqlite3-ruby", "~> 1.3.1"
|
||||
gem "seamless_database_pool", "~> 1.0.11"
|
||||
end
|
||||
|
||||
platforms :jruby do
|
||||
|
@ -36,7 +37,6 @@ group :test do
|
|||
end
|
||||
|
||||
platforms :mri_19 do
|
||||
# TODO: Remove the conditional when ruby-debug19 supports Ruby >= 1.9.3
|
||||
gem "ruby-debug19" if RUBY_VERSION < "1.9.3"
|
||||
gem "debugger"
|
||||
end
|
||||
end
|
||||
|
|
87
Gemfile.lock
87
Gemfile.lock
|
@ -1,84 +1,85 @@
|
|||
GEM
|
||||
remote: http://rubygems.org/
|
||||
specs:
|
||||
activemodel (3.1.0.rc4)
|
||||
activesupport (= 3.1.0.rc4)
|
||||
bcrypt-ruby (~> 2.1.4)
|
||||
activemodel (3.2.7)
|
||||
activesupport (= 3.2.7)
|
||||
builder (~> 3.0.0)
|
||||
i18n (~> 0.6)
|
||||
activerecord (3.1.0.rc4)
|
||||
activemodel (= 3.1.0.rc4)
|
||||
activesupport (= 3.1.0.rc4)
|
||||
arel (~> 2.1.1)
|
||||
tzinfo (~> 0.3.27)
|
||||
activerecord-jdbc-adapter (1.1.1)
|
||||
activerecord-jdbcmysql-adapter (1.1.1)
|
||||
activerecord-jdbc-adapter (= 1.1.1)
|
||||
activerecord (3.2.7)
|
||||
activemodel (= 3.2.7)
|
||||
activesupport (= 3.2.7)
|
||||
arel (~> 3.0.2)
|
||||
tzinfo (~> 0.3.29)
|
||||
activerecord-jdbc-adapter (1.2.2)
|
||||
activerecord-jdbcmysql-adapter (1.2.2)
|
||||
activerecord-jdbc-adapter (~> 1.2.2)
|
||||
jdbc-mysql (~> 5.1.0)
|
||||
activesupport (3.1.0.rc4)
|
||||
activesupport (3.2.7)
|
||||
i18n (~> 0.6)
|
||||
multi_json (~> 1.0)
|
||||
archive-tar-minitar (0.5.2)
|
||||
arel (2.1.1)
|
||||
bcrypt-ruby (2.1.4)
|
||||
bcrypt-ruby (2.1.4-java)
|
||||
arel (3.0.2)
|
||||
builder (3.0.0)
|
||||
chronic (0.3.0)
|
||||
columnize (0.3.2)
|
||||
chronic (0.7.0)
|
||||
columnize (0.3.6)
|
||||
debugger (1.2.0)
|
||||
columnize (>= 0.3.1)
|
||||
debugger-linecache (~> 1.1.1)
|
||||
debugger-ruby_core_source (~> 1.1.3)
|
||||
debugger-linecache (1.1.2)
|
||||
debugger-ruby_core_source (>= 1.1.1)
|
||||
debugger-ruby_core_source (1.1.3)
|
||||
delorean (0.2.1)
|
||||
chronic
|
||||
factory_girl (1.3.3)
|
||||
git (1.2.5)
|
||||
i18n (0.6.0)
|
||||
jdbc-mysql (5.1.13)
|
||||
jeweler (1.5.2)
|
||||
bundler (~> 1.0.0)
|
||||
jeweler (1.8.4)
|
||||
bundler (~> 1.0)
|
||||
git (>= 1.2.5)
|
||||
rake
|
||||
linecache (0.43)
|
||||
linecache19 (0.5.12)
|
||||
ruby_core_source (>= 0.1.4)
|
||||
multi_json (1.0.3)
|
||||
rdoc
|
||||
json (1.7.4)
|
||||
json (1.7.4-java)
|
||||
linecache (0.46)
|
||||
rbx-require-relative (> 0.0.4)
|
||||
multi_json (1.3.6)
|
||||
mysql (2.8.1)
|
||||
mysql2 (0.2.7)
|
||||
pg (0.11.0)
|
||||
rake (0.8.7)
|
||||
mysql2 (0.3.11)
|
||||
pg (0.14.0)
|
||||
rake (0.9.2.2)
|
||||
rbx-require-relative (0.0.9)
|
||||
rdoc (3.12)
|
||||
json (~> 1.4)
|
||||
ruby-debug (0.10.4)
|
||||
columnize (>= 0.1)
|
||||
ruby-debug-base (~> 0.10.4.0)
|
||||
ruby-debug-base (0.10.4)
|
||||
linecache (>= 0.3)
|
||||
ruby-debug-base (0.10.4-java)
|
||||
ruby-debug-base19 (0.11.25)
|
||||
columnize (>= 0.3.1)
|
||||
linecache19 (>= 0.5.11)
|
||||
ruby_core_source (>= 0.1.4)
|
||||
ruby-debug19 (0.11.6)
|
||||
columnize (>= 0.3.1)
|
||||
linecache19 (>= 0.5.11)
|
||||
ruby-debug-base19 (>= 0.11.19)
|
||||
ruby_core_source (0.1.5)
|
||||
archive-tar-minitar (>= 0.5.2)
|
||||
sqlite3 (1.3.3)
|
||||
seamless_database_pool (1.0.11)
|
||||
activerecord (>= 2.2.2)
|
||||
sqlite3 (1.3.6)
|
||||
sqlite3-ruby (1.3.3)
|
||||
sqlite3 (>= 1.3.3)
|
||||
tzinfo (0.3.28)
|
||||
tzinfo (0.3.33)
|
||||
|
||||
PLATFORMS
|
||||
java
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
activerecord (~> 3.0pre)
|
||||
activerecord (~> 3.0)
|
||||
activerecord-jdbcmysql-adapter
|
||||
debugger
|
||||
delorean (~> 0.2.0)
|
||||
factory_girl (~> 1.3.3)
|
||||
jdbc-mysql
|
||||
jeweler (>= 1.4.0)
|
||||
mysql (~> 2.8.1)
|
||||
mysql2 (~> 0.2.4)
|
||||
mysql2 (~> 0.3.0)
|
||||
pg (~> 0.9)
|
||||
rake
|
||||
ruby-debug (= 0.10.4)
|
||||
ruby-debug-base (= 0.10.4)
|
||||
ruby-debug19
|
||||
seamless_database_pool (~> 1.0.11)
|
||||
sqlite3-ruby (~> 1.3.1)
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
activerecord-import is a library for bulk inserting data using ActiveRecord.
|
||||
|
||||
For more information on activerecord-import please see its wiki: http://wiki.github.com/zdennis/activerecord-import/
|
||||
For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
|
||||
|
||||
# License
|
||||
|
||||
|
|
4
Rakefile
4
Rakefile
|
@ -36,7 +36,7 @@ namespace :display do
|
|||
end
|
||||
task :default => ["display:notice"]
|
||||
|
||||
ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3)
|
||||
ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool mysqlspatial mysql2spatial spatialite postgis)
|
||||
ADAPTERS.each do |adapter|
|
||||
namespace :test do
|
||||
desc "Runs #{adapter} database tests."
|
||||
|
@ -63,7 +63,7 @@ rescue LoadError
|
|||
end
|
||||
end
|
||||
|
||||
require 'rake/rdoctask'
|
||||
require 'rdoc/task'
|
||||
Rake::RDocTask.new do |rdoc|
|
||||
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
||||
|
||||
|
|
68
activerecord-import.gemspec
Normal file
68
activerecord-import.gemspec
Normal file
|
@ -0,0 +1,68 @@
|
|||
# Generated by jeweler
|
||||
# DO NOT EDIT THIS FILE DIRECTLY
|
||||
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
||||
# -*- encoding: utf-8 -*-
|
||||
|
||||
Gem::Specification.new do |s|
|
||||
s.name = "activerecord-import"
|
||||
s.version = "0.2.10"
|
||||
|
||||
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
||||
s.authors = ["Zach Dennis"]
|
||||
s.date = "2012-08-30"
|
||||
s.description = "Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond"
|
||||
s.email = "zach.dennis@gmail.com"
|
||||
s.extra_rdoc_files = [
|
||||
"README.markdown"
|
||||
]
|
||||
s.files = [
|
||||
"README.markdown",
|
||||
"Rakefile",
|
||||
"VERSION",
|
||||
"lib/activerecord-import.rb",
|
||||
"lib/activerecord-import/active_record/adapters/abstract_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/mysql2_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/mysql_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/postgresql_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb",
|
||||
"lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb",
|
||||
"lib/activerecord-import/adapters/abstract_adapter.rb",
|
||||
"lib/activerecord-import/adapters/mysql_adapter.rb",
|
||||
"lib/activerecord-import/adapters/postgresql_adapter.rb",
|
||||
"lib/activerecord-import/adapters/sqlite3_adapter.rb",
|
||||
"lib/activerecord-import/base.rb",
|
||||
"lib/activerecord-import/import.rb",
|
||||
"lib/activerecord-import/mysql.rb",
|
||||
"lib/activerecord-import/mysql2.rb",
|
||||
"lib/activerecord-import/postgresql.rb",
|
||||
"lib/activerecord-import/sqlite3.rb",
|
||||
"lib/activerecord-import/synchronize.rb"
|
||||
]
|
||||
s.homepage = "http://github.com/zdennis/activerecord-import"
|
||||
s.require_paths = ["lib"]
|
||||
s.rubygems_version = "1.8.24"
|
||||
s.summary = "Bulk-loading extension for ActiveRecord"
|
||||
|
||||
if s.respond_to? :specification_version then
|
||||
s.specification_version = 3
|
||||
|
||||
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
||||
s.add_runtime_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
s.add_development_dependency(%q<rake>, [">= 0"])
|
||||
s.add_development_dependency(%q<jeweler>, [">= 1.4.0"])
|
||||
s.add_runtime_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
else
|
||||
s.add_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
s.add_dependency(%q<rake>, [">= 0"])
|
||||
s.add_dependency(%q<jeweler>, [">= 1.4.0"])
|
||||
s.add_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
end
|
||||
else
|
||||
s.add_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
s.add_dependency(%q<rake>, [">= 0"])
|
||||
s.add_dependency(%q<jeweler>, [">= 1.4.0"])
|
||||
s.add_dependency(%q<activerecord>, ["~> 3.0"])
|
||||
end
|
||||
end
|
||||
|
|
@ -2,15 +2,15 @@ class ActiveRecord::Base
|
|||
class << self
|
||||
def establish_connection_with_activerecord_import(*args)
|
||||
establish_connection_without_activerecord_import(*args)
|
||||
ActiveSupport.run_load_hooks(:active_record_connection_established, connection)
|
||||
ActiveSupport.run_load_hooks(:active_record_connection_established, connection_pool)
|
||||
end
|
||||
alias_method_chain :establish_connection, :activerecord_import
|
||||
end
|
||||
end
|
||||
|
||||
ActiveSupport.on_load(:active_record_connection_established) do |connection|
|
||||
if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection)
|
||||
ActiveSupport.on_load(:active_record_connection_established) do |connection_pool|
|
||||
if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
|
||||
require File.join File.dirname(__FILE__), "activerecord-import/base"
|
||||
end
|
||||
ActiveRecord::Import.load_from_connection connection
|
||||
ActiveRecord::Import.load_from_connection_pool connection_pool
|
||||
end
|
||||
|
|
|
@ -2,5 +2,5 @@ require "active_record/connection_adapters/mysql_adapter"
|
|||
require "activerecord-import/adapters/mysql_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::MysqlAdapter
|
||||
include ActiveRecord::Import::MysqlAdapter::InstanceMethods
|
||||
include ActiveRecord::Import::MysqlAdapter
|
||||
end
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
require "active_record/connection_adapters/mysql2_adapter"
|
||||
require "activerecord-import/adapters/mysql_adapter"
|
||||
require "activerecord-import/adapters/mysql2_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
||||
include ActiveRecord::Import::MysqlAdapter::InstanceMethods
|
||||
include ActiveRecord::Import::Mysql2Adapter
|
||||
end
|
|
@ -2,5 +2,5 @@ require "active_record/connection_adapters/mysql_adapter"
|
|||
require "activerecord-import/adapters/mysql_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::MysqlAdapter
|
||||
include ActiveRecord::Import::MysqlAdapter::InstanceMethods
|
||||
include ActiveRecord::Import::MysqlAdapter
|
||||
end
|
|
@ -2,6 +2,6 @@ require "active_record/connection_adapters/postgresql_adapter"
|
|||
require "activerecord-import/adapters/postgresql_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
|
||||
include ActiveRecord::Import::PostgreSQLAdapter::InstanceMethods
|
||||
include ActiveRecord::Import::PostgreSQLAdapter
|
||||
end
|
||||
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
require "seamless_database_pool"
|
||||
require "active_record/connection_adapters/seamless_database_pool_adapter"
|
||||
require "activerecord-import/adapters/mysql_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter
|
||||
include ActiveRecord::Import::MysqlAdapter
|
||||
end
|
|
@ -1,7 +1,7 @@
|
|||
require "active_record/connection_adapters/sqlite3_adapter"
|
||||
require "activerecord-import/adapters/sqlite3_adapter"
|
||||
|
||||
class ActiveRecord::ConnectionAdapters::Sqlite3Adapter
|
||||
include ActiveRecord::Import::Sqlite3Adapter::InstanceMethods
|
||||
class ActiveRecord::ConnectionAdapters::SQLite3Adapter
|
||||
include ActiveRecord::Import::SQLite3Adapter
|
||||
end
|
||||
|
||||
|
|
|
@ -3,12 +3,6 @@ module ActiveRecord::Import::AbstractAdapter
|
|||
QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
||||
|
||||
module ClassMethods
|
||||
# Returns the sum of the sizes of the passed in objects. This should
|
||||
# probably be moved outside this class, but to where?
|
||||
def sum_sizes( *objects ) # :nodoc:
|
||||
objects.inject( 0 ){ |sum,o| sum += o.bytesize }
|
||||
end
|
||||
|
||||
def get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc:
|
||||
value_sets = []
|
||||
arr, current_arr_values_size, current_size = [], 0, 0
|
||||
|
@ -40,7 +34,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|||
# elements that are in position >= 1 will be appended to the final SQL.
|
||||
def insert_many( sql, values, *args ) # :nodoc:
|
||||
# the number of inserts default
|
||||
number_of_inserts = 0
|
||||
number_of_inserts, last_inserted_id = 0, nil
|
||||
|
||||
base_sql,post_sql = if sql.is_a?( String )
|
||||
[ sql, '' ]
|
||||
|
@ -51,7 +45,7 @@ module ActiveRecord::Import::AbstractAdapter
|
|||
sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
|
||||
|
||||
# the number of bytes the requested insert statement values will take up
|
||||
values_in_bytes = self.class.sum_sizes( *values )
|
||||
values_in_bytes = values.sum {|value| value.bytesize }
|
||||
|
||||
# the number of bytes (commas) it will take to comma separate our values
|
||||
comma_separated_bytes = values.size-1
|
||||
|
@ -65,17 +59,17 @@ module ActiveRecord::Import::AbstractAdapter
|
|||
if NO_MAX_PACKET == max or total_bytes < max
|
||||
number_of_inserts += 1
|
||||
sql2insert = base_sql + values.join( ',' ) + post_sql
|
||||
insert( sql2insert, *args )
|
||||
last_inserted_id = insert( sql2insert, *args )
|
||||
else
|
||||
value_sets = self.class.get_insert_value_sets( values, sql_size, max )
|
||||
value_sets.each do |values|
|
||||
number_of_inserts += 1
|
||||
sql2insert = base_sql + values.join( ',' ) + post_sql
|
||||
insert( sql2insert, *args )
|
||||
last_inserted_id = insert( sql2insert, *args )
|
||||
end
|
||||
end
|
||||
|
||||
number_of_inserts
|
||||
[number_of_inserts, last_inserted_id]
|
||||
end
|
||||
|
||||
def pre_sql_statements(options)
|
||||
|
|
5
lib/activerecord-import/adapters/mysql2_adapter.rb
Normal file
5
lib/activerecord-import/adapters/mysql2_adapter.rb
Normal file
|
@ -0,0 +1,5 @@
|
|||
require File.dirname(__FILE__) + "/mysql_adapter"
|
||||
|
||||
module ActiveRecord::Import::Mysql2Adapter
|
||||
include ActiveRecord::Import::MysqlAdapter
|
||||
end
|
|
@ -1,20 +1,17 @@
|
|||
module ActiveRecord::Import::MysqlAdapter
|
||||
module InstanceMethods
|
||||
def self.included(klass)
|
||||
klass.instance_eval do
|
||||
include ActiveRecord::Import::ImportSupport
|
||||
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
||||
end
|
||||
end
|
||||
|
||||
# Returns the maximum number of bytes that the server will allow
|
||||
# in a single packet
|
||||
def max_allowed_packet # :nodoc:
|
||||
@max_allowed_packet ||= begin
|
||||
result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
|
||||
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
||||
val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
|
||||
val.to_i
|
||||
end
|
||||
end
|
||||
|
||||
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
||||
# in +args+.
|
||||
|
@ -55,5 +52,4 @@ module ActiveRecord::Import::MysqlAdapter
|
|||
def duplicate_key_update_error?(exception)# :nodoc:
|
||||
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
||||
end
|
||||
end
|
||||
end
|
|
@ -1,13 +1,7 @@
|
|||
module ActiveRecord::Import::PostgreSQLAdapter
|
||||
module InstanceMethods
|
||||
def self.included(klass)
|
||||
klass.instance_eval do
|
||||
include ActiveRecord::Import::ImportSupport
|
||||
end
|
||||
end
|
||||
|
||||
def next_value_for_sequence(sequence_name)
|
||||
%{nextval('#{sequence_name}')}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,7 +1,5 @@
|
|||
module ActiveRecord::Import::Sqlite3Adapter
|
||||
module InstanceMethods
|
||||
module ActiveRecord::Import::SQLite3Adapter
|
||||
def next_value_for_sequence(sequence_name)
|
||||
%{nextval('#{sequence_name}')}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -5,19 +5,25 @@ require "active_record/version"
|
|||
module ActiveRecord::Import
|
||||
AdapterPath = File.join File.expand_path(File.dirname(__FILE__)), "/active_record/adapters"
|
||||
|
||||
def self.base_adapter(adapter)
|
||||
case adapter
|
||||
when 'mysqlspatial' then 'mysql'
|
||||
when 'mysql2spatial' then 'mysql2'
|
||||
when 'spatialite' then 'sqlite3'
|
||||
when 'postgis' then 'postgresql'
|
||||
else adapter
|
||||
end
|
||||
end
|
||||
|
||||
# Loads the import functionality for a specific database adapter
|
||||
def self.require_adapter(adapter)
|
||||
require File.join(AdapterPath,"/abstract_adapter")
|
||||
require File.join(AdapterPath,"/#{adapter}_adapter")
|
||||
require File.join(AdapterPath,"/#{base_adapter(adapter)}_adapter")
|
||||
end
|
||||
|
||||
# Loads the import functionality for the passed in ActiveRecord connection
|
||||
def self.load_from_connection(connection)
|
||||
import_adapter = "ActiveRecord::Import::#{connection.class.name.demodulize}::InstanceMethods"
|
||||
unless connection.class.ancestors.map(&:name).include?(import_adapter)
|
||||
config = connection.instance_variable_get :@config
|
||||
require_adapter config[:adapter]
|
||||
end
|
||||
def self.load_from_connection_pool(connection_pool)
|
||||
require_adapter connection_pool.spec.config[:adapter]
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@ require "ostruct"
|
|||
module ActiveRecord::Import::ConnectionAdapters ; end
|
||||
|
||||
module ActiveRecord::Import #:nodoc:
|
||||
class Result < Struct.new(:failed_instances, :num_inserts)
|
||||
class Result < Struct.new(:failed_instances, :num_inserts, :last_inserted_id)
|
||||
end
|
||||
|
||||
module ImportSupport #:nodoc:
|
||||
|
@ -17,6 +17,12 @@ module ActiveRecord::Import #:nodoc:
|
|||
true
|
||||
end
|
||||
end
|
||||
|
||||
class MissingColumnError < StandardError
|
||||
def initialize(name, index)
|
||||
super "Missing column for value <#{name}> at index #{index}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
class ActiveRecord::Base
|
||||
|
@ -131,8 +137,8 @@ class ActiveRecord::Base
|
|||
#
|
||||
# # Example synchronizing unsaved/new instances in memory by using a uniqued imported field
|
||||
# posts = [BlogPost.new(:title => "Foo"), BlogPost.new(:title => "Bar")]
|
||||
# BlogPost.import posts, :synchronize => posts
|
||||
# puts posts.first.new_record? # => false
|
||||
# BlogPost.import posts, :synchronize => posts, :synchronize_keys => [:title]
|
||||
# puts posts.first.persisted? # => true
|
||||
#
|
||||
# == On Duplicate Key Update (MySQL only)
|
||||
#
|
||||
|
@ -156,9 +162,10 @@ class ActiveRecord::Base
|
|||
# BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title }
|
||||
#
|
||||
# = Returns
|
||||
# This returns an object which responds to +failed_instances+ and +num_inserts+.
|
||||
# This returns an object which responds to +failed_instances+, +num_inserts+, +last_inserted_id+.
|
||||
# * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
|
||||
# * num_inserts - the number of insert statements it took to import the data
|
||||
# * num_inserts - the number of insert statements it took to import the data.
|
||||
# * last_inserted_id - the last inserted id. Should be the id of the latest inserted row.
|
||||
def import( *args )
|
||||
options = { :validate=>true, :timestamps=>true }
|
||||
options.merge!( args.pop ) if args.last.is_a? Hash
|
||||
|
@ -185,7 +192,7 @@ class ActiveRecord::Base
|
|||
end
|
||||
# supports empty array
|
||||
elsif args.last.is_a?( Array ) and args.last.empty?
|
||||
return ActiveRecord::Import::Result.new([], 0) if args.last.empty?
|
||||
return ActiveRecord::Import::Result.new([], 0, nil) if args.last.empty?
|
||||
# supports 2-element array and array
|
||||
elsif args.size == 2 and args.first.is_a?( Array ) and args.last.is_a?( Array )
|
||||
column_names, array_of_attributes = args
|
||||
|
@ -212,8 +219,8 @@ class ActiveRecord::Base
|
|||
return_obj = if is_validating
|
||||
import_with_validations( column_names, array_of_attributes, options )
|
||||
else
|
||||
num_inserts = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
||||
ActiveRecord::Import::Result.new([], num_inserts)
|
||||
[num_inserts, last_inserted_id] = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
||||
ActiveRecord::Import::Result.new([], num_inserts, last_inserted_id)
|
||||
end
|
||||
|
||||
if options[:synchronize]
|
||||
|
@ -255,8 +262,12 @@ class ActiveRecord::Base
|
|||
end
|
||||
array_of_attributes.compact!
|
||||
|
||||
num_inserts = array_of_attributes.empty? ? 0 : import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
||||
ActiveRecord::Import::Result.new(failed_instances, num_inserts)
|
||||
num_inserts, last_inserted_id = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any?
|
||||
[0, nil]
|
||||
else
|
||||
import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
||||
end
|
||||
ActiveRecord::Import::Result.new(failed_instances, num_inserts, last_inserted_id)
|
||||
end
|
||||
|
||||
# Imports the passed in +column_names+ and +array_of_attributes+
|
||||
|
@ -266,7 +277,21 @@ class ActiveRecord::Base
|
|||
# information on +column_names+, +array_of_attributes_ and
|
||||
# +options+.
|
||||
def import_without_validations_or_callbacks( column_names, array_of_attributes, options={} )
|
||||
columns = column_names.map { |name| columns_hash[name.to_s] }
|
||||
number_inserted, last_inserted_id = 0, nil
|
||||
scope_columns, scope_values = scope_attributes.to_a.transpose
|
||||
|
||||
unless scope_columns.blank?
|
||||
column_names.concat scope_columns
|
||||
array_of_attributes.each { |a| a.concat scope_values }
|
||||
end
|
||||
|
||||
columns = column_names.each_with_index.map do |name, i|
|
||||
column = columns_hash[name.to_s]
|
||||
|
||||
raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil?
|
||||
|
||||
column
|
||||
end
|
||||
|
||||
columns_sql = "(#{column_names.map{|name| connection.quote_column_name(name) }.join(',')})"
|
||||
insert_sql = "INSERT #{options[:ignore] ? 'IGNORE ':''}INTO #{quoted_table_name} #{columns_sql} VALUES "
|
||||
|
@ -282,11 +307,11 @@ class ActiveRecord::Base
|
|||
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
||||
|
||||
# perform the inserts
|
||||
number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten,
|
||||
number_inserted, last_inserted_id = connection.insert_many( [ insert_sql, post_sql_statements ].flatten,
|
||||
values_sql,
|
||||
"#{self.class.name} Create Many Without Validations Or Callbacks" )
|
||||
end
|
||||
number_inserted
|
||||
[number_inserted, last_inserted_id]
|
||||
end
|
||||
|
||||
private
|
||||
|
@ -294,13 +319,22 @@ class ActiveRecord::Base
|
|||
# Returns SQL the VALUES for an INSERT statement given the passed in +columns+
|
||||
# and +array_of_attributes+.
|
||||
def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc:
|
||||
# connection gets called a *lot* in this high intensity loop.
|
||||
# Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports)
|
||||
connection_memo = connection
|
||||
array_of_attributes.map do |arr|
|
||||
my_values = arr.each_with_index.map do |val,j|
|
||||
column = columns[j]
|
||||
if !sequence_name.blank? && column.name == primary_key && val.nil?
|
||||
connection.next_value_for_sequence(sequence_name)
|
||||
|
||||
# be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
|
||||
if val.nil? && column.name == primary_key && !sequence_name.blank?
|
||||
connection_memo.next_value_for_sequence(sequence_name)
|
||||
else
|
||||
connection.quote(column.type_cast(val), column)
|
||||
if serialized_attributes.include?(column.name)
|
||||
connection_memo.quote(serialized_attributes[column.name].dump(val), column)
|
||||
else
|
||||
connection_memo.quote(val, column)
|
||||
end
|
||||
end
|
||||
end
|
||||
"(#{my_values.join(',')})"
|
||||
|
|
|
@ -43,6 +43,10 @@ module ActiveRecord # :nodoc:
|
|||
instance.clear_aggregation_cache
|
||||
instance.clear_association_cache
|
||||
instance.instance_variable_set '@attributes', matched_instance.attributes
|
||||
# Since the instance now accurately reflects the record in
|
||||
# the database, ensure that instance.persisted? is true.
|
||||
instance.instance_variable_set '@new_record', false
|
||||
instance.instance_variable_set '@destroyed', false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -50,3 +50,13 @@ describe "ActiveRecord::ConnectionAdapter::AbstractAdapter" do
|
|||
end
|
||||
|
||||
end
|
||||
|
||||
describe "ActiveRecord::Import DB-specific adapter class" do
|
||||
context "when ActiveRecord::Import is in use" do
|
||||
it "should appear in the AR connection adapter class's ancestors" do
|
||||
connection = ActiveRecord::Base.connection
|
||||
import_class_name = 'ActiveRecord::Import::' + connection.class.name.demodulize
|
||||
assert_includes connection.class.ancestors, import_class_name.constantize
|
||||
end
|
||||
end
|
||||
end
|
1
test/adapters/mysql2spatial.rb
Normal file
1
test/adapters/mysql2spatial.rb
Normal file
|
@ -0,0 +1 @@
|
|||
ENV["ARE_DB"] = "mysql2spatial"
|
1
test/adapters/mysqlspatial.rb
Normal file
1
test/adapters/mysqlspatial.rb
Normal file
|
@ -0,0 +1 @@
|
|||
ENV["ARE_DB"] = "mysqlspatial"
|
1
test/adapters/postgis.rb
Normal file
1
test/adapters/postgis.rb
Normal file
|
@ -0,0 +1 @@
|
|||
ENV["ARE_DB"] = "postgis"
|
1
test/adapters/seamless_database_pool.rb
Normal file
1
test/adapters/seamless_database_pool.rb
Normal file
|
@ -0,0 +1 @@
|
|||
ENV["ARE_DB"] = "seamless_database_pool"
|
1
test/adapters/spatialite.rb
Normal file
1
test/adapters/spatialite.rb
Normal file
|
@ -0,0 +1 @@
|
|||
ENV["ARE_DB"] = "spatialite"
|
|
@ -13,12 +13,28 @@ mysql2:
|
|||
<<: *common
|
||||
adapter: mysql2
|
||||
|
||||
mysqlspatial:
|
||||
<<: *mysql
|
||||
|
||||
mysqlspatial2:
|
||||
<<: *mysql2
|
||||
|
||||
seamless_database_pool:
|
||||
<<: *common
|
||||
adapter: seamless_database_pool
|
||||
pool_adapter: mysql2
|
||||
master:
|
||||
host: localhost
|
||||
|
||||
postgresql:
|
||||
<<: *common
|
||||
username: postgres
|
||||
adapter: postgresql
|
||||
min_messages: warning
|
||||
|
||||
postgis:
|
||||
<<: *postgresql
|
||||
|
||||
oracle:
|
||||
<<: *common
|
||||
adapter: oracle
|
||||
|
@ -31,3 +47,6 @@ sqlite:
|
|||
sqlite3:
|
||||
adapter: sqlite3
|
||||
database: test.db
|
||||
|
||||
spatialite:
|
||||
<<: *sqlite3
|
||||
|
|
|
@ -19,6 +19,16 @@ describe "#import" do
|
|||
end
|
||||
end
|
||||
|
||||
describe "with non-default ActiveRecord models" do
|
||||
context "that have a non-standard primary key (that is no sequence)" do
|
||||
it "should import models successfully" do
|
||||
assert_difference "Widget.count", +3 do
|
||||
Widget.import Build(3, :widgets)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "with :validation option" do
|
||||
let(:columns) { %w(title author_name) }
|
||||
let(:valid_values) { [[ "LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
|
||||
|
@ -36,6 +46,12 @@ describe "#import" do
|
|||
result = Topic.import columns, invalid_values, :validate => false
|
||||
end
|
||||
end
|
||||
|
||||
it 'should raise a specific error if a column does not exist' do
|
||||
assert_raises ActiveRecord::Import::MissingColumnError do
|
||||
Topic.import ['foo'], [['bar']], :validate => false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "with validation checks turned on" do
|
||||
|
@ -66,6 +82,44 @@ describe "#import" do
|
|||
end
|
||||
end
|
||||
|
||||
context "with :all_or_none option" do
|
||||
let(:columns) { %w(title author_name) }
|
||||
let(:valid_values) { [[ "LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
|
||||
let(:invalid_values) { [[ "The RSpec Book", ""], ["Agile+UX", ""]] }
|
||||
let(:mixed_values) { valid_values + invalid_values }
|
||||
|
||||
context "with validation checks turned on" do
|
||||
it "should import valid data" do
|
||||
assert_difference "Topic.count", +2 do
|
||||
result = Topic.import columns, valid_values, :all_or_none => true
|
||||
end
|
||||
end
|
||||
|
||||
it "should not import invalid data" do
|
||||
assert_no_difference "Topic.count" do
|
||||
result = Topic.import columns, invalid_values, :all_or_none => true
|
||||
end
|
||||
end
|
||||
|
||||
it "should not import valid data when mixed with invalid data" do
|
||||
assert_no_difference "Topic.count" do
|
||||
result = Topic.import columns, mixed_values, :all_or_none => true
|
||||
end
|
||||
end
|
||||
|
||||
it "should report the failed instances" do
|
||||
results = Topic.import columns, mixed_values, :all_or_none => true
|
||||
assert_equal invalid_values.size, results.failed_instances.size
|
||||
results.failed_instances.each { |e| assert_kind_of Topic, e }
|
||||
end
|
||||
|
||||
it "should report the zero inserts" do
|
||||
results = Topic.import columns, mixed_values, :all_or_none => true
|
||||
assert_equal 0, results.num_inserts
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
context "with :synchronize option" do
|
||||
context "synchronizing on new records" do
|
||||
let(:new_topics) { Build(3, :topics) }
|
||||
|
@ -80,8 +134,18 @@ describe "#import" do
|
|||
let(:new_topics) { Build(3, :topics) }
|
||||
|
||||
it "reloads data for existing in-memory instances" do
|
||||
Topic.import(new_topics, :synchronize => new_topics, :synchronize_key => [:title] )
|
||||
assert new_topics.all?(&:new_record?), "Records should have been reloaded"
|
||||
Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] )
|
||||
assert new_topics.all?(&:persisted?), "Records should have been reloaded"
|
||||
end
|
||||
end
|
||||
|
||||
context "synchronizing on destroyed records with explicit conditions" do
|
||||
let(:new_topics) { Generate(3, :topics) }
|
||||
|
||||
it "reloads data for existing in-memory instances" do
|
||||
new_topics.each &:destroy
|
||||
Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] )
|
||||
assert new_topics.all?(&:persisted?), "Records should have been reloaded"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
@ -239,4 +303,18 @@ describe "#import" do
|
|||
assert_equal "2010/05/14".to_date, Topic.last.last_read.to_date
|
||||
end
|
||||
end
|
||||
|
||||
context "importing through an association scope" do
|
||||
[ true, false ].each do |b|
|
||||
context "when validation is " + (b ? "enabled" : "disabled") do
|
||||
it "should automatically set the foreign key column" do
|
||||
books = [[ "David Chelimsky", "The RSpec Book" ], [ "Chad Fowler", "Rails Recipes" ]]
|
||||
topic = Factory.create :topic
|
||||
topic.books.import [ :author_name, :title ], books, :validate => b
|
||||
assert_equal 2, topic.books.count
|
||||
assert topic.books.all? { |b| b.topic_id == topic.id }
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
3
test/models/widget.rb
Normal file
3
test/models/widget.rb
Normal file
|
@ -0,0 +1,3 @@
|
|||
class Widget < ActiveRecord::Base
|
||||
self.primary_key = :w_id
|
||||
end
|
6
test/mysqlspatial/import_test.rb
Normal file
6
test/mysqlspatial/import_test.rb
Normal file
|
@ -0,0 +1,6 @@
|
|||
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions')
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples')
|
||||
|
||||
should_support_mysql_import_functionality
|
6
test/mysqlspatial2/import_test.rb
Normal file
6
test/mysqlspatial2/import_test.rb
Normal file
|
@ -0,0 +1,6 @@
|
|||
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
||||
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions')
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples')
|
||||
|
||||
should_support_mysql_import_functionality
|
4
test/postgis/import_test.rb
Normal file
4
test/postgis/import_test.rb
Normal file
|
@ -0,0 +1,4 @@
|
|||
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
|
||||
|
||||
should_support_postgresql_import_functionality
|
|
@ -1,20 +1,4 @@
|
|||
require File.expand_path('../../test_helper', __FILE__)
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
||||
require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
|
||||
|
||||
describe "#supports_imports?" do
|
||||
it "should support import" do
|
||||
assert ActiveRecord::Base.supports_import?
|
||||
end
|
||||
end
|
||||
|
||||
describe "#import" do
|
||||
it "should import with a single insert" do
|
||||
# see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics
|
||||
assert_difference "Topic.count", +10 do
|
||||
result = Topic.import Build(3, :topics)
|
||||
assert_equal 1, result.num_inserts
|
||||
|
||||
result = Topic.import Build(7, :topics)
|
||||
assert_equal 1, result.num_inserts
|
||||
end
|
||||
end
|
||||
end
|
||||
should_support_postgresql_import_functionality
|
|
@ -95,4 +95,8 @@ ActiveRecord::Schema.define do
|
|||
end
|
||||
|
||||
add_index :animals, [:name], :unique => true, :name => 'uk_animals'
|
||||
|
||||
create_table :widgets, :id => false, :force => true do |t|
|
||||
t.integer :w_id
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,4 +1,10 @@
|
|||
class SchemaInfo < ActiveRecord::Base
|
||||
if respond_to?(:table_name=)
|
||||
self.table_name = 'schema_info'
|
||||
else
|
||||
# this is becoming deprecated in ActiveRecord but not all adapters supported it
|
||||
# at this time
|
||||
set_table_name 'schema_info'
|
||||
end
|
||||
VERSION = 12
|
||||
end
|
||||
|
|
|
@ -11,3 +11,7 @@ Factory.define :topic do |m|
|
|||
m.sequence(:title){ |n| "Title #{n}"}
|
||||
m.sequence(:author_name){ |n| "Author #{n}"}
|
||||
end
|
||||
|
||||
Factory.define :widget do |m|
|
||||
m.sequence(:w_id){ |n| n}
|
||||
end
|
||||
|
|
|
@ -9,7 +9,7 @@ def should_support_mysql_import_functionality
|
|||
"('7','8','9')" ]
|
||||
|
||||
adapter = ActiveRecord::Base.connection.class
|
||||
values_size_in_bytes = adapter.sum_sizes( *values )
|
||||
values_size_in_bytes = values.sum {|value| value.bytesize }
|
||||
base_sql_size_in_bytes = 15
|
||||
max_bytes = 30
|
||||
|
||||
|
@ -38,7 +38,7 @@ def should_support_mysql_import_functionality
|
|||
base_sql_size_in_bytes = 15
|
||||
max_bytes = 26
|
||||
|
||||
values_size_in_bytes = adapter.sum_sizes( *values )
|
||||
values_size_in_bytes = values.sum {|value| value.bytesize }
|
||||
value_sets = adapter.get_insert_value_sets( values, base_sql_size_in_bytes, max_bytes )
|
||||
|
||||
assert_equal 2, value_sets.size, 'Two value sets were expected!'
|
||||
|
|
21
test/support/postgresql/import_examples.rb
Normal file
21
test/support/postgresql/import_examples.rb
Normal file
|
@ -0,0 +1,21 @@
|
|||
# encoding: UTF-8
|
||||
def should_support_postgresql_import_functionality
|
||||
describe "#supports_imports?" do
|
||||
it "should support import" do
|
||||
assert ActiveRecord::Base.supports_import?
|
||||
end
|
||||
end
|
||||
|
||||
describe "#import" do
|
||||
it "should import with a single insert" do
|
||||
# see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics
|
||||
assert_difference "Topic.count", +10 do
|
||||
result = Topic.import Build(3, :topics)
|
||||
assert_equal 1, result.num_inserts
|
||||
|
||||
result = Topic.import Build(7, :topics)
|
||||
assert_equal 1, result.num_inserts
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
|
@ -44,3 +44,8 @@ adapter_schema = test_dir.join("schema/#{adapter}_schema.rb")
|
|||
require adapter_schema if File.exists?(adapter_schema)
|
||||
|
||||
Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file }
|
||||
|
||||
# Prevent this deprecation warning from breaking the tests.
|
||||
module Rake::DeprecatedObjectDSL
|
||||
remove_method :import
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue