From 12b34c8115bbb35524880bca4f21ad0939de4b77 Mon Sep 17 00:00:00 2001 From: Stefan Morgenthaler Date: Mon, 9 Jul 2012 23:45:43 +0200 Subject: [PATCH 1/2] add: rake task to backup/restore gitlab db and repos --- config/gitlab.yml.example | 2 + lib/tasks/gitlab/backup.rake | 197 +++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 lib/tasks/gitlab/backup.rake diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 12c28675..473be138 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -21,6 +21,8 @@ email: # Like default project limit for user etc app: default_projects_limit: 10 + # backup_path: "/vol/backups" # default: Rails.root + backups/ + # backup_keep_time: 7.days # default: 0 (forever) # diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake new file mode 100644 index 00000000..3735d5b2 --- /dev/null +++ b/lib/tasks/gitlab/backup.rake @@ -0,0 +1,197 @@ +require 'active_record/fixtures' + +# load config +GITLAB_OPTS = YAML.load_file("#{Rails.root}/config/gitlab.yml")['app'] + +# backup path options +backup_path = GITLAB_OPTS['backup_path'].nil? ? "backups/" : GITLAB_OPTS['backup_path'] # set default path if not specified in settings. +backup_path = /^\//.match(backup_path) ? backup_path : File.join(Rails.root + backup_path) # check for absolut or relative path. +FileUtils.mkdir_p(backup_path) until Dir.exists?(backup_path) + +# keep last n backups +backup_keep_time = GITLAB_OPTS['backup_keep_time'].nil? ? 0 : eval(GITLAB_OPTS['backup_keep_time']) + +backup_path_repo = File.join(backup_path, "repositories") +backup_path_db = File.join(backup_path, "db") + +namespace :gitlab do + namespace :app do + + # Create backup of gitlab system + desc "GITLAB | Create a backup of the gitlab system" + task :backup_create => :environment do + + Dir.chdir(backup_path) + + Rake::Task["gitlab:app:db_dump"].invoke + Rake::Task["gitlab:app:repo_dump"].invoke + + # saving additional informations + s = Hash.new + s["db_version"] = "#{ActiveRecord::Migrator.current_version}" + s["backup_created_at"] = "#{Time.now}" + s["gitlab_version"] = %x{git rev-parse HEAD}.gsub(/\n/,"") + s["tar_version"] = %x{tar --version | head -1}.gsub(/\n/,"") + + File.open("#{backup_path}/backup_information.yml", "w+") do |file| + file << s.to_yaml.gsub(/^---\n/,'') + end + + # create archive + print "Creating backup archive: #{Time.now.to_i}_gitlab_backup.tar " + if Kernel.system("tar -cf #{Time.now.to_i}_gitlab_backup.tar repositories/ db/ backup_information.yml") + puts "[DONE]".green + else + puts "[FAILED]".red + end + + # cleanup: remove tmp files + print "Deletion of tmp directories..." + if Kernel.system("rm -rf repositories/ db/ backup_information.yml") + puts "[DONE]".green + else + puts "[FAILED]".red + end + + # delete backups + print "Deleting old backups... " + if backup_keep_time > 0 + file_list = Dir.glob("*_gitlab_backup.tar").map { |f| f.split(/_/).first.to_i } + file_list.sort.each do |timestamp| + if Time.at(timestamp) < (Time.now - backup_keep_time) + %x{rm #{timestamp}_gitlab_backup.tar} + end + end + puts "[DONE]".green + else + puts "[SKIPPING]".yellow + end + + end + + + desc "GITLAB | Restore a previously created backup" + task :backup_restore => :environment do + + Dir.chdir(backup_path) + + file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } + puts "no backup found" if file_list.count == 0 + if file_list.count > 1 && ENV["BACKUP"].nil? + puts "Found more than one backup, please specify which one you want to restore:" + puts "rake gitlab:app:backup_restore BACKUP=timestamp_of_backup" + exit 1; + end + + tar_file = ENV["BACKUP"].nil? ? File.join(file_list.first.to_s + "_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") + + if ! File.exists?(tar_file) + puts "The specified backup doesn't exist!" + end + + print "Unpacking backup... " + if ! Kernel.system("tar -xf #{tar_file}") + puts "[FAILED]".red + exit 1 + else + puts "[DONE]".green + end + + settings = YAML.load_file("backup_information.yml") + ENV["VERSION"] = "#{settings["db_version"]}" if settings["db_version"].to_i > 0 + + # restoring mismatching backups can lead to unexpected problems + if settings["gitlab_version"] != %x{git rev-parse HEAD}.gsub(/\n/,"") + puts "gitlab_version mismatch:".red + puts " Your current HEAD differs from the HEAD in the backup!".red + puts " Please switch to the following revision and try again:".red + puts " revision: #{settings["gitlab_version"]}".red + exit 1 + end + + Rake::Task["gitlab:app:db_restore"].invoke + Rake::Task["gitlab:app:repo_restore"].invoke + + # cleanup: remove tmp files + print "Deletion of tmp directories..." + if Kernel.system("rm -rf repositories/ db/ backup_information.yml") + puts "[DONE]".green + else + puts "[FAILED]".red + end + + end + + + ################################################################################ + ################################# invoked tasks ################################ + + ################################# REPOSITORIES ################################# + + task :repo_dump => :environment do + FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo) + puts "Dumping repositories:" + project = Project.all.map { |n| [n.name,n.path_to_repo] } + project << ["gitolite-admin.git", File.join(File.dirname(project.first.second), "gitolite-admin.git")] + project.each do |project| + print "- Dumping repository #{project.first}... " + if Kernel.system("cd #{project.second} > /dev/null 2>&1 && git bundle create #{backup_path_repo}/#{project.first}.bundle --all > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end + end + end + + task :repo_restore => :environment do + puts "Restoring repositories:" + project = Project.all.map { |n| [n.name,n.path_to_repo] } + project << ["gitolite-admin.git", File.join(File.dirname(project.first.second), "gitolite-admin.git")] + project.each do |project| + print "- Restoring repository #{project.first}... " + FileUtils.rm_rf(project.second) if File.dirname(project.second) # delet old stuff + if Kernel.system("cd #{File.dirname(project.second)} > /dev/null 2>&1 && git clone --bare #{backup_path_repo}/#{project.first}.bundle #{project.first}.git > /dev/null 2>&1") + puts "[DONE]".green + else + puts "[FAILED]".red + end + end + end + + ###################################### DB ###################################### + + task :db_dump => :environment do + FileUtils.mkdir_p(backup_path_db) until Dir.exists?(backup_path_db) + puts "Dumping database tables:" + ActiveRecord::Base.connection.tables.each do |tbl| + print "- Dumping table #{tbl}... " + count = 1 + File.open(File.join(backup_path_db, tbl + ".yml"), "w+") do |file| + ActiveRecord::Base.connection.select_all("SELECT * FROM #{tbl}").each do |line| + line.delete_if{|k,v| v.blank?} + output = {tbl + '_' + count.to_s => line} + file << output.to_yaml.gsub(/^---\n/,'') + "\n" + count += 1 + end + puts "[DONE]".green + end + end + end + + task :db_restore=> :environment do + puts "Restoring database tables:" + Rake::Task["db:reset"].invoke + Dir.glob(File.join(backup_path_db, "*.yml") ).each do |dir| + fixture_file = File.basename(dir, ".*" ) + print "- Loading fixture #{fixture_file}..." + if File.size(dir) > 0 + ActiveRecord::Fixtures.create_fixtures(backup_path_db, fixture_file) + puts "[DONE]".green + else + puts "[SKIPPING]".yellow + end + end + end + + end # namespace end: app +end # namespace end: gitlab From 4dae41d5dc90473e3c5c08064c25433fecf5abd3 Mon Sep 17 00:00:00 2001 From: Stefan Morgenthaler Date: Tue, 10 Jul 2012 16:12:19 +0200 Subject: [PATCH 2/2] Changes done as per feedback --- config/gitlab.yml.example | 2 +- config/initializers/1_settings.rb | 10 +++++++++ lib/tasks/gitlab/backup.rake | 37 +++++++++++++------------------ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 473be138..1818f2c0 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -22,7 +22,7 @@ email: app: default_projects_limit: 10 # backup_path: "/vol/backups" # default: Rails.root + backups/ - # backup_keep_time: 7.days # default: 0 (forever) + # backup_keep_time: 604800 # default: 0 (forever) (in seconds) # diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 8b9ed8ae..bbf5976b 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -101,5 +101,15 @@ class Settings < Settingslogic def default_projects_limit app['default_projects_limit'] || 10 end + + def backup_path + t = app['backup_path'] || "backups/" + t = /^\//.match(t) ? t : File.join(Rails.root + t) + t + end + + def backup_keep_time + app['backup_keep_time'] || 0 + end end end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 3735d5b2..4fbde37e 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -1,19 +1,5 @@ require 'active_record/fixtures' -# load config -GITLAB_OPTS = YAML.load_file("#{Rails.root}/config/gitlab.yml")['app'] - -# backup path options -backup_path = GITLAB_OPTS['backup_path'].nil? ? "backups/" : GITLAB_OPTS['backup_path'] # set default path if not specified in settings. -backup_path = /^\//.match(backup_path) ? backup_path : File.join(Rails.root + backup_path) # check for absolut or relative path. -FileUtils.mkdir_p(backup_path) until Dir.exists?(backup_path) - -# keep last n backups -backup_keep_time = GITLAB_OPTS['backup_keep_time'].nil? ? 0 : eval(GITLAB_OPTS['backup_keep_time']) - -backup_path_repo = File.join(backup_path, "repositories") -backup_path_db = File.join(backup_path, "db") - namespace :gitlab do namespace :app do @@ -21,11 +7,11 @@ namespace :gitlab do desc "GITLAB | Create a backup of the gitlab system" task :backup_create => :environment do - Dir.chdir(backup_path) - Rake::Task["gitlab:app:db_dump"].invoke Rake::Task["gitlab:app:repo_dump"].invoke + Dir.chdir(Gitlab.config.backup_path) + # saving additional informations s = Hash.new s["db_version"] = "#{ActiveRecord::Migrator.current_version}" @@ -33,7 +19,7 @@ namespace :gitlab do s["gitlab_version"] = %x{git rev-parse HEAD}.gsub(/\n/,"") s["tar_version"] = %x{tar --version | head -1}.gsub(/\n/,"") - File.open("#{backup_path}/backup_information.yml", "w+") do |file| + File.open("#{Gitlab.config.backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/,'') end @@ -55,10 +41,10 @@ namespace :gitlab do # delete backups print "Deleting old backups... " - if backup_keep_time > 0 + if Gitlab.config.backup_keep_time > 0 file_list = Dir.glob("*_gitlab_backup.tar").map { |f| f.split(/_/).first.to_i } file_list.sort.each do |timestamp| - if Time.at(timestamp) < (Time.now - backup_keep_time) + if Time.at(timestamp) < (Time.now - Gitlab.config.backup_keep_time) %x{rm #{timestamp}_gitlab_backup.tar} end end @@ -70,11 +56,13 @@ namespace :gitlab do end + # Restore backup of gitlab system desc "GITLAB | Restore a previously created backup" task :backup_restore => :environment do - Dir.chdir(backup_path) + Dir.chdir(Gitlab.config.backup_path) + # check for existing backups in the backup dir file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } puts "no backup found" if file_list.count == 0 if file_list.count > 1 && ENV["BACKUP"].nil? @@ -85,12 +73,13 @@ namespace :gitlab do tar_file = ENV["BACKUP"].nil? ? File.join(file_list.first.to_s + "_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") - if ! File.exists?(tar_file) + unless File.exists?(tar_file) puts "The specified backup doesn't exist!" + exit 1; end print "Unpacking backup... " - if ! Kernel.system("tar -xf #{tar_file}") + unless Kernel.system("tar -xf #{tar_file}") puts "[FAILED]".red exit 1 else @@ -129,6 +118,7 @@ namespace :gitlab do ################################# REPOSITORIES ################################# task :repo_dump => :environment do + backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") FileUtils.mkdir_p(backup_path_repo) until Dir.exists?(backup_path_repo) puts "Dumping repositories:" project = Project.all.map { |n| [n.name,n.path_to_repo] } @@ -144,6 +134,7 @@ namespace :gitlab do end task :repo_restore => :environment do + backup_path_repo = File.join(Gitlab.config.backup_path, "repositories") puts "Restoring repositories:" project = Project.all.map { |n| [n.name,n.path_to_repo] } project << ["gitolite-admin.git", File.join(File.dirname(project.first.second), "gitolite-admin.git")] @@ -161,6 +152,7 @@ namespace :gitlab do ###################################### DB ###################################### task :db_dump => :environment do + backup_path_db = File.join(Gitlab.config.backup_path, "db") FileUtils.mkdir_p(backup_path_db) until Dir.exists?(backup_path_db) puts "Dumping database tables:" ActiveRecord::Base.connection.tables.each do |tbl| @@ -179,6 +171,7 @@ namespace :gitlab do end task :db_restore=> :environment do + backup_path_db = File.join(Gitlab.config.backup_path, "db") puts "Restoring database tables:" Rake::Task["db:reset"].invoke Dir.glob(File.join(backup_path_db, "*.yml") ).each do |dir|