From 9107a4b2d4ed0b4d00d2e6119cbf991c6d61ff2d Mon Sep 17 00:00:00 2001 From: duy Date: Wed, 17 Jul 2013 14:51:34 -0700 Subject: [PATCH 1/6] Allowing you to configure the name of the S3 backup directory through an environment variable --- lib/heroku-mongo-backup.rb | 18 +++++++++++++++--- lib/s3_helpers.rb | 34 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/lib/heroku-mongo-backup.rb b/lib/heroku-mongo-backup.rb index 1f43cad..5331878 100644 --- a/lib/heroku-mongo-backup.rb +++ b/lib/heroku-mongo-backup.rb @@ -118,6 +118,18 @@ def s3_connect bucket = ENV['S3_BUCKET'] end + dir_name = ENV['S3_BACKUP_DIR'] + if dir_name.nil? + dir_name = ENV['S3_BACKUP_DIRNAME'] + end + if dir_name.nil? + dir_name = ENV['S3_BACKUP_DIR_NAME'] + end + if dir_name.nil? + dir_name = 'backups' + end + @dir_name = dir_name + access_key_id = ENV['S3_KEY_ID'] if access_key_id.nil? access_key_id = ENV['S3_KEY'] @@ -138,12 +150,12 @@ def s3_connect end def s3_upload - HerokuMongoBackup::s3_upload(@bucket, @file_name) + HerokuMongoBackup::s3_upload(@bucket, @dir_name, @file_name) end def s3_download open(@file_name, 'w') do |file| - file_content = HerokuMongoBackup::s3_download(@bucket, @file_name) + file_content = HerokuMongoBackup::s3_download(@bucket, @dir_name, @file_name) file.binmode file.write file_content end @@ -215,7 +227,7 @@ def backup files_number_to_leave=0 end if files_number_to_leave > 0 - HerokuMongoBackup::remove_old_backup_files(@bucket, files_number_to_leave) + HerokuMongoBackup::remove_old_backup_files(@bucket, @dir_name, files_number_to_leave) end end diff --git a/lib/s3_helpers.rb b/lib/s3_helpers.rb index 9f5eff1..0faf329 100644 --- a/lib/s3_helpers.rb +++ b/lib/s3_helpers.rb @@ -20,14 +20,14 @@ def HerokuMongoBackup::s3_connect(bucket, key, secret) return bucket end - def HerokuMongoBackup::s3_upload(bucket, filename) - object = bucket.objects.build("backups/#{filename}") + def HerokuMongoBackup::s3_upload(bucket, dirname, filename) + object = bucket.objects.build("#{dirname}/#{filename}") object.content = open(filename) object.save end - def HerokuMongoBackup::s3_download(bucket, filename) - object = bucket.objects.find("backups/#{filename}") + def HerokuMongoBackup::s3_download(bucket, dirname, filename) + object = bucket.objects.find("#{dirname}/#{filename}") content = object.content(reload=true) puts "Backup file:" @@ -39,8 +39,8 @@ def HerokuMongoBackup::s3_download(bucket, filename) return content end - def HerokuMongoBackup::remove_old_backup_files(bucket, files_number_to_leave) - excess = ( object_keys = bucket.objects.find_all(:prefix => "backups/").map { |o| o.key }.sort ).count - files_number_to_leave + def HerokuMongoBackup::remove_old_backup_files(bucket, dirname, files_number_to_leave) + excess = ( object_keys = bucket.objects.find_all(:prefix => "#{dirname}/").map { |o| o.key }.sort ).count - files_number_to_leave (0..excess-1).each { |i| bucket.objects.find(object_keys[i]).destroy } if excess > 0 end @@ -68,17 +68,17 @@ def HerokuMongoBackup::s3_connect(bucket, key, secret) return bucket end - def HerokuMongoBackup::s3_upload(bucket, filename) - AWS::S3::S3Object.store("backups/#{filename}", open(filename), bucket) + def HerokuMongoBackup::s3_upload(bucket, dirname, filename) + AWS::S3::S3Object.store("#{dirname}/#{filename}", open(filename), bucket) end - def HerokuMongoBackup::s3_download(bucket, filename) - content = AWS::S3::S3Object.value("backups/#{filename}", bucket) + def HerokuMongoBackup::s3_download(bucket, dirname, filename) + content = AWS::S3::S3Object.value("#{dirname}/#{filename}", bucket) return content end - def HerokuMongoBackup::remove_old_backup_files(bucket, files_number_to_leave) - excess = ( object_keys = AWS::S3::Bucket.find(bucket).objects(:prefix => 'backups/').map { |o| o.key }.sort ).count - files_number_to_leave + def HerokuMongoBackup::remove_old_backup_files(bucket, dirname, files_number_to_leave) + excess = ( object_keys = AWS::S3::Bucket.find(bucket).objects(:prefix => "#{dirname}/").map { |o| o.key }.sort ).count - files_number_to_leave (0..excess-1).each { |i| AWS::S3::S3Object.find(object_keys[i], bucket).delete } if excess > 0 end @@ -111,19 +111,19 @@ def HerokuMongoBackup::s3_connect(bucket, key, secret) return directory end - def HerokuMongoBackup::s3_upload(directory, filename) + def HerokuMongoBackup::s3_upload(directory, dirname, filename) file = directory.files.create( - :key => "backups/#{filename}", + :key => "#{dirname}/#{filename}", :body => open(filename) ) end - def HerokuMongoBackup::s3_download(directory, filename) - file = directory.files.get("backups/#{filename}") + def HerokuMongoBackup::s3_download(directory, dirname, filename) + file = directory.files.get("#{dirname}/#{filename}") return file.body end - def HerokuMongoBackup::remove_old_backup_files(directory, files_number_to_leave) + def HerokuMongoBackup::remove_old_backup_files(directory, dirname, files_number_to_leave) total_backups = directory.files.all.size if total_backups > files_number_to_leave From 8d3c15e682fa286edc0a8f37ffd131b48cbca3dd Mon Sep 17 00:00:00 2001 From: strayduy Date: Thu, 18 Jul 2013 13:59:44 -0700 Subject: [PATCH 2/6] Fixing a bug with the Fog interface The script was looking at all of the files in the specified bucket when determining which ones to delete. I only want it to look under the designated backup directory, so I added the directory prefix when it's considering which files to delete. --- lib/s3_helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/s3_helpers.rb b/lib/s3_helpers.rb index 0faf329..646646e 100644 --- a/lib/s3_helpers.rb +++ b/lib/s3_helpers.rb @@ -124,11 +124,11 @@ def HerokuMongoBackup::s3_download(directory, dirname, filename) end def HerokuMongoBackup::remove_old_backup_files(directory, dirname, files_number_to_leave) - total_backups = directory.files.all.size + total_backups = directory.files.all({:prefix => "#{dirname}/"}).size if total_backups > files_number_to_leave - files_to_destroy = (0..total_backups-files_number_to_leave-1).collect{|i| directory.files.all[i] } + files_to_destroy = (0..total_backups-files_number_to_leave-1).collect{|i| directory.files.all({:prefix => "#{dirname}/"})[i] } files_to_destroy.each do |f| f.destroy From 0a084a2d165475be877e137c5c3c62f6c463b4ff Mon Sep 17 00:00:00 2001 From: strayduy Date: Thu, 18 Jul 2013 14:01:38 -0700 Subject: [PATCH 3/6] Correcting a comment --- lib/s3_helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/s3_helpers.rb b/lib/s3_helpers.rb index 646646e..e53035e 100644 --- a/lib/s3_helpers.rb +++ b/lib/s3_helpers.rb @@ -98,9 +98,9 @@ def HerokuMongoBackup::remove_old_backup_files(bucket, dirname, files_number_to_ if defined?(Fog) # - # Using 'aws/s3' gem as Amazon S3 interface + # Using 'fog' gem as Amazon S3 interface # - #puts "Using \'aws/s3\' gem as Amazon S3 interface." + #puts "Using \'fog\' gem as Amazon S3 interface." def HerokuMongoBackup::s3_connect(bucket, key, secret) connection = Fog::Storage.new({ :provider => 'AWS', From f91476540c0c42d0e237d65b46d0ac315dbc1d42 Mon Sep 17 00:00:00 2001 From: strayduy Date: Sun, 11 Aug 2013 01:36:09 -0700 Subject: [PATCH 4/6] Adding explanation of S3_BACKUP_DIR parameter --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index d70e277..359a68c 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ If you want to automatically remove old backup files pass ```MAX_BACKUPS``` para * ```heroku run rake mongo:backup MAX_BACKUPS=7``` +If you're uploading to S3, backup files will be stored as ```backups/YYYY-MM-DD_hh-mm-ss.gz``` by default. To change the directory name, pass in the ```S3_BACKUP_DIR``` parameter: + +* ```heroku run rake mongo:backup S3_BACKUP_DIR=daily``` +* Backup files would then be stored as ```daily/backup-file-name.gz``` instead of ```backups/backup-file-name.gz```. + Restore from backup: * ```heroku run rake mongo:restore FILE=backup-file-name.gz``` From 4c05422cbbc26bd37352dac99792795e50a0ca30 Mon Sep 17 00:00:00 2001 From: strayduy Date: Sun, 11 Aug 2013 01:39:16 -0700 Subject: [PATCH 5/6] Adding myself to Gem contributors --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 359a68c..6c3c01a 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ For Rails 2 add this to your Rakefile to import rake tasks: 5. [wolfpakz](https://github.com/wolfpakz "Dan Porter") - Rails2 support 6. [solacreative](http://sola-la.com/creative "Taro Murao") - Max backups feature for aws/s3 and s3 gems 7. [aarti](https://github.com/aarti "aarti") - minor fixes - +8. [strayduy](https://github.com/strayduy "strayduy") - [Configurable S3 directory name](https://github.com/alexkravets/heroku-mongo-backup/pull/17) From c76873f514c31e454db5c2953aa71a068de65f76 Mon Sep 17 00:00:00 2001 From: duy Date: Sun, 11 Aug 2013 01:42:20 -0700 Subject: [PATCH 6/6] Bumping the version number --- heroku-mongo-backup.gemspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heroku-mongo-backup.gemspec b/heroku-mongo-backup.gemspec index 217cc17..dd739e5 100644 --- a/heroku-mongo-backup.gemspec +++ b/heroku-mongo-backup.gemspec @@ -1,10 +1,10 @@ Gem::Specification.new do |s| s.name = 'heroku-mongo-backup' - s.version = '0.4.31' + s.version = '0.4.32' s.summary = 'Rake task backups mongo database on Heroku and push gzipped file to Amazon S3 or FTP.' s.description = 'Rake task for backing up mongo database on heroku and push it to S3 or FTP. Library can be used as rake task or be easily integrated into daily cron job.' - s.authors = ['Alex Kravets', 'matyi', 'Stef Lewandowski', 'David Hall', 'Dan Porter', 'aarti'] + s.authors = ['Alex Kravets', 'matyi', 'Stef Lewandowski', 'David Hall', 'Dan Porter', 'aarti', 'strayduy'] s.email = 'santyor@gmail.com' s.homepage = 'https://github.com/alexkravets/heroku-mongo-backup'