Skip to content

Commit

Permalink
feat(path): use node-uuid as default random path generator
Browse files Browse the repository at this point in the history 
The benefit with Universally unique identifiers, over traditional random path
generators, are that they are guaranteed to be unique. We no longer need to
check for colliding paths before uploading which will speed things up a little.

BREAKING CHANGE: this will change the path structure for new uploaded images.
The old format `aa/bb/cc` will be replaced by `aaaaaa-bbbb-cccc-dddd-eeeeee`.

Closes #52
  • Loading branch information
Hans Kristian Flaatten committed Nov 10, 2015
1 parent fd9fab7 commit e887cc0
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 123 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ client.upload('/some/image.jpg', {}, function(err, versions, meta) {

versions.forEach(function(image) {
console.log(image.width, image.height, image.url);
// 1234 4567 https://my-bucket.s3.amazonaws.com/path/ab/cd/ef.jpg
// 1024 760 https://my-bucket.s3.amazonaws.com/path/110ec58a-a0f2-4ac4-8393-c866d813b8d1.jpg
});
});
```
Expand Down
35 changes: 5 additions & 30 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ var S3 = require('aws-sdk').S3;
var auto = require('async').auto;
var each = require('async').each;
var map = require('async').map;
var retry = require('async').retry;

var resize = require('im-resize');
var metadata = require('im-metadata');
Expand Down Expand Up @@ -52,32 +51,12 @@ Upload = module.exports = function(bucketName, opts) {
].join('');
}

this._randomPath = this.opts.randomPath || require('@starefossen/rand-path');
this._randomPath = this.opts.randomPath || require('uuid');
this.s3 = new S3(this.opts.aws);

return this;
};

Upload.prototype._getDestPath = function(prefix, callback) {
var $this = this;

retry(5, function(cb) {
var path = prefix + $this._randomPath();

$this.s3.listObjects({
Prefix: path
}, function(err, data) {
if (err) {
return cb(err);
}
if (data.Contents.length === 0) {
return cb(null, path);
}
return cb(new Error('Path ' + path + ' not avaiable'));
});
}, callback);
};

Upload.prototype.upload = function(src, opts, cb) {
var image = new Image(src, opts, this);

Expand Down Expand Up @@ -113,15 +92,11 @@ Image.prototype.getMetadata = function(src, cb) {

Image.prototype.getDest = function(cb) {
var prefix = this.opts.awsPath || this.upload.opts.aws.path;
var $this = this;

if (this.opts.path) {
return process.nextTick(function() {
cb(null, prefix + $this.opts.path);
});
}
var path = this.opts.path || this.upload._randomPath();

return this.upload._getDestPath(prefix, cb);
process.nextTick(function() {
cb(null, prefix + path);
});
};

Image.prototype.resizeVersions = function(cb, results) {
Expand Down
6 changes: 3 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,16 @@
},
"homepage": "https://github.com/Turistforeningen/node-s3-uploader",
"devDependencies": {
"@starefossen/rand-path": "^1",
"mocha": "~2",
"uuid": "^2",
"jshint": "^2.8.0"
},
"dependencies": {
"@starefossen/rand-path": "^1.0.1",
"async": "~1.4",
"aws-sdk": "^2.2.9",
"im-metadata": "^3.0.0",
"im-resize": "~2.3"
"im-resize": "~2.3.1",
"node-uuid": "^1.4.3"
},
"engines": {
"node": ">=0.10",
Expand Down
117 changes: 28 additions & 89 deletions test/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,82 +158,16 @@ describe('Upload', function() {
describe('#_randomPath()', function() {
it('returns a new random path', function() {
var path = upload._randomPath();
assert(/^\w{2}(\/\w{2}){2}$/.test(path));
assert(/^\w+(-\w+){4}$/.test(path));
});

it('returns custom random path', function() {
var upload = new Upload(process.env.AWS_BUCKET_NAME, {
randomPath: require('uuid').v1
randomPath: require('@starefossen/rand-path')
});

var path = upload._randomPath();
assert(/^\w+(-\w+){4}$/.test(path));
});
});

describe('#_getDestPath()', function() {
beforeEach(function() {
upload._randomPath = function() {
return 'aa/bb/cc';
};
});

it('returns a random avaiable path', function(done) {
upload.s3.listObjects = function(opts, cb) {
process.nextTick(function() {
cb(null, {
Contents: []
});
});
};

upload._getDestPath('some/prefix/', function(err, path) {
assert.ifError(err);
assert.equal(path, 'some/prefix/aa/bb/cc');
done();
});
});

it('returns error if no available path can be found', function(done) {
upload.s3.listObjects = function(opts, cb) {
process.nextTick(function() {
cb(null, {
Contents: [opts.Prefix]
});
});
};

upload._getDestPath('some/prefix/', function(err) {
assert(err instanceof Error);
assert.equal(err.message, 'Path some/prefix/aa/bb/cc not avaiable');
done();
});
});

it('retries five 5 times to find an avaiable path', function(done) {
var count = 0;

upload.s3.listObjects = function(opts, cb) {
if (++count < 5) {
return process.nextTick(function() {
cb(null, {
Contents: [opts.Prefix]
});
});
}

process.nextTick(function() {
cb(null, {
Contents: []
});
});
};

upload._getDestPath('some/prefix/', function(err, path) {
assert.ifError(err);
assert.equal(path, 'some/prefix/aa/bb/cc');
done();
});
assert(/^\w{2}(\/\w{2}){2}$/.test(path));
});
});
});
Expand All @@ -244,7 +178,7 @@ describe('Image', function() {
beforeEach(function() {
image = new Upload.Image(__dirname + '/assets/photo.jpg', {}, upload);
image.upload._randomPath = function() {
return 'aa/bb/cc';
return '110ec58a-a0f2-4ac4-8393-c866d813b8d1';
};
});

Expand Down Expand Up @@ -274,11 +208,11 @@ describe('Image', function() {
};

image.upload.s3.putObject = function(opts) {
assert.equal(opts.Key, 'aa/bb/cc.jpg');
assert.equal(opts.Key, '110ec58a-a0f2-4ac4-8393-c866d813b8d1.jpg');
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload key suffix', function(done) {
Expand All @@ -288,11 +222,12 @@ describe('Image', function() {
};

image.upload.s3.putObject = function(opts) {
assert.equal(opts.Key, 'aa/bb/cc-small.jpg');
var dest = '110ec58a-a0f2-4ac4-8393-c866d813b8d1-small.jpg';
assert.equal(opts.Key, dest);
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload key format', function(done) {
Expand All @@ -301,11 +236,11 @@ describe('Image', function() {
};

image.upload.s3.putObject = function(opts) {
assert.equal(opts.Key, 'aa/bb/cc.png');
assert.equal(opts.Key, '110ec58a-a0f2-4ac4-8393-c866d813b8d1.png');
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets default ACL', function(done) {
Expand All @@ -318,7 +253,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets specific ACL', function(done) {
Expand All @@ -332,7 +267,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload body', function(done) {
Expand All @@ -346,7 +281,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload content type for png', function(done) {
Expand All @@ -359,7 +294,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload content type for jpg', function(done) {
Expand All @@ -372,7 +307,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload expire header for version', function(done) {
Expand All @@ -386,7 +321,7 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('sets upload cache-control header for version', function(done) {
Expand All @@ -400,15 +335,16 @@ describe('Image', function() {
done();
};

image._upload('aa/bb/cc', version);
image._upload('110ec58a-a0f2-4ac4-8393-c866d813b8d1', version);
});

it('returns etag for uploaded version', function(done) {
var version = {
path: '/some/image.jpg'
};

image._upload('aa/bb/cc', version, function(err, version) {
var dest = '110ec58a-a0f2-4ac4-8393-c866d813b8d1';
image._upload(dest, version, function(err, version) {
assert.ifError(err);
assert.equal(version.etag, '"f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1f1"');
done();
Expand All @@ -420,9 +356,10 @@ describe('Image', function() {
path: '/some/image.jpg'
};

image._upload('aa/bb/cc', version, function(err, version) {
var dest = '110ec58a-a0f2-4ac4-8393-c866d813b8d1';
image._upload(dest, version, function(err, version) {
assert.ifError(err);
assert.equal(version.url, image.upload.opts.url + 'aa/bb/cc.jpg');
assert.equal(version.url, image.upload.opts.url + dest + '.jpg');
done();
});
});
Expand Down Expand Up @@ -460,9 +397,11 @@ describe('Image', function() {

describe('#getDest()', function() {
it('returns destination path', function(done) {
var dest = '110ec58a-a0f2-4ac4-8393-c866d813b8d1';

image.getDest(function(err, path) {
assert.ifError(err);
assert.equal(path, image.upload.opts.aws.path + 'aa/bb/cc');
assert.equal(path, image.upload.opts.aws.path + dest);
done();
});
});
Expand All @@ -471,7 +410,7 @@ describe('Image', function() {
image.opts.awsPath = 'custom/path/';
image.getDest(function(err, path) {
assert.ifError(err);
assert.equal(path, 'custom/path/aa/bb/cc');
assert.equal(path, 'custom/path/110ec58a-a0f2-4ac4-8393-c866d813b8d1');
done();
});
});
Expand Down Expand Up @@ -653,7 +592,7 @@ describe('Integration Tests', function() {
beforeEach(function() {
if (process.env.INTEGRATION_TEST !== 'true') {
upload._randomPath = function() {
return 'aa/bb/cc';
return '110ec58a-a0f2-4ac4-8393-c866d813b8d1';
};
}
});
Expand Down

0 comments on commit e887cc0

Please sign in to comment.