Skip to content

Instantly share code, notes, and snippets.

@lkptrzk

lkptrzk/fetch-s3-folder.js

Created November 14, 2014 00:48
Show Gist options
  • Save lkptrzk/afa8413999419deb04b1 to your computer and use it in GitHub Desktop.
Save lkptrzk/afa8413999419deb04b1 to your computer and use it in GitHub Desktop.
Download ZIP
fetch all files from s3 bucket with given prefix and stream them to disk
Raw
fetch-s3-folder.js
// fetch-s3-folder.js
//
// usage: node fetch-s3-folder.js bucketName folderPath
// ex: node fetcb-s3-folder.js our-bucket /my-folder/cool-subfolder
//
// to install dependencies, run following in same folder as this script:
// mkdir node_modules && npm install async knox mkdirp
//
// also expects a `config.json` in same directory with AWS credentials
var Fs = require('fs');
var Async = require('async');
var Knox = require('knox');
var Mkdirp = require('mkdirp');
var bucketName = process.argv[2];
var prefix = process.argv[3];
var config = require('./config.json');
config.bucket = bucketName;
var s3Client = Knox.createClient(config);
function log (message) {
console.error('' + Date.now() + ' ' + message);
};
s3Client.list({ prefix: prefix }, function (err, data) {
if (err) {
throw err;
}
Mkdirp.sync(prefix);
Async.mapLimit(data.Contents, 20, function (file, callback) {
var outputStream = Fs.createWriteStream(file.Key);
log('start ' + file.Key);
s3Client.getFile(file.Key, function (err, res) {
if (err) {
return callback(err);
}
res.pipe(outputStream);
res.on('end', function () {
log('done ' + file.Key);
return callback(null, file);
});
});
},
function (err, results) {
if (err) {
log('error');
log(err.message);
log(err);
process.exit(1);
}
log('done');
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment