Last active
September 3, 2019 10:34
-
-
Save LiamKarlMitchell/027b77b9e80ab518c23dd423963cbcda to your computer and use it in GitHub Desktop.
node.js download file outputs progress and does not re-download already downloaded file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var fs = require('fs'); | |
var http = require('http'); | |
var forceredownload = false; | |
function downloadFile(url, file, callback, redirect_count, known_size) { | |
console.log('Attempting to download ' + url + ' to ' + file); | |
if (redirect_count) { | |
if (redirect_count > 5) { | |
callback('Max redirects reached', url); | |
return; | |
} | |
} else { | |
redirect_count = 0; | |
} | |
var filename = DOWNLOAD_DIR + file; | |
var outfile; | |
if (callback === undefined) { | |
console.log('WARNING downloadFile callback is undefined for ' + url + ' to ' + filename); | |
callback = function(err, data) { | |
console.log('downloadFile callback result undefined: ', err, data) | |
}; | |
} | |
var request = http.get(url, function(response) { | |
var content_length = 0; | |
var downloaded_bytes = 0; | |
function outputInfo() { | |
if (content_length === undefined) { | |
console.log(file + ' Progress: unknown\t' + downloaded_bytes + '/' + '?'); | |
} | |
percent = parseInt((downloaded_bytes / content_length) * 100); | |
console.log(file + ' Progress: ' + percent + '%\t' + downloaded_bytes + '/' + content_length); | |
} | |
console.log(file + ' ' + response.statusCode); | |
switch (response.statusCode) { | |
case 200: | |
// Try to get the content length. | |
// If the server does not tell us try to use known_size argument. | |
content_length = response.headers['content-length'] || known_size; | |
// Check if file was already downloaded. | |
// Note: we are just checking length not a checksum of any kind or created date. | |
fs.stat(filename, function(err, stats) { | |
if (!err) { | |
console.log('Have already downloaded ' + filename); | |
if (stats.size == content_length) { | |
console.log('And the file size matches (' + content_length + ')'); | |
if (forceredownload === false) { | |
console.log('Not Redownloading'); | |
callback(null, filename); | |
request.abort(); | |
return; | |
} else { | |
console.log('Redownloading'); | |
} | |
} else { | |
console.log('But the file size does not match so re-downloading Old(' + stats.size + ') vs New(' + content_length + ')'); | |
} | |
} | |
outfile = fs.createWriteStream(file); | |
response.pipe(outfile); | |
response.on('data', function(data) { | |
downloaded_bytes += data.length; | |
outputInfo(); | |
}); | |
outfile.on('finish', function() { | |
console.log('Downloaded ' + url + ' to ' + file); | |
callback(null, file); | |
}); | |
}); | |
break; | |
// Handle redirects. | |
case 302: | |
new_remote = response.headers.location; | |
console.log('Redirect ' + url + ' to ' + new_remote); | |
if (redirect_count <5) { | |
downloadFile(new_remote, file, callback, redirect_count++); | |
} else { | |
console.error('Redirect limit encountered for '+url+'.'); | |
callback(302, null); | |
} | |
// Note: Some servers redirect requests to another url like deny.site.com if there are too many rapidly/failing requests. | |
// You might want to detect that and delay a redirect to the same location. | |
return; | |
break; | |
case 404: | |
console.log("File Not Found " + url); | |
// Note: You might want to break here and call callback with a null if you don't care if the file was unable to be downloaded. | |
default: | |
console.log('Problem downloading ' + url + ' STATUS: ' + response.statusCode); | |
request.abort(); | |
callback(response.statusCode, null); | |
return; | |
} | |
}).on('error', function(e) { | |
console.log('Error Downloading: ' + url + ' ' + e.message); | |
callback(e, null); | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Just something I made to download files without overwriting the file if it already exists.