Created
March 8, 2021 16:52
-
-
Save keepitsimple/2c9770a4ca993d01936096f112dc1b27 to your computer and use it in GitHub Desktop.
Download to file & download to buffer Typescript, Node 12.x, support for redirects. http/https
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import fs from 'fs' | |
| import https from 'https' | |
| import http from 'http' | |
| import { basename } from 'path' | |
| import { URL } from 'url' | |
| const TIMEOUT = 10000 | |
| const MAX_DOWNLOAD_FILE_SIZE = 1024 * 1024 * 200 // 200MB | |
| export function downloadAsBuffer (url: string): Promise<Buffer> { | |
| const uri = new URL(url) | |
| const pkg = url.toLowerCase().startsWith('https:') ? https : http | |
| return new Promise((resolve, reject) => { | |
| const request = pkg.get(uri.href).on('response', (res) => { | |
| if (res.statusCode === 200) { | |
| const chunks:Uint8Array[] = [] | |
| const fileSize = parseInt(res.headers['content-length'] as string, 10) | |
| if (fileSize > MAX_DOWNLOAD_FILE_SIZE) { | |
| reject(new Error(`File is too big for download. The file cannot exceed ${MAX_DOWNLOAD_FILE_SIZE} bytes`)) | |
| return | |
| } | |
| res | |
| .on('data', (chunk: Buffer) => { | |
| chunks.push(chunk) | |
| }) | |
| .on('end', () => { | |
| if (!res.complete) { | |
| reject(new Error('The connection was terminated while the file was still being sent')) | |
| } | |
| resolve(Buffer.concat(chunks)) | |
| }) | |
| .on('error', (err) => { | |
| reject(err) | |
| }) | |
| } else if (res.statusCode === 302 || res.statusCode === 301) { | |
| // Recursively follow redirects, only a 200 will resolve. | |
| downloadAsBuffer(res.headers.location as string).then(() => resolve()) | |
| } else { | |
| reject(new Error(`Download request failed, response status: ${res.statusCode} ${res.statusMessage}`)) | |
| } | |
| }) | |
| request.setTimeout(TIMEOUT, function () { | |
| request.abort() | |
| reject(new Error(`Request timeout after ${TIMEOUT / 1000.0}s`)) | |
| }) | |
| }) | |
| } | |
| /** | |
| * Downloads specified `url` as a file and store it to `dest` | |
| * @param url The URL of the file for download | |
| * @param dest THe destination path and file name | |
| */ | |
| export function downloadAsFile (url: string, dest?: string): Promise<string> { | |
| const uri = new URL(url) | |
| if (!dest) { | |
| dest = basename(uri.pathname) | |
| } | |
| const pkg = url.toLowerCase().startsWith('https:') ? https : http | |
| return new Promise((resolve, reject) => { | |
| const request = pkg.get(uri.href).on('response', (res) => { | |
| if (res.statusCode === 200) { | |
| const fileSize = parseInt(res.headers['content-length'] as string, 10) | |
| if (fileSize > MAX_DOWNLOAD_FILE_SIZE) { | |
| reject(new Error(`File is too big for download. The file cannot exceed ${MAX_DOWNLOAD_FILE_SIZE} bytes`)) | |
| return | |
| } | |
| const file = fs.createWriteStream(dest as string, { flags: 'w' }) | |
| res | |
| .on('end', () => { | |
| if (!res.complete) { | |
| fs.unlink(dest as string, () => reject(new Error('The connection was terminated while the file was still being sent'))) | |
| } | |
| file.end() | |
| // console.log(`${uri.pathname} downloaded to: ${path}`) | |
| resolve(dest) | |
| }) | |
| .on('error', (err) => { | |
| file.destroy() | |
| fs.unlink(dest as string, () => reject(err)) | |
| }).pipe(file) | |
| } else if (res.statusCode === 302 || res.statusCode === 301) { | |
| // Recursively follow redirects, only a 200 will resolve. | |
| downloadAsFile(res.headers.location as string, dest).then(() => resolve()) | |
| } else { | |
| reject(new Error(`Download request failed, response status: ${res.statusCode} ${res.statusMessage}`)) | |
| } | |
| }) | |
| request.setTimeout(TIMEOUT, function () { | |
| request.abort() | |
| reject(new Error(`Request timeout after ${TIMEOUT / 1000.0}s`)) | |
| }) | |
| }) | |
| } | |
| export default downloadAsFile |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment