Created
July 28, 2021 20:47
-
-
Save elsangedy/14dd09686d66e4f2a87518c31755f0cc to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const S3Client = require('aws-sdk/clients/s3') | |
const sharp = require('sharp') | |
const ACCESS_KEY_ID = '' | |
const SECRET_ACCESS_KEY = '' | |
const REGION = '' | |
const BUCKET = '' | |
const PREFIX = '' | |
const CHUNK_SIZE = 500 | |
const FILTER = (filesName) => { | |
return filesName.filter( | |
(fileName) => | |
(fileName.endsWith('.jpg') || fileName.endsWith('.png')) && | |
!fileName.includes('-x2.') && | |
!filesName.includes(fileName.replace('.', '-x2.')) | |
) | |
} | |
const PROCESS = async (fileName) => { | |
const result = await getFile(fileName) | |
const imageStream = sharp(result.Body) | |
const imageMetadata = await imageStream.metadata() | |
const { data } = await imageStream | |
.resize({ | |
width: imageMetadata.width * 2, | |
height: imageMetadata.height * 2, | |
fit: 'inside', | |
}) | |
.toBuffer({ | |
resolveWithObject: true, | |
}) | |
await s3 | |
.upload({ | |
Key: fileName.replace('.', '-x2.'), | |
Body: data, | |
Bucket: BUCKET, | |
ContentType: result.ContentType, | |
ACL: 'public-read', | |
}) | |
.promise() | |
} | |
const s3 = new S3Client({ | |
accessKeyId: ACCESS_KEY_ID, | |
secretAccessKey: SECRET_ACCESS_KEY, | |
region: REGION, | |
}) | |
const splitEvery = (n, arr) => { | |
let idx = 0 | |
const chunks = [] | |
while (idx < arr.length) { | |
chunks.push(arr.slice(idx, (idx += n))) | |
} | |
return chunks | |
} | |
const getFile = async (fileName) => { | |
return s3 | |
.getObject({ | |
Bucket: BUCKET, | |
Key: fileName, | |
}) | |
.promise() | |
} | |
const getFiles = async (files = []) => { | |
const result = await s3 | |
.listObjectsV2({ | |
Bucket: BUCKET, | |
Prefix: PREFIX, | |
MaxKeys: 1000, | |
StartAfter: files.length > 0 ? files[files.length - 1] : undefined, | |
}) | |
.promise() | |
const filesName = result.Contents.map((file) => file.Key) | |
const list = [...files, ...filesName] | |
return filesName.length === 1000 ? getFiles(list) : list | |
} | |
const run = async () => { | |
const allFiles = await getFiles() | |
console.log('allFiles', allFiles.length) | |
const filteredFiles = FILTER(allFiles) | |
console.log('filteredFiles', filteredFiles.length) | |
const chunks = splitEvery(CHUNK_SIZE, filteredFiles) | |
console.log('chunks', chunks.length) | |
await Promise.all( | |
chunks.map(async (chunk, index) => { | |
const chunkNumber = (index + 1).toString().padStart(2, '0') | |
const chunkLength = chunk.length | |
let i = 1 | |
for (let fileName of chunk) { | |
console.log(`[chunk ${chunkNumber}] ${i++}/${chunkLength}`) | |
await PROCESS(fileName) | |
} | |
}) | |
) | |
} | |
run() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment