Skip to content

Instantly share code, notes, and snippets.

@febritecno
Last active February 29, 2024 09:21
Show Gist options
  • Save febritecno/d54bbd3d9d81e624796c6bdcf7ea1977 to your computer and use it in GitHub Desktop.
Save febritecno/d54bbd3d9d81e624796c6bdcf7ea1977 to your computer and use it in GitHub Desktop.
AWS S3 operation
s3.listObjectsV2(params, (err, data) => {
const folderExists = data.Contents.length > 0;
// (folderExists == true) if the folder exists
});
async function checkFolderExist(key) {
try {
const objects = await s3.listObjectsV2({ Bucket: BUCKET_NAME, Prefix: key }).promise();
return objects.Contents.length > 0;
} catch (error) {
return false;
}
}
await checkFolderExist("companies/documents");
async function checkKeyExist(key) {
try {
await s3.headObject({ Bucket: BUCKET_NAME, Key: key }).promise();
return true;
} catch (_) {
return false;
}
}
await checkKeyExist("companies/documents/");
async function copyS3FolderFile(sourcePath, destinationPath) {
try {
const objects = await s3.listObjectsV2({ Bucket: BUCKET_NAME, Prefix: sourcePath }).promise();
await Promise.all(
objects.Contents.map(async (obj) => {
const split = obj.Key.split("/");
// magic automatic copy with dynamic path
const commonPrefix = getCommonPrefix(sourcePath, destinationPath);
const adjustedDestinationPath = destinationPath.replace(commonPrefix, '');
const copyParams = {
Bucket: BUCKET_NAME,
CopySource: `${BUCKET_NAME}/${obj.Key}`,
Key: `${adjustedDestinationPath}/${split.slice(2).join('/')}`,
};
await s3.copyObject(copyParams).promise();
Logger.warn(`copying files from ${copyParams.CopySource} to ${copyParams.Key}`);
})
);
Logger.info('Folder and files copied successfully');
Logger.info("");
} catch (error) {
console.log(error);
Logger.error(`Error copying files from ${sourcePath} to ${destinationPath}`);
Logger.info("");
}
}
function getCommonPrefix(path1, path2) {
const path1Parts = path1.split('/');
const path2Parts = path2.split('/');
let commonPrefix = '';
for (let i = 0; i < Math.min(path1Parts.length, path2Parts.length); i++) {
if (path1Parts[i] === path2Parts[i]) {
commonPrefix += `${path1Parts[i]}/`;
} else {
break;
}
}
return commonPrefix;
}
/// sample copy all files/folder
const pastePath = "companies_test/6082047d7ef642687091f015/FOLDERS/sourci-suppliers"
const copySrc = "products/63c4f62f48bd81001309d2c3/"
await copyS3FolderFile(copySrc, pastePath);
//// check file exist?
async function checkKeyExist(key) {
try {
await s3.headObject({ Bucket: BUCKET_NAME, Key: key }).promise();
return true;
} catch (error) {
return false;
}
}
//// create folder
async function createS3Folder(key) {
try {
await s3.putObject({ Bucket: BUCKET_NAME, Key: key }).promise();
Logger.info(`Folder created: ${key}`);
} catch (error) {
Logger.error(`Error creating folder: ${key}`, error);
}
}
//// check if duplicate then create with new name
async function checkKeyExistAndRename(key) {
let labelNumber = 1;
let newKey = key;
while (true) {
try {
await s3.headObject({ Bucket: BUCKET_NAME, Key: newKey }).promise();
newKey = `${key} (${labelNumber})`;
labelNumber++;
} catch (_) {
if (newKey !== key) {
return newKey;
} else {
return key;
}
}
}
}
// create folder with validation
async function createS3Folder(folderPath) {
// Check if the folder already exists
const folderExists = await checkKeyExist(folderPath);
if (folderExists) {
Logger.info(`Folder already exists: ${folderPath} [v]`);
return;
}
const params = {
Bucket: BUCKET_NAME,
Key: folderPath,
Body: '',
};
try {
await s3.putObject(params).promise();
Logger.info(`Folder created: ${folderPath} [v]`);
} catch (error) {
Logger.error(`Error creating folder: ${folderPath} [X]`, error);
}
}
async function removeS3Folder(prefix) {
try {
const objects = await s3.listObjectsV2({ Bucket: BUCKET_NAME, Prefix: prefix }).promise();
if (objects.Contents.length === 0) {
Logger.info(`No objects found with prefix ${prefix}. Nothing to remove.`);
return;
}
await Promise.all(
objects.Contents.map(async (obj) => {
const params = {
Bucket: BUCKET_NAME,
Key: obj.Key,
};
await s3.deleteObject(params).promise();
Logger.warn(`Deleted object: ${obj.Key}`);
})
);
Logger.info(`All objects with prefix ${prefix} removed successfully.`);
} catch (error) {
console.error(error);
Logger.error(`Error removing objects with prefix ${prefix}`);
}
}
await removeS3Folder(`${projectFolder}/documents/`);
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment