Created
March 14, 2019 16:08
-
-
Save Al-un/f07b7d7ef5d6c0ddb15296f7fbb7e23c to your computer and use it in GitHub Desktop.
Multiple "aws s3 ls" to fetch buckets info and parse it
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# ---------- Help | |
# https://stackoverflow.com/a/51911626/4906586 | |
__help=" | |
Usage: | |
./load.sh --awsBuckets=pouet --awsProfile=MyProfile --parser=../haha | |
Options: | |
-b, --awsBuckets=<buckets> list of buckets to load from AWS. If | |
-u, --awsProfile= AWS CLI profile to use to load S3 data. Require having | |
AWS CLI installed and configured. If awsBuckets is defined | |
while awsProfile is not defined, default AWS CLI is used | |
instead | |
-p, --parser <parserFile> Python parse file to read S3 buckets info. Default is | |
parse.py | |
" | |
# ---------- Load arguments | |
# https://stackoverflow.com/a/33826763/4906586 | |
while [[ "$#" -gt 0 ]]; do case $1 in | |
-b|--awsBuckets) | |
# bucketStr="$2" | |
awsBuckets="$2@"; | |
shift;; | |
-u|--awsProfile) | |
awsProfile="$2"; | |
shift;; | |
-p|--parser) | |
parserFile="$2" | |
shift;; | |
esac; shift;done | |
echo "aws $awsBuckets $awsProfile" | |
echo "parser $parserFile" | |
# no arguments were provided | |
# https://stackoverflow.com/a/13864829/4906586 | |
if [ -z ${awsBuckets+x} ] && [ -z ${awsProfile+x} ] && [ -z ${parse+x} ] | |
then | |
echo "$__help" | |
exit 0 | |
fi | |
# # ---------- AWS Load | |
# https://www.shellhacks.com/yes-no-bash-script-prompt-confirmation/ | |
read -p "Script is going to \"aws s3 ls\" ${awsBuckets[*]}: Continue? (y/n)" -n 1 -r | |
echo | |
if [[ ! $REPLY =~ ^[Yy]$ ]] | |
then | |
echo Script cancelled by user | |
echo | |
exit 1 | |
fi | |
for bucketName in ${awsBuckets[*]} | |
do | |
# check if an old file exists | |
if [ -f "$bucketName" ]; then | |
printf "%s file already exists. Deleting...\n" $bucketName | |
rm $bucketName | |
fi | |
# Loading | |
printf "Loading bucket: %s\n" $bucketName | |
if [ -z ${awsProfile+x} ]; then | |
aws s3 ls $bucketName --recursive >> $bucketName | |
else | |
aws s3 ls $bucketName --recursive --profile $awsProfile >> $bucketName | |
fi | |
done | |
# ---------- Parsing | |
# If parse.py is present | |
parseFile="parse.py" | |
if [ -f "$parserFile" ] | |
then | |
python3 $parserFile ${awsBuckets[*]} | |
else | |
printf "%s not found. Parsing skipped..." $parseFile | |
fi |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment