Last active
September 8, 2022 06:16
-
-
Save colebrooke/b9259d7a45a5cc7dec57e14c0312f6ee to your computer and use it in GitHub Desktop.
Quickly list sizes of AWS S3 buckets without traversing all objects.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# Justin Miller 12/01/17 | |
# Script to calculate the human readable size of S3 buckets | |
# The script uses cloudwatch metrics. These may need time to update if you've added data to a bucket within 24hrs. | |
# It happily DOES NOT need to travese potentially millions of files to calculate your S3 bucket sizes. | |
# | |
# Usage: | |
# ./ls-s3-sizes.sh | |
# | |
# TODO: Could be further parameterised depending on usage requirements. | |
PROFILE="default" # update to relect the Credentials profile you'd like to use, typicaly in ~/.aws/credentials | |
REGION="eu-west-1" # specify a region | |
STORAGE_CLASS="StandardIAStorage" # a storage class is required. Either "StandardStorage" or "StandardIAStorage" for infrequent access | |
SHOW_ZERO_SIZE_BUCKETS="false" | |
# Check if jq is available | |
type jq >/dev/null 2>&1 || { echo >&2 "The jq utility is required for this script to run."; exit 1; } | |
# Check if aws cli is available | |
type aws >/dev/null 2>&1 || { echo >&2 "The aws cli is required for this script to run."; exit 1; } | |
BUCKETS=$(aws --profile $PROFILE s3 ls | cut -d' ' -f3) | |
function BucketSize () { | |
BUCKET=$1 | |
NOW=$(date +%s) | |
BUCKET_SIZE_BYTES=$(aws --profile $PROFILE cloudwatch \ | |
get-metric-statistics \ | |
--namespace AWS/S3 \ | |
--start-time "$(echo "$NOW - 86400" | bc)" \ | |
--end-time "$NOW" \ | |
--period 86400 \ | |
--statistics Average \ | |
--region $REGION \ | |
--metric-name BucketSizeBytes \ | |
--dimensions \ | |
Name=BucketName,Value="$BUCKET" \ | |
Name=StorageType,Value=$STORAGE_CLASS | jq '.Datapoints[].Average') | |
if [[ $BUCKET_SIZE_BYTES > 1 ]]; then | |
BUCKET_SIZE_GB=$(echo "scale=2; $BUCKET_SIZE_BYTES/1073741824" | bc -l) | |
echo -e "${BUCKET_SIZE_GB}GB - $BUCKET" | |
else | |
BUCKET_SIZE_GB=0 | |
if [ $SHOW_ZERO_SIZE_BUCKETS == "true" ]; then | |
echo -e "${BUCKET_SIZE_GB}GB - $BUCKET" | |
fi | |
fi | |
} | |
TOTAL_SIZE=0 | |
for MYBUCKET in ${BUCKETS[*]}; do | |
BucketSize "$MYBUCKET" | |
TOTAL_SIZE=$(echo "scale=2; $TOTAL_SIZE+$BUCKET_SIZE_GB" | bc -l) | |
done | |
echo "${TOTAL_SIZE}GB - GRAND TOTAL" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
didn't work for me.