Last active
August 20, 2021 03:08
-
-
Save jonico/cd0bd4d9b30a7867ca2c36f01d1b3ec4 to your computer and use it in GitHub Desktop.
Co-pilot examples (only comments have been written by me)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# copy Azure bloc storage files to S3 buckets | |
# | |
# Usage: | |
# ./copy-azure-blob-storage-to-s3.sh <blob storage container> <s3 bucket> | |
# | |
# Example: | |
# ./copy-azure-blob-storage-to-s3.sh my-container s3://my-bucket | |
# | |
# Note: | |
# This script requires a working Azure CLI. | |
# See https://docs.microsoft.com/en-us/cli/azure/install-azure-cli | |
# | |
# This script requires the following environment variables to be set: | |
# AZURE_STORAGE_ACCOUNT | |
# AZURE_STORAGE_KEY | |
# AZURE_STORAGE_CONNECTION_STRING | |
# | |
# This script requires the following AWS S3 environment variables to be set: | |
# AWS_ACCESS_KEY_ID | |
# AWS_SECRET_ACCESS_KEY | |
# | |
# This script requires the following AWS S3 CLI environment variables to be set: | |
# AWS_DEFAULT_REGION | |
# exit on error | |
set -e | |
# check for required arguments | |
if [ $# -ne 2 ]; then | |
echo "Usage: $0 <blob storage container> <s3 bucket>" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AZURE_STORAGE_ACCOUNT" ]; then | |
echo "Environment variable AZURE_STORAGE_ACCOUNT is not set" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AZURE_STORAGE_KEY" ]; then | |
echo "Environment variable AZURE_STORAGE_KEY is not set" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AZURE_STORAGE_CONNECTION_STRING" ]; then | |
echo "Environment variable AZURE_STORAGE_CONNECTION_STRING is not set" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AWS_ACCESS_KEY_ID" ]; then | |
echo "Environment variable AWS_ACCESS_KEY_ID is not set" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AWS_SECRET_ACCESS_KEY" ]; then | |
echo "Environment variable AWS_SECRET_ACCESS_KEY is not set" | |
exit 1 | |
fi | |
# check for required environment variables | |
if [ -z "$AWS_DEFAULT_REGION" ]; then | |
echo "Environment variable AWS_DEFAULT_REGION is not set" | |
exit 1 | |
fi | |
# login azcopy | |
azcopy login | |
# use Azcopy to copy the Azure Blob Storage directory to local directory | |
azcopy copy \ | |
"https://$AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$1" \ | |
--destination "." \ | |
--recursive | |
# authorize AWS CLI | |
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID | |
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY | |
# set AWS default region | |
aws configure set default.region $AWS_DEFAULT_REGION | |
# use aws s3 to copy the local directory to the S3 bucket dry run | |
aws s3 cp . $2 --recursive --dryrun | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# check if report already exist, if yes, exit | |
if [ -f report.txt ]; then | |
echo "Report already exist, please delete it before running this script" | |
exit | |
fi | |
# generate report by running generate_report.sh and pipe the result to report.txt | |
echo "Generating report..." | |
./generate_report.sh | tee report.txt | |
# check if report.txt is empty, if yes, exit | |
if [ -s report.txt ]; then | |
echo "Report generated successfully" | |
else | |
echo "Report generated failed" | |
exit | |
# sort report by second column | |
echo "Sorting report..." | |
sort -k2 report.txt > report_sorted.txt | |
# swap first and second column | |
echo "Swapping first and second column..." | |
awk '{print $2, $1}' report_sorted.txt > report_swapped.txt | |
# remove duplicates based on second column | |
echo "Removing duplicates..." | |
uniq -d report_swapped.txt > report_uniq.txt | |
# show first 10 lines | |
echo "Showing first 10 lines..." | |
head -n 10 report_uniq.txt | |
# show last 10 lines | |
echo "Showing last 10 lines..." | |
tail -n 10 report_uniq.txt | |
# check whether report contains entry foobar | |
echo "Checking if report contains entry foobar..." | |
grep -q foobar report_uniq.txt | |
if [ $? -eq 0 ]; then | |
echo "Report contains entry foobar" | |
else | |
echo "Report does not contain entry foobar" | |
fi | |
# publish report to azure storage | |
echo "Publishing report to azure storage..." | |
az storage blob upload -f report_uniq.txt -n report.txt --account-name $storageAccountName --account-key $storageAccountKey --container-name $storageContainerName | |
# publish report to aws s3 | |
echo "Publishing report to aws s3..." | |
aws s3 cp report_uniq.txt s3://$s3BucketName/$s3BucketFolderName/report.txt | |
# tweet report | |
echo "Tweeting report..." | |
twitter --oauth1 --force --consumer-key $twitterConsumerKey --consumer-secret $twitterConsumerSecret --access-token $twitterAccessToken --access-token-secret $twitterAccessTokenSecret --status "Report: $(head -n 10 report_uniq.txt)" |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment