Created
December 5, 2023 07:40
-
-
Save devops-school/4a3f6f2c46e25af6aa891627bec26411 to your computer and use it in GitHub Desktop.
AWS Tutorials: Python code to sync S3 Buckets
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import boto3 | |
import os | |
def sync_s3_bucket_to_local(bucket_name, local_directory): | |
# Initialize S3 client | |
s3_client = boto3.client('s3') | |
# Create local directory if it does not exist | |
if not os.path.exists(local_directory): | |
os.makedirs(local_directory) | |
# List all objects in the S3 bucket | |
bucket_objects = s3_client.list_objects_v2(Bucket=bucket_name) | |
if 'Contents' in bucket_objects: | |
for obj in bucket_objects['Contents']: | |
key = obj['Key'] | |
local_file_path = os.path.join(local_directory, key) | |
# Check if directory, if so, create directory | |
if '/' in key: | |
directory = os.path.dirname(local_file_path) | |
if not os.path.exists(directory): | |
os.makedirs(directory) | |
if key.endswith('/'): | |
# It's a directory, no need to download | |
continue | |
# Download file from S3 to local directory | |
print(f"Downloading {key} to {local_file_path}") | |
s3_client.download_file(bucket_name, key, local_file_path) | |
else: | |
print("No objects found in the bucket.") | |
# Sync the specified S3 bucket to the local directory | |
bucket_name = 'devops4342343244344342' | |
local_directory = 'raj' | |
sync_s3_bucket_to_local(bucket_name, local_directory) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment