Last active
April 27, 2020 09:38
-
-
Save Hydriz/4413028 to your computer and use it in GitHub Desktop.
Multipart uploading using Python boto.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
# -*- coding: utf-8 -*- | |
# Copyright (C) 2012-2015 Hydriz Scholz | |
# This program is free software: you can redistribute it and/or modify | |
# it under the terms of the GNU General Public License as published by | |
# the Free Software Foundation, either version 3 of the License, or | |
# (at your option) any later version. | |
# | |
# This program is distributed in the hope that it will be useful, | |
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
# GNU General Public License for more details. | |
# | |
# You should have received a copy of the GNU General Public License | |
# along with this program. If not, see <http://www.gnu.org/licenses/>. | |
import boto | |
from boto.s3.connection import OrdinaryCallingFormat | |
from filechunkio import FileChunkIO | |
import math | |
import os | |
import settings | |
import sys | |
class DumpsMultipart: | |
def __init__( self ): | |
self.accesskey = settings.accesskey | |
self.secretkey = settings.secretkey | |
self.conn = boto.connect_s3( self.accesskey, self.secretkey, host='s3.us.archive.org', is_secure=False, calling_format=OrdinaryCallingFormat() ) | |
def upload( self, dumpfile, identifier ): | |
item = self.conn.get_bucket( identifier ) | |
filesize = os.stat( dumpfile ).st_size | |
header = { | |
'x-archive-queue-derive': '0' | |
} | |
# Initiate the multipart upload request | |
mp = item.initiate_multipart_upload( os.path.basename( dumpfile ), headers=header ) | |
# Use a chunk size of 200 MB | |
chunksize = 209715200 | |
chunkcount = int( math.ceil( filesize / chunksize ) ) | |
# Send the file parts, using FileChunkIO to create a file-like object | |
# that points to a certain byte range within the original file. We | |
# set bytes to never exceed the original file size. | |
for i in range( chunkcount + 1 ): | |
offset = chunksize * i | |
bytes = min( chunksize, filesize - offset ) | |
with FileChunkIO( dumpfile, 'r', offset=offset, bytes=bytes ) as fp: | |
mp.upload_part_from_file( fp, part_num=i + 1, headers=header ) | |
# Complete the multipart upload | |
mp.complete_upload() | |
def process( self ): | |
self.upload( sys.argv[1], sys.argv[2] ) | |
if __name__ == "__main__": | |
DumpsMultipart = DumpsMultipart() | |
DumpsMultipart.process() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment