Skip to content

Instantly share code, notes, and snippets.

@ankurcha
Created January 16, 2020 05:15
Show Gist options
  • Save ankurcha/d09eccddba4d82df552a44a12f8f3610 to your computer and use it in GitHub Desktop.
Save ankurcha/d09eccddba4d82df552a44a12f8f3610 to your computer and use it in GitHub Desktop.
Simple python script to list running transfer jobs
import googleapiclient.discovery
import warnings
warnings.filterwarnings("ignore", "Your application has authenticated using end user credentials")
storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')
def printTable(myDict, colList=None):
""" Pretty print a list of dictionaries (myDict) as a dynamically sized table.
If column names (colList) aren't specified, they will show in random order.
Author: Thierry Husson - Use it as you want but don't blame me.
"""
if not colList: colList = list(myDict[0].keys() if myDict else [])
myList = [colList] # 1st row = header
for item in myDict: myList.append([str(item[col] if item[col] is not None else '') for col in colList])
colSize = [max(map(len,col)) for col in zip(*myList)]
formatStr = ' | '.join(["{{:<{}}}".format(i) for i in colSize])
myList.insert(1, ['-' * i for i in colSize]) # Seperating line
for item in myList: print(formatStr.format(*item))
def comma(num):
'''Add comma to every 3rd digit. Takes int or float and
returns string.'''
if type(num) == int:
return '{:,}'.format(num)
elif type(num) == float:
return '{:,.2f}'.format(num) # Rounds to 2 decimal places
else:
print("Need int or float as input to function comma()!")
def list_in_progress_jobs(project_id):
filterString = ('{{"project_id": "{}", "transfer_statuses": ["IN_PROGRESS"]}}').format(project_id)
result = storagetransfer.transferOperations().list(name="transferOperations", filter=filterString).execute()
rows = []
i = 0
for op in result['operations']:
i = i + 1
name = op['name']
counters = op['metadata']['counters']
done = int(counters.get('bytesCopiedToSink',0))
found = int(counters.get('bytesFoundFromSource',0))
percent = 0.0
if found > 0.0:
percent = round(100*done/found, 2)
if percent >= 100.0:
continue
row = {'project_id':project_id, 'name': name, 'percent': percent, 'done': comma(done), 'found': comma(found)}
rows.append(row)
rows = sorted(rows, key=lambda k: -1*k['percent'])
printTable(rows)
print("Total jobs: {}\n".format(i))
if __name__ == "__main__":
project_id = os.environ.get('GOOGLE_CLOUD_PROJECT')
list_in_progress_jobs(project_id)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment