Last active
September 14, 2021 21:01
-
-
Save markcmiller86/2c360eb41129c50c386394360f0890c2 to your computer and use it in GitHub Desktop.
Python3 GraphQl cript to transfer issues
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Copyright (c) 2021, Lawrence Livermore National Security, LLC | |
# | |
# Python3 script using GraphQL interface to GitHub to transfer | |
# issues between GitHub repos. | |
# | |
# Programmer: Mark C. Miller, Tue Jul 20 10:21:40 PDT 2021 | |
# | |
import datetime, email.header, glob, mailbox, os, pytz | |
import re, requests, shutil, sys, textwrap, time | |
from difflib import SequenceMatcher | |
# | |
# Capture failure details to a continuously appending file | |
# | |
def captureGraphQlFailureDetails(gqlQueryName, gqlQueryString, gqlResultString): | |
with open("email2discussions-failures-log.txt", 'a') as f: | |
f.write("%s - %s\n"%(datetime.datetime.now().strftime('%y%b%d %I:%M:%S'),gqlQueryName)) | |
f.write("--------------------------------------------------------------------------\n") | |
f.write(gqlResultString) | |
f.write("\n") | |
f.write("--------------------------------------------------------------------------\n") | |
f.write(gqlQueryString) | |
f.write("\n") | |
f.write("--------------------------------------------------------------------------\n\n\n\n") | |
# | |
# Read token from 'ghToken.txt' | |
# | |
def GetGHToken(): | |
if not hasattr(GetGHToken, 'ghToken'): | |
try: | |
with open('ghToken.txt', 'r') as f: | |
GetGHToken.ghToken = f.readline().strip() | |
except: | |
raise RuntimeError('Put a GitHub token in \'ghToken.txt\' readable only by you.') | |
return GetGHToken.ghToken | |
# | |
# Build standard header for URL queries | |
# | |
headers = \ | |
{ | |
'Content-Type': 'application/json', | |
'Authorization': 'bearer %s'%GetGHToken(), | |
'GraphQL-Features': 'discussions_api' | |
} | |
# | |
# Workhorse routine for performing a GraphQL query | |
# | |
def run_query(query): # A simple function to use requests.post to make the API call. Note the json= section. | |
if not hasattr(run_query, 'numSuccessiveFailures'): | |
run_query.numSuccessiveFailures = 0; | |
# Post the request. Time it and keep 100 most recent times in a queue | |
try: | |
request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers) | |
result = request.json() | |
run_query.numSuccessiveFailures = 0 | |
except: | |
captureGraphQlFailureDetails('run_query', query, "") | |
run_query.numSuccessiveFailures += 1 | |
if run_query.numSuccessiveFailures > 3: | |
raise Exception(">3 successive query failures, exiting...") | |
sys.exit(1) | |
if request.status_code == 200: | |
return request.json() | |
else: | |
raise Exception("run_query failed with code of {}. {} {}".format(request.status_code, query, request.json())) | |
# | |
# A method to periodically call to ensure we don't | |
# exceed GitHub's rate limits | |
# | |
def throttleRate(): | |
# set the *last* check 61 seconds in the past to force a check | |
# the very *first* time we run this | |
if not hasattr(throttleRate, 'lastCheckNow'): | |
throttleRate.lastCheckNow = datetime.datetime.now()-datetime.timedelta(seconds=61) | |
query = """ | |
query | |
{ | |
viewer | |
{ | |
login | |
} | |
rateLimit | |
{ | |
limit | |
remaining | |
resetAt | |
} | |
} | |
""" | |
# Perform this check only about once a minute | |
now = datetime.datetime.now() | |
if (now - throttleRate.lastCheckNow).total_seconds() < 60: | |
return | |
throttleRate.lastCheckNow = now | |
try: | |
result = run_query(query) | |
zuluOffset = 7 * 3600 # subtract PDT timezone offset from Zulu | |
if 'errors' in result.keys(): | |
toSleep = (throttleRate.resetAt-now).total_seconds() - zuluOffset + 1 | |
print("Reached end of available queries for this cycle. Sleeping %g seconds..."%toSleep) | |
time.sleep(toSleep) | |
return | |
# Gather rate limit info from the query result | |
limit = result['data']['rateLimit']['limit'] | |
remaining = result['data']['rateLimit']['remaining'] | |
# resetAt is given in Zulu (UTC-Epoch) time | |
resetAt = datetime.datetime.strptime(result['data']['rateLimit']['resetAt'],'%Y-%m-%dT%H:%M:%SZ') | |
toSleep = (resetAt-now).total_seconds() - zuluOffset | |
print("GraphQl Throttle: limit=%d, remaining=%d, resetAt=%g seconds"%(limit, remaining, toSleep)) | |
# Capture the first valid resetAt point in the future | |
throttleRate.resetAt = resetAt | |
if remaining < 200: | |
print("Reaching end of available queries for this cycle. Sleeping %g seconds..."%toSleep) | |
time.sleep(toSleep) | |
except: | |
captureGraphQlFailureDetails('rateLimit', query, "") | |
# | |
# Get various visit-dav org. repo ids. Caches results so that subsequent | |
# queries don't do any graphql work. | |
# | |
def GetRepoID(orgname, reponame): | |
query = """ | |
query | |
{ | |
repository(owner: \"%s\", name: \"%s\") | |
{ | |
id | |
} | |
} | |
"""%(orgname, reponame) | |
if not hasattr(GetRepoID, reponame): | |
result = run_query(query) | |
# result = {'data': {'repository': {'id': 'MDEwOlJlcG9zaXRvcnkzMjM0MDQ1OTA='}}} | |
setattr(GetRepoID, reponame, result['data']['repository']['id']) | |
return getattr(GetRepoID, reponame) | |
# | |
# Get object id by name for given repo name and org/user name. | |
# Caches reponame/objname pair so that subsequent queries don't do any | |
# graphql work. | |
# | |
def GetObjectIDByName(orgname, reponame, gqlObjname, gqlCount, objname): | |
query = """ | |
query | |
{ | |
repository(owner: \"%s\", name: \"%s\") | |
{ | |
%s(first:%d) | |
{ | |
edges | |
{ | |
node | |
{ | |
description, | |
id, | |
name | |
} | |
} | |
} | |
} | |
} | |
"""%(orgname, reponame, gqlObjname, gqlCount) | |
if not hasattr(GetObjectIDByName, "%s.%s"%(reponame,objname)): | |
result = run_query(query) | |
# result = d['data']['repository']['discussionCategories']['edges'][0] = | |
edges = result['data']['repository'][gqlObjname]['edges'] | |
for e in edges: | |
if e['node']['name'] == objname: | |
setattr(GetObjectIDByName, "%s.%s"%(reponame,objname), e['node']['id']) | |
break | |
return getattr(GetObjectIDByName, "%s.%s"%(reponame,objname)) | |
def getIssueIDByNumber(orgname, reponame, issuenum): | |
query = """ | |
query | |
{ | |
repository(owner: \"%s\", name: \"%s\") | |
{ | |
issue(number:%d) | |
{ | |
id, | |
title | |
} | |
} | |
} | |
"""%(orgname, reponame, issuenum) | |
try: | |
result = run_query(query) | |
if 'errors' in result and len(result['errors']) and \ | |
'type' in result['errors'][0]: | |
if result['errors'][0]['type'] == 'NOT_FOUND': | |
return False | |
else: | |
return result['data']['repository']['issue']['id'] | |
except: | |
captureGraphQlFailureDetails('issue (by-number) %d'%issuenum, query, | |
repr(result) if 'result' in locals() else "") | |
return None | |
# | |
# lock an object (primarily to lock a discussion) | |
# | |
def lockLockable(nodeid): | |
query = """ | |
mutation | |
{ | |
lockLockable(input: | |
{ | |
clientMutationId:\"scratlantis:emai2discussions.py\", | |
lockReason:RESOLVED, | |
lockableId:\"%s\" | |
}) | |
{ | |
lockedRecord | |
{ | |
locked | |
} | |
} | |
}"""%nodeid | |
try: | |
result = run_query(query) | |
except: | |
captureGraphQlFailureDetails('lockLockable %s'%nodeid, query, | |
repr(result) if 'result' in locals() else "") | |
# | |
# Add a convenience label to each discussion | |
# The label id was captured during startup | |
# | |
def addLabelsToLabelable(nodeid, labid): | |
query = """ | |
mutation | |
{ | |
addLabelsToLabelable(input: | |
{ | |
clientMutationId:\"scratlantis:emai2discussions.py\", | |
labelIds:[\"%s\"], | |
labelableId:\"%s\" | |
}) | |
{ | |
labelable | |
{ | |
labels(first:1) | |
{ | |
edges | |
{ | |
node | |
{ | |
id | |
} | |
} | |
} | |
} | |
} | |
}"""%(labid, nodeid) | |
try: | |
result = run_query(query) | |
except: | |
captureGraphQlFailureDetails('addLabelsToLabelable %s'%nodeid, query, | |
repr(result) if 'result' in locals() else "") | |
def transferIssue(issueid, repoid): | |
query = """ | |
mutation | |
{ | |
transferIssue(input: | |
{ | |
issueId:\"%s\", | |
repositoryId:\"%s\" | |
}) | |
{ | |
issue | |
{ | |
id | |
} | |
} | |
}"""%(issueid, repoid) | |
try: | |
result = run_query(query) | |
return result['data']['transferIssue']['issue']['id'] | |
except: | |
captureGraphQlFailureDetails('transferIssue %s'%issueid, query, | |
repr(result) if 'result' in locals() else "") | |
# | |
# Main Program | |
# | |
# Get the repository id where the discussions will be created | |
dstrepoid = GetRepoID("visit-dav", "visit") | |
# Get the label id for the 'visit-uers email' | |
labid = GetObjectIDByName("visit-dav", "visit", "labels", 30, "sre") | |
for i in range(350): | |
print("Working on issue %d"%i) | |
if i%20 == 0: | |
throttleRate() | |
issueid = getIssueIDByNumber("visit-dav", "live-customer-response", i) | |
if issueid: | |
newid = transferIssue(issueid, dstrepoid) | |
print("Completed transfer") | |
addLabelsToLabelable(newid, labid) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment