Last active
August 18, 2018 22:19
-
-
Save VelocityRa/c01699914c0179eb05d78bee2aeaf9c1 to your computer and use it in GitHub Desktop.
Retrieves log files from given repositories' Issues for offline/batch processing/searching
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
""" | |
Retrieves log files from given ("REPO_NAMES") repositories | |
for offline/batch processing/searching. | |
Iterates thought all issues, finds the last posted comment | |
that contains at least one file and downloads all files to | |
appropriately named directories, with the filename being | |
the Issue title (game name/titleID). | |
Replaces invalid characters in that file name with '-' and | |
if there are multiple files in an issue, appends the | |
filenames with "_N" where "N" is a number starting from 0. | |
Author: VelocityRa | |
Date: 18/8/2018 | |
""" | |
from github import * | |
import re | |
import os | |
import os.path | |
import urllib.request | |
# Enter your GitHub token here | |
GITHUB_TOKEN = "" | |
REPO_NAMES = ["Vita3K/compatibility", "Vita3K/homebrew-compatibility"] | |
LOGS_BASE_PATH = "logs" | |
def find_log_files(comment): | |
return log_files_re.findall(comment) | |
def fixup_log_file_paths(log_files, repo_name): | |
return [r"https://github.com/" + repo_name + "/" + log_file for log_file in log_files] | |
def normalize_file_name(file_name): | |
return file_name.replace(":", "-").replace(r"/", "-").replace(r"\\", "-").replace("*", "-") | |
try: | |
log_files_re = re.compile(r"/files/\d+/\w+\.\w+") | |
logs_path = os.path.normpath(os.path.join(os.getcwd(), LOGS_BASE_PATH)) | |
if not os.path.exists(logs_path): | |
os.mkdir(logs_path) | |
# enable_console_debug_logging() | |
g = Github(login_or_token=GITHUB_TOKEN) | |
for repo_full_name in REPO_NAMES: | |
print("Getting logs for repo: {}".format(repo_full_name)) | |
(_, repo_name) = os.path.split(repo_full_name) | |
log_base_path = os.path.join(logs_path, repo_name) | |
if not os.path.exists(log_base_path): | |
os.mkdir(log_base_path) | |
repo = g.get_repo(repo_full_name, lazy=False) | |
issues = repo.get_issues() | |
for issue in issues: | |
comments_num = issue.comments | |
logs_posted = [] | |
if comments_num == 0: | |
last_comment = issue.body | |
else: | |
last_comment = issue.get_comments()[comments_num - 1].body | |
# if on logs are found on last message, revert to main issue | |
logs_posted = find_log_files(last_comment) | |
if len(logs_posted) == 0: | |
last_comment = issue.body | |
if len(logs_posted) == 0: | |
logs_posted = find_log_files(last_comment) | |
logs_posted = fixup_log_file_paths(logs_posted, repo_full_name) | |
print("Issue #{} for game {}".format(issue.id, issue.title)) | |
for log_id, log in enumerate(logs_posted): | |
normalized_file_name = normalize_file_name(issue.title) | |
# name multiple files in a comment appropriately | |
if len(logs_posted) > 1: | |
normalized_file_name += "_" + str(log_id) | |
normalized_file_name += ".log" | |
log_full_path_cur = os.path.join(log_base_path, normalized_file_name) | |
print("Retrieving: {} as {}".format(log, normalized_file_name)) | |
urllib.request.urlretrieve(log, log_full_path_cur) | |
except GithubException as e: | |
print(e) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment