Last active
February 9, 2021 08:46
-
-
Save raczben/a541949d5207404849186c022dd67cf5 to your computer and use it in GitHub Desktop.
Simple python script to fetch gitlab's artifact from a job / from privately
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import logging | |
import sys | |
import os | |
import subprocess | |
import argparse | |
import configparser | |
try: | |
import gitlab | |
except ImportError: | |
print('ERROR: cannot import gitlab package (python-gitlab). Please install them with pip install python-gitlab') | |
sys.exit(-1) | |
here = os.path.dirname(__file__) | |
assert sys.version_info >= (3, 6) | |
logging.basicConfig( | |
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', | |
datefmt='%H:%M:%S', | |
level=logging.INFO) | |
l = logging.getLogger() | |
""" | |
Parse command line arguments. | |
""" | |
p = argparse.ArgumentParser( | |
description='GetLab - Fetch artifacts from GitLab server.') | |
p.add_argument('--config-file', default='getlab.ini', | |
help=('Config file of getlab. The config file contains sections. Each section defines' | |
' an artifact. The section header is the defines the directory where the ' | |
'artifact should be extracted as well.'), | |
) | |
p.add_argument('-v', '--verbose', help='verbose', action='store_true') | |
p.add_argument('--url', help="URL of the GitLab server", | |
default=os.environ.get('CI_SERVER_URL', None)) | |
p.add_argument('--job-token', | |
help="GitLab's Job token: Use this from pipeline-job. More: https://docs.gitlab.com/ee/ci/triggers/#ci-job-token", | |
default=os.environ.get('CI_JOB_TOKEN', None)) | |
p.add_argument('--private-token', | |
help="GitLab's private token: Use this from custom build. More: https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html", | |
default=os.environ.get('GITLAB_PRIVATE_TOKEN', None)) | |
args = p.parse_args() | |
if args.verbose: | |
l.setLevel(logging.DEBUG) | |
print(args) | |
config = configparser.ConfigParser() | |
try: | |
with open(args.config_file) as f: | |
config.read_file(f) | |
except IOError as e: | |
l.error(f'Failed to open config file!') | |
print(e) | |
sys.exit(-1) | |
except Exception as e: | |
l.error(f'Config file parse error') | |
print(e) | |
sys.exit(-1) | |
if args.verbose: | |
print(config._sections) | |
""" | |
Do the fetch. The config file contains sections. Each section defines an artifact. The section | |
header is the defines the directory where the artifact should be extracted as well. | |
""" | |
for dependency in config.sections(): | |
try: | |
url = args.url if args.url else config[dependency].get('url', 'https://gitlab.com/') | |
project_id = config[dependency]['project-id'] | |
job_id = config[dependency]['job-id'] | |
except Exception as e: | |
l.error( | |
f'One of < url, project_id, job_id > cannot fetched for {dependency}') | |
print(e) | |
sys.exit(-1) | |
if args.private_token or args.job_token: | |
gl = gitlab.Gitlab( | |
url, private_token=args.private_token, job_token=args.job_token) | |
else: | |
print( | |
f'Error: Cannot find any token. Please generate it at: {url}/-/profile/personal_access_tokens and give to me with --private-token') | |
sys.exit(-1) | |
# Job token cannot fetch project information https://gitlab.com/gitlab-org/gitlab/-/issues/17511 | |
project = gl.projects.get(project_id, lazy=True) | |
buildjob = project.jobs.get(job_id, lazy=True) | |
# Create directory for the unzipped artifact. | |
if not os.path.exists(dependency): | |
os.makedirs(dependency) | |
try: | |
# Fetch the artifact and extract it. | |
zipfn = "___artifacts.zip" | |
with open(zipfn, "wb") as f: | |
buildjob.artifacts(streamed=True, action=f.write) | |
subprocess.run(["unzip", "-bo", zipfn, '-d', dependency]) | |
os.unlink(zipfn) | |
except gitlab.exceptions.GitlabAuthenticationError as e: | |
l.error(f'Cannot reach site: {url} with token ****') | |
print(e) | |
sys.exit(-1) | |
except gitlab.exceptions.GitlabGetError as e: | |
l.error(f'Project id: {project_id}') | |
l.error(f'Job id: {job_id}') | |
print(e) | |
sys.exit(-1) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment