Created
August 26, 2022 12:56
-
-
Save YsHaNgM/690d8658bcd414298fd9486ad0d58dec to your computer and use it in GitHub Desktop.
Build chromium behind a proxy needs some tricks to download hooks
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
@@ -25,6 +25,7 @@ import time | |
import subprocess2 | |
+import download_help | |
# Env vars that tempdir can be gotten from; minimally, this | |
# needs to match python's tempfile module and match normal | |
@@ -276,27 +277,27 @@ def _downloader_worker_thread(thread_num, q, force, base_url, | |
# Check if file exists. | |
file_url = '%s/%s' % (base_url, input_sha1_sum) | |
(code, _, err) = gsutil.check_call('ls', file_url) | |
- if code != 0: | |
- if code == 404: | |
- out_q.put('%d> File %s for %s does not exist, skipping.' % ( | |
- thread_num, file_url, output_filename)) | |
- ret_codes.put((1, 'File %s for %s does not exist.' % ( | |
- file_url, output_filename))) | |
- elif code == 401: | |
- out_q.put( | |
- """%d> Failed to fetch file %s for %s due to unauthorized access, | |
- skipping. Try running `gsutil.py config` and pass 0 if you don't | |
- know your project id.""" % (thread_num, file_url, output_filename)) | |
- ret_codes.put( | |
- (1, 'Failed to fetch file %s for %s due to unauthorized access.' % | |
- (file_url, output_filename))) | |
- else: | |
- # Other error, probably auth related (bad ~/.boto, etc). | |
- out_q.put('%d> Failed to fetch file %s for %s, skipping. [Err: %s]' % | |
- (thread_num, file_url, output_filename, err)) | |
- ret_codes.put((1, 'Failed to fetch file %s for %s. [Err: %s]' % | |
- (file_url, output_filename, err))) | |
- continue | |
# Fetch the file. | |
if verbose: | |
out_q.put('%d> Downloading %s@%s...' % | |
@@ -308,7 +309,8 @@ def _downloader_worker_thread(thread_num, q, force, base_url, | |
if os.path.exists(output_filename): | |
out_q.put('%d> Warning: deleting %s failed.' % ( | |
thread_num, output_filename)) | |
- code, _, err = gsutil.check_call('cp', file_url, output_filename) | |
+ download_help.download_gs_to_file(file_url, output_filename) | |
if code != 0: | |
out_q.put('%d> %s' % (thread_num, err)) | |
ret_codes.put((code, err)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import urllib.request | |
import os | |
httpsPrefix = "https://storage.googleapis.com/" | |
gsPrefix = "gs://" | |
def download_gs_to_file(url, fileName): | |
download_http_to_file(url.replace(gsPrefix, httpsPrefix), fileName) | |
def download_http_to_file(url, fileName): | |
path=os.path.dirname(fileName) | |
if not os.path.exists(path): | |
os.makedirs(path) | |
response = urllib.request.urlopen(url) | |
CHUNK = 16 * 1024 | |
with open(fileName, 'wb') as f: | |
while True: | |
chunk = response.read(CHUNK) | |
if not chunk: | |
break | |
f.write(chunk) | |
print ('download ......ok') | |
if __name__ == "__main__": | |
print ('Download from gs:// behind proxy') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment