Created
April 5, 2020 02:24
-
-
Save jrmdev/cb84b0058ad155329e7f20813c7c5e64 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python | |
# ExplorOz EOTopo MAP | |
# Download tiles from https://www.exploroz.com/eotopo | |
# Get some cookies first and add them below. | |
# To remove empty files when it finishes use: find . -type d -empty -delete | |
# To compress the files use: advpng -z4 | |
# To build an .mbtiles flies from the resulting folders, use https://github.com/mapbox/mbutil | |
import sys | |
import os | |
import math | |
import requests | |
import random | |
import threading | |
import queue | |
import base64 | |
import os.path | |
from pathlib import Path | |
headers = { | |
'Host': 'ts.exploroz.com', | |
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0', | |
'Accept': 'image/webp,*/*', | |
'Accept-Language': 'en-US,en;q=0.5', | |
'Accept-Encoding': 'gzip, deflate, br', | |
'Referer': 'https://www.exploroz.com/eotopo', | |
'Origin': 'https://www.exploroz.com', | |
'Connection': 'keep-alive' | |
} | |
cookies = { | |
'CloudFront-Policy': '<cookie here>', | |
'CloudFront-Signature': '<cookie here>', | |
'CloudFront-Key-Pair-Id': '<cookie here>', | |
# Add more cookies if necessary. | |
} | |
class ThreadClass(threading.Thread): | |
def __init__(self, queue): | |
threading.Thread.__init__(self) | |
self.queue = queue | |
def run(self): | |
while True: | |
arg = self.queue.get() | |
download_url(arg[0], arg[1], arg[2]) | |
self.queue.task_done() | |
def download_url(zoom, xtile, ytile): | |
url = "https://ts.exploroz.com/2019/%d/%d/%d?mb=1" % (zoom, xtile, ytile) | |
dir_path = "/root/Toshiba/maps/EOTopo/%d/%d/" % (zoom, xtile) | |
download_path = "/root/Toshiba/maps/EOTopo/%d/%d/%d.png" % (zoom, xtile, ytile) | |
if not os.path.exists(dir_path): | |
try: | |
os.makedirs(dir_path) | |
except: | |
pass | |
if not os.path.isfile(download_path): | |
res = requests.get(url, headers=headers, cookies=cookies) | |
if res.status_code == 200: | |
print("downloaded %r" % url) | |
fp = open(download_path, 'wb') | |
fp.write(res.content) | |
fp.close() | |
elif res.status_code == 404: | |
print('empty %r' % url) | |
Path(download_path).touch() | |
else: | |
return | |
else: | |
pass | |
if __name__ == '__main__': | |
minzoom = 0 | |
maxzoom = 13 | |
ranges = { | |
# zoom: [(minx, maxx), (miny, maxy)] | |
0: [(0, 2), (0, 2)], | |
1: [(0, 4), (0, 4)], | |
2: [(0, 8), (0, 8)], | |
3: [(5, 7), (4, 5)], | |
4: [(13, 14), (8, 10)], | |
5: [(26, 29), (16, 20)], | |
6: [(52, 59), (33, 40)], | |
7: [(104, 118), (67, 81)], | |
8: [(207, 237), (134, 162)], | |
9: [(416, 472), (268, 324)], | |
10: [(832, 950), (538, 651)], | |
11: [(1665, 1897), (1074, 1302)], | |
12: [(3331, 3801), (2149, 2606)], | |
13: [(6662, 7602), (4299, 5213)], | |
} | |
queue = queue.Queue() | |
# Create number process | |
for i in range(int(sys.argv[1])): | |
t = ThreadClass(queue) | |
t.setDaemon(True) | |
t.start() | |
for zoom in range(minzoom, int(maxzoom)+1, 1): | |
xtile, final_xtile = ranges[zoom][0] | |
ytile, final_ytile = ranges[zoom][1] | |
print("%d:%d-%d/%d-%d" % (zoom, xtile, final_xtile, ytile, final_ytile)) | |
for x in range(xtile, final_xtile + 1): | |
for y in range(ytile, final_ytile + 1): | |
queue.put((zoom, x, y)) | |
queue.join() | |
try: | |
while not queue.empty(): | |
time.sleep(1) | |
except KeyboardInterrupt: | |
sys.stdout.write("\r\n") | |
sys.stdout.flush() | |
sys.exit() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment