Skip to content

Instantly share code, notes, and snippets.

@andysingal

andysingal/download.py

Created September 25, 2023 03:34
Show Gist options
  • Save andysingal/613de29e75c794512896a2fb4d80fc4b to your computer and use it in GitHub Desktop.
Save andysingal/613de29e75c794512896a2fb4d80fc4b to your computer and use it in GitHub Desktop.
Download ZIP
Raw
download.py
#@title 1.1 CivitAI.com - Tool download đầy đủ thông tin: Model file, config file, hình preview, file mô tả...
#@markdown [![Badge with Link](https://img.shields.io/badge/CivitAI.com-DOWNLOAD_MODEL_ALL-%2300A8E1?style=flat&logo=GITLAB&logoColor=84a805)](https://stablediffusion.edu.vn/)
from google.colab import drive
drive.mount('/content/gdrive')
# -*- coding: UTF-8 -*-
# handle msg between js and python side
import os
import time
import json
import re
from html import unescape
import requests
import gdown
import shutil
import urllib
from urllib.parse import parse_qsl, urljoin, urlparse
from bs4 import BeautifulSoup
from collections import defaultdict
from pathlib import Path
#Giao diện
url = "https://civitai.com/models/43331/majicmix-realistic" #@param {type:"string"}
Target_Path = "StableDiffusion" #@param {type:"string"}
#khai báo biến mặc định
suffix = ".civitai"
version = "1.6.4"
def_headers = {'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}
root_path ="/content/gdrive/MyDrive/"
proxies = None
imgcount = 1
imgstring = [""]
#Tạo các thư mục nếu chưa có
sSDDir = root_path + Target_Path
sample= !ls $sSDDir
if sample == ["ls: cannot access '" + sSDDir + "': No such file or directory"] :
print("Khởi tạo thư mục lần đầu")
!mkdir $sSDDir
os.chdir(sSDDir)
url_dict = {
"modelPage":"https://civitai.com/models/",
"modelId": "https://civitai.com/api/v1/models/",
"modelVersionId": "https://civitai.com/api/v1/model-versions/",
"hash": "https://civitai.com/api/v1/model-versions/by-hash/"
}
model_type_dict = {
"Checkpoint": "ckp",
"TextualInversion": "ti",
"Hypernetwork": "hyper",
"LORA": "lora",
"LoCon": "LyCORIS",
"Controlnet": "ControlNet",
"VAE": "VAE",
"Poses": "Poses",
"Wildcards": "Wildcards",
"Other": "Others"
}
folders = {
"ti": os.path.join(root_path + Target_Path + "/", "embeddings"),
"hyper": os.path.join(root_path + Target_Path + "/", "hypernetworks"),
"ckp": os.path.join(root_path + Target_Path + "/", "Model"),
"lora": os.path.join(root_path + Target_Path + "/", "Lora"),
"LyCORIS": os.path.join(root_path + Target_Path + "/", "LyCORIS"),
"ControlNet": os.path.join(root_path + Target_Path + "/", "ControlNet"),
"VAE": os.path.join(root_path + Target_Path + "/", "VAE"),
"Poses": os.path.join(root_path + Target_Path + "/", "Poses"),
"Wildcards": os.path.join(root_path + Target_Path + "/", "Wildcards"),
"Others": os.path.join(root_path + Target_Path + "/", "Others"),
}
# get id from url
def get_model_id_from_url(url:str) -> str:
#print("Run get_model_id_from_url")
id = ""
if not url:
print("url or model id can not be empty")
return ""
if url.isnumeric():
# is already an id
id = str(url)
return id
s = re.sub("\?.+$", "", url).split("/")
if len(s) < 2:
print("url is not valid")
return ""
if s[-2].isnumeric():
id = s[-2]
elif s[-1].isnumeric():
id = s[-1]
else:
print("There is no model id in this url")
return ""
return id
def get_model_info_by_id(id:str) -> dict:
#print("Request model info from civitai")
if not id:
print("id is empty")
return
r = requests.get(url_dict["modelId"]+str(id), headers=def_headers, proxies=proxies)
if not r.ok:
if r.status_code == 404:
# this is not a civitai model
print("Civitai does not have this model")
return {}
else:
print("Get error code: " + str(r.status_code))
print("LÀM GÌ CŨNG NÊN TỪ TỪ...VỘI VÀNG LÀ HƯ HẾT NHA !!!!")
print(r.text)
return
# try to get content
content = None
try:
content = r.json()
except Exception as e:
print("Parse response json failed")
print(str(e))
print("response:")
print(r.text)
return
if not content:
print("error, content from civitai is None")
return
return content
###########
def html_to_text(html):
# use non-greedy for remove scripts and styles
text = re.sub("", "", html, flags=re.DOTALL)
text = re.sub("", "", text, flags=re.DOTALL)
# remove other tags
text = re.sub("<[^>]+>", " ", text)
# strip whitespace
text = " ".join(text.split())
# unescape html entities
text = unescape(text)
return text
###########
def get_versioninfo(model_version: dict):
if not model_version:
print("model Version is Empty")
return
string =[""]
#print("debug model version hoho", model_version[0]["modelId"])
string[0] = "Model ID: " + str(model_version[0]["modelId"])
string.append("Model version: " + model_version[0]["name"])
string.append("Ngày tạo: " + model_version[0]["createdAt"])
string.append("Ngày update: " + model_version[0]["updatedAt"])
tmplist = model_version[0]["trainedWords"]
convertlist = ''
for i, trig in enumerate(tmplist):
convertlist += "*trigger "+ str(i+1) + ': ' + trig + ', '
# Remove the trailing comma and space
convertlist = convertlist[:-2]
#print("test convert: ",convertlist)
string.append("Các loại Triggers: " + convertlist)
string.append("Base model: " + model_version[0]["baseModel"])
temptxt = model_version[0]["description"]
if temptxt != None:
temptxt = html_to_text(temptxt)
string.append("Mô tả version này: " + temptxt)
else:
string.append("Mô tả version này: " + "không có !")
filelist = model_version[0]["files"]
string.append("Tên file: " + filelist[0]["name"])
string.append("Kích thuước file: " + str(round(filelist[0]["sizeKB"]/1024))+ " Mb")
string.append("Kiểu model: " + filelist[0]["type"])
string.append("Link down model: " + filelist[0]["downloadUrl"])
string.append("Link model gốc: " + url)
#print("test debug-----> write model version: ", string)
return string
def get_subfolders(folder:str) -> list:
#print("File model sẽ được lưu ở: " + folder)
if not folder:
print("folder can not be None")
return
if not os.path.isdir(folder):
print("path is not a folder")
return
prefix_len = len(folder)
subfolders = []
for root, dirs, files in os.walk(folder, followlinks=True):
for dir in dirs:
full_dir_path = os.path.join(root, dir)
# get subfolder path from it
subfolder = full_dir_path[prefix_len:]
subfolders.append(subfolder)
return subfolders
def get_model_info_by_url(model_url_or_id:str) -> tuple:
print("Getting model info by: " + model_url_or_id)
# parse model id
model_id = get_model_id_from_url(model_url_or_id)
if not model_id:
print("failed to parse model id from url or id")
return
model_info = get_model_info_by_id(model_id)
if model_info is None:
print("Kết nối tới API CivitAI thất bại")
return
if not model_info:
print("failed to get model info from url or id")
return
# parse model type, model name, subfolder, version from this model info
# get model type
if "type" not in model_info.keys():
print("model type is not in model_info")
return
civitai_model_type = model_info["type"]
if civitai_model_type not in model_type_dict.keys():
print("This model type is not supported:"+civitai_model_type)
return
model_type = model_type_dict[civitai_model_type]
# get model type
if "name" not in model_info.keys():
print("model name is not in model_info")
return
model_name = model_info["name"]
if not model_name:
print("model name is Empty")
model_name = ""
# get version list
if "modelVersions" not in model_info.keys():
print("modelVersions is not in model_info")
return
modelVersions = model_info["modelVersions"]
if not modelVersions:
print("modelVersions is Empty")
return
version_strs = []
for version in modelVersions:
# version name can not be used as id
# version id is not readable
# so , we use name_id as version string
version_str = version["name"]+"_"+str(version["id"])
version_strs.append(version_str)
#print("debug model version ---->", modelVersions)
#print("debug stringversion ---->", version_str)
# get folder by model type
folder = folders[model_type]
#print("debug:",folder)
# get subfolders
subfolders = get_subfolders(folder)
if not subfolders:
subfolders = []
# add default root folder
subfolders.append("/")
version_file = get_versioninfo(modelVersions)
#print("Get following info for downloading:")
#print(f"model_name:{model_name}")
#print(f"model_type:{model_type}")
#print(f"subfolders:{subfolders}")
#print(f"version_strs:{version_strs}")
return (model_info, model_name, model_type, folder, version_file)
def load_model_info(path):
# print("Load model info from file: " + path)
model_info = None
with open(os.path.realpath(path), 'r') as f:
try:
model_info = json.load(f)
except Exception as e:
print("Selected file is not json: " + path)
print(e)
return
return model_info
def swap(str1,str2):
temp = str2
str2 = str1
str1 = temp
return str1, str2
def get_general_info(model_id:str, delay:float=3) -> tuple:
# get model info by id from civitai
model_info = get_model_info_by_id(model_id)
# delay before next request, to prevent to be treat as DDoS
print(f"delay:{delay} second")
time.sleep(delay)
if not model_info:
return
#print("Debug model info id: ---->", model_info["id"])
if "modelVersions" not in model_info.keys():
return
modelVersions = model_info["modelVersions"]
#print("Debug model version: ---->", modelVersions[0])
if not modelVersions:
return
if not len(modelVersions):
return
current_version = modelVersions[0]
if not current_version:
return
model_name = ""
if "name" in model_info.keys():
model_name = model_info["name"]
if not model_name:
model_name = ""
new_version_name = ""
if "name" in current_version.keys():
new_version_name = current_version["name"]
if not new_version_name:
new_version_name = ""
description = ""
if "description" in model_info.keys():
description = model_info["description"]
#print("test: description: ",model_info["description"])
if not description:
description = " CivitAI.com Downloader - Designed by Andy N Le Zalo: 0908231181"
current_version_id =""
description = html_to_text(description)
filelist = []
file1_string = [] # file name, file type, file size,file url
temptext = ""
tempnum = 0
if "files" in current_version.keys():
#print("debug---> id hú hú: ",len(current_version["files"]))
n = 0
file_len = len(current_version["files"])
filelist.append([file_len])
while n < len(current_version["files"]):
filename_str = ""
file1_string = []
filename_str = current_version["files"][n]
fname,temptext = list(filename_str.items())[3] #name
file1_string.append(temptext)
name,temptext = list(filename_str.items())[4] #type
file1_string.append(temptext)
fname,tempnum = list(filename_str.items())[2] #size
file1_string.append(tempnum)
fname,temptext = list(filename_str.items())[12] #url
file1_string.append(temptext)
filelist.append(file1_string)
n += 1
#print("debug--->fname, file type: ",fname,model_filename)
#print("debug--->model filename,file size, file type: ", model_filename, model_filesize, model_filetype)
#print(filename_str)
#print("mục 11: ",list(filename_str.items())[11])
#print("debug--> download URL:", downloadUrl)
# get multi preview images
img_url = ""
#print("debug img url string 1---->: ", current_version["images"][1])
if "images" in current_version.keys():
if current_version["images"][0]:
imgstring[0] = current_version["images"][0]["url"]
imgstring[0] = imgstring[0].replace('width=450/', '')
#print("debug img string inside ---->: ", imgstring[0])
imgcount = 1
while (imgcount < len(current_version["images"])):
if "url" in current_version["images"][imgcount].keys():
img_dict = current_version["images"][imgcount]["url"]
img_dict = img_dict.replace('width=450/', '')
imgstring.append(img_dict)
#print("test--->",imgstring)
#print("count: ",imgcount)
imgcount +=1
else:
print("Not found !!!!!")
break
return (filelist, current_version_id, new_version_name, description, imgcount)
def get_filename_from_url(url):
if url.find('https://civitai.com/')!=-1:
convert = '' + url.replace('download/models', 'v1/model-versions')
req = requests.get(convert, stream=True)
basename, extension = os.path.splitext(req.json()['files'][0]['name'].replace(' ', '_'))
else:
parse = urlparse(url).path
req = parse[parse.rfind('/') + 1:].replace(' ', '_')
basename, extension = os.path.splitext(req)
return basename, extension
def mass_imgs_down(Strings ="",totalnumber = 0,download_path =""):
n = 0
if Strings == "":
print("Không có gì để down hết :( ")
return
if totalnumber == 0:
print("Không biết số lượng sao down hả pa ? ")
return
if download_path == "":
print("Không biết lưu chỗ nào luôn ? ")
return
final_target = f'{download_path}'
os.chdir(final_target)
#print("test final target---->",final_target)
while (n < totalnumber):
OIname,OIext = get_filename_from_url(Strings[n])
OIfullname = OIname+ OIext
gdown.download(Strings[n], OIfullname, quiet=False, fuzzy=True)
n +=1
def download_firstimg(imgurl, downloadpath, filename):
fname, extension = os.path.splitext(filename)
imgname = fname+ ".png"
imgdest = os.path.join(downloadpath,imgname)
OIname,OIext = get_filename_from_url(imgurl)
OIfullname = OIname+ OIext
final_target = f'{downloadpath}'
os.chdir(final_target)
gdown.download(imgurl, OIfullname, quiet=False, fuzzy=True)
os.rename(OIfullname, imgname)
def start_downloading(url, downloadpath, filename):
fname, extension = os.path.splitext(filename)
final_target = f'{downloadpath}'
os.chdir(final_target)
gdown.download(url, fname+extension, quiet=False, fuzzy=True)
print("file của bạn đã download thành công về thư mục: ",final_target,"tên file: ",fname+extension)
#Bắt đầu download
version_file = [""]
if not url.find('https://civitai.com/')!=-1: #not Civit
print("Tool này chỉ hỗ trợ download CivitAI.com !")
else:
model_id = get_model_id_from_url(url)
model_info, model_name, model_type, folder, version_file = get_model_info_by_url(url)
#print("debug---> model info: ", model_info)
filelist, current_version_id, new_version_name, description, imgcount = get_general_info(model_id)
#print("DEBUG: ---> filelist, current_version_id, new_version_name, description, imgcount")
#print(filelist, current_version_id, new_version_name, description, imgcount)
model_name = ''.join(e for e in model_name if (e.isalnum() or e.isspace()))
print("Loại model: ",model_type)
print("Có ",filelist[0][0], " models :")
print("version: ",new_version_name)
#Tạo thư mục nếu chưa có
sample= !ls $folder
if sample == ["ls: cannot access '" + folder + "': No such file or directory"] :
!mkdir $folder
os.chdir(folder)
if not os.path.exists(model_name): os.mkdir(model_name)
else: print("Thư mục: ", model_name,"đã tồn tại trên drive, tui sẽ ghi đè lên nha !")
dst_folder = os.path.join(folder,model_name)
os.chdir(dst_folder)
n = 0
description = description +"\n\n"+"CivitAI.com Downloader - stablediffusion.edu.vn"
with open('Description.txt', 'w') as f:
f.write(description)
if not os.path.exists("Model Images"): os.mkdir("Model Images")
dst_folder= os.path.join(dst_folder,"Model Images")
os.chdir(dst_folder)
mass_imgs_down(imgstring,imgcount,dst_folder)
dst_folder = str(Path(dst_folder).parents[0])
os.chdir(dst_folder)
while n < filelist[0][0]:
print("\n Đang tải file thứ:",n+1, filelist[n+1][3],"tới thư mục: ", dst_folder)
start_downloading(filelist[n+1][3], dst_folder, filelist[n+1][0])
n += 1
if(n)>1:
version_file.append("Lưu ý: Model này có:"+ str(n)+ " file model")
version_file.append("Tên file bổ sung: "+ filelist[n][0])
version_file.append("Loại model: "+ filelist[n][1])
version_file.append("Kích thuớc: "+ str(round(filelist[n][2]/1024))+ " Mb")
version_file.append("Link down model: "+ filelist[n][3])
#print("TEST Version file:----> ",n," ----", version_file)
#print("TEST Version file name:----> ", filelist[2])
with open('Model info.txt', 'w') as file:
for item in version_file:
file.write("%s\n" % item)
img_url = imgstring[0]
download_firstimg(img_url, dst_folder, filelist[1][0])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment