Last active
August 22, 2022 13:09
-
-
Save z3nth10n/0aa672cb045c6aaa3e991afdaaad3ad1 to your computer and use it in GitHub Desktop.
Divides files into chunks of 1000 VALUES for INSERT INTO and then divide it into chunks of 100k lines
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import glob | |
from math import floor | |
def rreplace(s, old, new, count): | |
return (s[::-1].replace(old[::-1], new[::-1], count))[::-1] | |
i = 0 | |
j=0 | |
files = glob.glob("./*.sql") | |
created = False | |
creating = False | |
create_index = False | |
insert_statment = "" | |
dump_filename="" | |
file_in=None | |
file_out=None | |
for ffile in files: | |
file=ffile.replace('.sql', '') | |
ii = floor(i/100000) | |
dump_filename=f"dumps/{file}_{ii}.sql" | |
try: | |
if not file_in.closed: | |
file_in.close() | |
except: | |
print("") | |
try: | |
if not file_out.closed: | |
file_out.close() | |
except: | |
print("") | |
file_in=open(ffile, 'r') | |
file_out=open(dump_filename, 'w') | |
sb = "" | |
print(f"Starting with {ffile}") | |
for line in file_in: | |
if (not created and line.startswith('CREATE TABLE')) or creating: | |
creating = True | |
file_out.write(line) | |
if line.strip().startswith('SELECT'): | |
created = True | |
creating = False | |
else: | |
print(f"Creating: {creating} | Created: {created}") | |
continue | |
if line.startswith('CREATE INDEX') and not create_index: | |
print(f"Create index: {create_index}") | |
create_index = True | |
file_out.write(line) | |
continue | |
if not line.startswith("INSERT INTO"): | |
continue | |
elif len(insert_statment) == 0: | |
insert_statment = line.split(' VALUES ')[0] + ' VALUES ' | |
if i % 1000 == 0: | |
print(f"{i} done!") | |
if i > 0: | |
file_out.write(sb) | |
sb = insert_statment | |
j=0 | |
try: | |
l = line.split(' VALUES ')[1] | |
if (i + 1) % 1000 != 0: | |
l = l.replace(';', ',') | |
sb += l | |
i += 1 | |
j += 1 | |
except Exception as ex: | |
print(ex) | |
print(f"[{i}] at line: {line}") | |
iii = floor(i/100000) | |
ddump_filename=f"dumps/{file}_{iii}.sql" | |
if ddump_filename != dump_filename: | |
dump_filename = ddump_filename | |
file_out.close() | |
file_out=open(dump_filename, 'w') | |
if j > 0: | |
file_out.write(rreplace(sb, ',', ';', 1)) | |
j = 0 | |
i=0 | |
created=False | |
creating=False | |
create_index=False | |
insert_statment="" | |
file_in.close() | |
file_out.close() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment