Created
June 15, 2012 11:54
-
-
Save pilt/2936089 to your computer and use it in GitHub Desktop.
Riak storage comparisons
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Test case | |
---------- | |
Riak version 1.1.2. Insert 5,000 items. Do it five times and compute | |
means. The data dir size is calculated before and after iterations. | |
Bitcask vs. LevelDB | |
------------------- | |
Disk space: 1.113 : 1.000 | |
Time taken: 1.000 : 1.016 | |
Bitcask | |
------- | |
times...........: [12.415955781936646, 11.905525922775269, 12.327656030654907, 11.583511114120483, 11.949213027954102] | |
mean time.......: 12.0363723755 | |
size changes....: [48871379, 48871763, 48871616, 48871697, 48871487] | |
mean size change: 48871588.4 | |
LevelDB | |
------- | |
times...........: [12.596890926361084, 12.285731077194214, 12.137356996536255, 12.081082820892334, 12.035972118377686] | |
mean time.......: 12.2274067879 | |
size changes....: [43905054, 43905057, 43904910, 43904759, 43904896] | |
mean size change: 43904935.2 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Test case | |
---------- | |
Riak version 1.1.2. Insert 500,000 items. | |
Bitcask vs. LevelDB | |
------------------- | |
Disk space: 3.003 : 1.000 | |
Time taken: 1.000 : 1.600 | |
Bitcask | |
------- | |
times...........: [1201.6725490093231] | |
mean time.......: 1201.67254901 | |
size changes....: [4886613449] | |
mean size change: 4886613449.0 | |
LevelDB | |
------- | |
times...........: [1921.870239019394] | |
mean time.......: 1921.87023902 | |
size changes....: [1627507469] | |
mean size change: 1627507469.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
from subprocess import check_call | |
import functools | |
import shutil | |
import uuid | |
import gc | |
import time | |
import sys | |
import riak | |
data_dir = "/usr/local/Cellar/riak/1.1.2-x86_64/libexec/data" | |
dummy_text = """\ | |
Bacon ipsum dolor sit amet capicola beef beef ribs short loin, pastrami speck | |
kielbasa tongue drumstick. Tongue shank ground round beef, pastrami chuck | |
prosciutto drumstick. Pig hamburger ham shoulder, bacon cow pork chop bresaola | |
beef pork loin brisket jerky beef ribs frankfurter. Ribeye meatball turkey short | |
loin. | |
Speck hamburger kielbasa tenderloin fatback ball tip, chuck boudin prosciutto | |
brisket beef ribs. Swine biltong ball tip capicola filet mignon, andouille | |
pastrami pig spare ribs leberkas. T-bone rump spare ribs, tongue swine sausage | |
meatball pancetta ham hock shank sirloin fatback capicola bacon frankfurter. | |
Ribeye beef strip steak, t-bone pork chop chicken meatball flank jowl hamburger | |
tri-tip. Sausage leberkas beef pastrami, spare ribs frankfurter chicken. Fatback | |
turkey shank strip steak meatball frankfurter andouille. Jowl filet mignon shank | |
prosciutto chuck flank. | |
Swine meatball filet mignon bacon. Jerky pork chop beef, kielbasa shank ribeye | |
jowl cow drumstick pig meatball swine rump biltong. Rump meatball hamburger, | |
shank beef shankle turducken andouille filet mignon venison pork belly pastrami | |
kielbasa. Pancetta jerky turkey, meatball salami venison brisket tongue. Jowl | |
salami chicken, short loin swine beef short ribs sausage pork chop hamburger | |
venison shank. Ham hock pastrami pig, beef brisket short ribs jowl capicola | |
fatback. | |
Corned beef pastrami filet mignon chicken, cow shoulder meatball sausage tongue | |
meatloaf pork brisket. Short ribs capicola bacon ham hock pancetta. Sausage | |
t-bone pastrami prosciutto. Rump turducken kielbasa ribeye meatloaf venison | |
pancetta spare ribs jerky. Boudin leberkas beef ribs, biltong flank prosciutto | |
ball tip bacon cow pastrami venison meatloaf. Salami sausage hamburger, venison | |
ribeye drumstick short ribs spare ribs swine shankle strip steak corned beef ham | |
cow biltong. Turkey bacon chicken, ball tip jerky prosciutto pork chop ground | |
round. | |
Venison pork belly chuck kielbasa filet mignon drumstick. Fatback shoulder | |
prosciutto, chuck ribeye pork chop biltong andouille. Strip steak kielbasa | |
shankle turkey, leberkas capicola ribeye tail jowl shoulder short loin spare | |
ribs drumstick prosciutto. Corned beef tail ground round tenderloin bacon | |
sirloin andouille t-bone pastrami, turducken hamburger meatloaf shoulder. Pork | |
chop shoulder pork belly tenderloin, spare ribs sirloin beef pig beef ribs cow | |
bacon pancetta short loin. Ham ball tip filet mignon, sirloin biltong ribeye | |
jerky meatball tri-tip hamburger venison leberkas. Chicken venison corned beef, | |
pancetta shoulder flank shank andouille beef ribs pig. | |
""" | |
def service(task): | |
assert task in ["start", "stop"] | |
return check_call(["riak", task]) | |
start = functools.partial(service, "start") | |
stop = functools.partial(service, "stop") | |
def get_folder_size(folder): | |
total_size = os.path.getsize(folder) | |
for item in os.listdir(folder): | |
item_path = os.path.join(folder, item) | |
if os.path.isfile(item_path): | |
total_size += os.path.getsize(item_path) | |
elif os.path.isdir(item_path): | |
total_size += get_folder_size(item_path) | |
return total_size | |
def get_backend_dir_size(name): | |
folder = os.path.join(data_dir, name) | |
return get_folder_size(folder) | |
def remove_backend_dir(name): | |
folder = os.path.join(data_dir, name) | |
if os.path.exists(folder): | |
shutil.rmtree(folder) | |
def insert_multi(num): | |
client = riak.RiakClient(port=8087, transport_class=riak.RiakPbcTransport) | |
bucket = client.bucket("test") | |
for i in range(num): | |
dummy = bucket.new(uuid.uuid4().hex, data={"text_s": dummy_text}) | |
dummy.store() | |
def run_batch(backend, insert_count): | |
print "backend:", backend | |
remove_backend_dir(backend) | |
start() | |
time.sleep(10) | |
size_before = get_backend_dir_size(backend) | |
gc.disable() | |
tic = time.time() | |
insert_multi(insert_count) | |
toc = time.time() | |
gc.enable() | |
stop() | |
time.sleep(10) | |
size_after = get_backend_dir_size(backend) | |
return toc - tic, size_after - size_before | |
def main(): | |
backend = sys.argv[1] | |
insert_count = int(sys.argv[2]) | |
batches = int(sys.argv[3]) | |
results = [] | |
for i in range(batches): | |
results.append(run_batch(backend, insert_count)) | |
times = [r[0] for r in results] | |
size_changes = [r[1] for r in results] | |
print "times...........: ", times | |
print "mean time.......: ", float(sum(times)) / batches | |
print "size changes....: ", size_changes | |
print "mean size change: ", float(sum(size_changes)) / batches | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment