Skip to content

Instantly share code, notes, and snippets.

@addisonElliott

addisonElliott/read_fixed_experiment.py

Created March 30, 2019 05:00
Show Gist options
  • Save addisonElliott/097de1ca1311026e2e116541c9eed0c5 to your computer and use it in GitHub Desktop.
Save addisonElliott/097de1ca1311026e2e116541c9eed0c5 to your computer and use it in GitHub Desktop.
Download ZIP
Analysis of pynrrd compressed data performance
Raw
read_fixed_experiment.py
import nrrd
import time
import numpy as np
print('Reading 1KB data file with 1MB chunk size')
print('---------------------')
start = time.perf_counter()
nrrd.reader._READ_CHUNKSIZE = 2 ** 20
nrrd.read('C:/Users/Addison/Documents/PythonProjects/pynrrd/nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one_nifti.nhdr')
end = time.perf_counter()
print('Time taken: %fs' % (end - start))
print('---------------------')
print('')
print('Reading 1GB data file with 1MB chunk size')
print('---------------------')
start = time.perf_counter()
nrrd.reader._READ_CHUNKSIZE = 2 ** 20
nrrd.read('output.nrrd')
end = time.perf_counter()
print('Time taken: %fs' % (end - start))
print('---------------------')
print('')
print('Reading 1KB data file with 1GB chunk size')
print('---------------------')
start = time.perf_counter()
nrrd.reader._READ_CHUNKSIZE = 2 ** 32
nrrd.read('C:/Users/Addison/Documents/PythonProjects/pynrrd/nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one_nifti.nhdr')
end = time.perf_counter()
print('Time taken: %fs' % (end - start))
print('---------------------')
print('')
print('Reading 1GB data file with 1GB chunk size')
print('---------------------')
start = time.perf_counter()
nrrd.reader._READ_CHUNKSIZE = 2 ** 32
nrrd.read('output.nrrd')
end = time.perf_counter()
print('Time taken: %fs' % (end - start))
print('---------------------')
print('')
# Reading 1KB data file with 1MB chunk size
# ---------------------
# Time taken: 0.000818s
# ---------------------
#
# Reading 1GB data file with 1MB chunk size
# ---------------------
# Time taken: 14.427538s
# ---------------------
#
# Reading 1KB data file with 1GB chunk size
# ---------------------
# Time taken: 0.000777s
# ---------------------
#
# Reading 1GB data file with 1GB chunk size
# ---------------------
# Time taken: 10.555254s
# ---------------------
Raw
read_normal_experiment.py
import nrrd
import time
import numpy as np
print('Reading 1KB data file with 1MB chunk size')
print('---------------------')
nrrd.reader._READ_CHUNKSIZE = 2 ** 20
nrrd.read('C:/Users/Addison/Documents/PythonProjects/pynrrd/nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one_nifti.nhdr')
print('---------------------')
print('')
print('Reading 1GB data file with 1MB chunk size')
print('---------------------')
nrrd.reader._READ_CHUNKSIZE = 2 ** 20
nrrd.read('output.nrrd')
print('---------------------')
print('')
print('Reading 1KB data file with 1GB chunk size')
print('---------------------')
nrrd.reader._READ_CHUNKSIZE = 2 ** 32
nrrd.read('C:/Users/Addison/Documents/PythonProjects/pynrrd/nrrd/tests/data/BallBinary30x30x30_byteskip_minus_one_nifti.nhdr')
print('---------------------')
print('')
print('Reading 1GB data file with 1GB chunk size')
print('---------------------')
nrrd.reader._READ_CHUNKSIZE = 2 ** 32
nrrd.read('output.nrrd')
print('---------------------')
print('')
# Reading 1KB data file with 1MB chunk size
# ---------------------
# Read took 0.000626s, decompress took 0.000281s, total of 2 iterations
# Total time = 907us
# ---------------------
#
# Reading 1GB data file with 1MB chunk size
# ---------------------
# Read took 0.654045s, decompress took 11.884516s, total of 1061 iterations
# Total time = 12.538s
# ---------------------
#
# Reading 1KB data file with 1GB chunk size
# ---------------------
# Read took 1.398114s, decompress took 0.000137s, total of 2 iterations
# Total time = 1.398s
# ---------------------
#
# Reading 1GB data file with 1GB chunk size
# ---------------------
# Read took 2.109819s, decompress took 8.609425s, total of 2 iterations
# Total time = 10.72s
# ---------------------
Raw
write_experiment.py
import nrrd
import time
import numpy as np
x = np.random.randn(144000000).reshape((288, 500, 1000))
start = time.perf_counter()
nrrd.writer._WRITE_CHUNKSIZE = 2 ** 20
nrrd.write('output.nrrd', x, {'encoding': 'gzip'}, compression_level=1)
end = time.perf_counter()
print('Took with 1MB chunk size & compression_level=1: %fs' % (end - start))
start = time.perf_counter()
nrrd.writer._WRITE_CHUNKSIZE = 2 ** 32
nrrd.write('output2.nrrd', x, {'encoding': 'gzip'}, compression_level=1)
end = time.perf_counter()
print('Took with 1GB chunk size & compression_level=1: %fs' % (end - start))
start = time.perf_counter()
nrrd.writer._WRITE_CHUNKSIZE = 2 ** 20
nrrd.write('output3.nrrd', x, {'encoding': 'gzip'}, compression_level=6)
end = time.perf_counter()
print('Took with 1MB chunk size & compression_level=6: %fs' % (end - start))
start = time.perf_counter()
nrrd.writer._WRITE_CHUNKSIZE = 2 ** 32
nrrd.write('output4.nrrd', x, {'encoding': 'gzip'}, compression_level=6)
end = time.perf_counter()
print('Took with 1GB chunk size & compression_level=6: %fs' % (end - start))
# Output: Took with 1MB chunk size & compression_level=1: 61.991413s
# Output: Took with 1GB chunk size & compression_level=1: 65.233409s
# Output: Took with 1MB chunk size & compression_level=6: 65.669527s
# Output: Took with 1GB chunk size & compression_level=6: 71.311288s
# Output file size is 1GB
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment