Skip to content

Instantly share code, notes, and snippets.

@nyov

nyov/rados-binarytest.py

Created September 20, 2014 04:58
Show Gist options
  • Save nyov/4c5742c15c98fbc7f1c3 to your computer and use it in GitHub Desktop.
Save nyov/4c5742c15c98fbc7f1c3 to your computer and use it in GitHub Desktop.
Download ZIP
python-rados Ioctx.aio_read() not escaping \0
Raw
rados-binarytest.py
#!/usr/bin/python
import rados
# choose whatever you have, lzma, zlib, snappy...
# lzo seems to nicely represent the bug
compressor = __import__('lzo', fromlist=[''])
def _complete(completion, data_read):
print "oncomplete called:"
# print "co.wait_for_safe() is: %s" % completion.wait_for_safe()
# print "co.wait_for_complete() is: %s" % completion.wait_for_complete()
# print "co.get_return_value() is: %s" % completion.get_return_value()
print " * Supposed chunksize read: %s" % completion.get_return_value()
print " * Actual data gotten: %s " % len(data_read)
def main():
print "Creating a context for the 'test' pool"
if not cluster.pool_exists('test'):
raise RuntimeError('No test pool exists')
ioctx = cluster.open_ioctx('test')
print "\nRunning test\n"
file_contents = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
print "Original file size is: %s" % len(file_contents)
print "Compressing file, using '%s'" % compressor.__name__
file_contents = compressor.compress(file_contents)
print "Compressed file size is: %s" % len(file_contents)
obj_name = 'loremipsum'
print "Writing object '%s' to pool 'test'." % obj_name
# sync. write
ioctx.write_full(obj_name, file_contents)
### Synchronous read
print "\n### read()"
print "Reading contents of object '%s'." % obj_name
sync_obj = ioctx.read(obj_name)
print "Decompressing object:"
print "----"
print compressor.decompress(sync_obj)
print "----"
### Async read
print "\n### aio_read()"
(filesize, timestamp) = ioctx.stat(obj_name)
print "Object size is (stat): %s\n" % filesize
chunksize = filesize
chunksize = 8192
chunksize = 128
chunksize = 32
print "Read test using chunksize of %s\n" % chunksize
bytes_read = 0
while bytes_read < filesize:
if (bytes_read + chunksize) > filesize:
chunksize = (filesize - bytes_read)
# async reading
ioctx.aio_read(obj_name, length=chunksize, offset=bytes_read, oncomplete=_complete)
bytes_read += chunksize
print "Done."
ioctx.close()
if __name__ == '__main__':
settings=dict(conffile='/etc/ceph/ceph.conf', conf=dict(keyring='ceph.client.admin.keyring'))
try:
cluster = rados.Rados(**settings)
except TypeError as e:
print 'Argument validation error: ', e
raise e
print "librados version: " + str(cluster.version())
try:
cluster.connect()
except Exception as e:
print "connection error: ", e
raise e
else:
print "Connected to the cluster."
# testing
main()
cluster.shutdown()
Raw
test-output.txt
$ ./rados-binarytest.py
librados version: 0.69.0
Connected to the cluster.
Creating a context for the 'test' pool
Running test
Original file size is: 445
Compressing file, using 'lzo'
Compressed file size is: 443
Writing object 'loremipsum' to pool 'test'.
### read()
Reading contents of object 'loremipsum'.
Decompressing object:
----
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
----
### aio_read()
Object size is (stat): 443
Read test using chunksize of 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 1
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 7
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 0
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 18
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 26
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 32
oncomplete called:
* Supposed chunksize read: 32
* Actual data gotten: 2
oncomplete called:
* Supposed chunksize read: 27
* Actual data gotten: 2
Done.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment