Created
February 23, 2017 07:05
-
-
Save peter-jung/1570e69a93e4fe46b5f736a68399d85c to your computer and use it in GitHub Desktop.
Minimal JSON-Store based on LevelDB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# leveldb json store | |
import leveldb | |
import struct | |
def i2s(i): | |
res = '' | |
while i > 255: | |
res += chr(i % 256) | |
i = i >> 8 | |
res += chr(i) | |
return res | |
def s2i(s): | |
res = 0 | |
for e, c in enumerate(s): | |
res += ord(c) << e * 8 | |
return res | |
def f2s(f): | |
return struct.pack('d', f) | |
def s2f(s): | |
return struct.unpack('d',s)[0] | |
class LJS(object): | |
_NONE = chr(1) | |
_TRUE = chr(2) | |
_FALSE = chr(3) | |
_LIST = chr(4) | |
_DICT = chr(5) | |
_STR = chr(6) | |
_INT = chr(7) | |
_DBL = chr(8) | |
_TDIC = { | |
_NONE : None, | |
_FALSE : False, | |
_TRUE : True, | |
_LIST : [], | |
_DICT : {}, | |
_STR : '', | |
_INT : 0, | |
_DBL : 0.0 | |
} | |
def __init__(self, dbpath, root): | |
self.db = leveldb.LevelDB(dbpath) | |
self.root = root | |
self.cursor = root | |
def check(self, v): | |
vtype = type(v) | |
if vtype in (type(None), bool, unicode, str, int, float): | |
return True | |
elif vtype in (list, tuple): | |
for e in v: | |
if not self.check(e): | |
return False | |
return True | |
elif vtype == dict: | |
for k in v: | |
if not type(k) == str: | |
return False | |
if not self.check(v[k]): | |
return False | |
return True | |
return False | |
def delete(self, K): | |
keys = [k for k, _ in self.db.RangeIter(key_from = K, key_to = K + chr(47) )] | |
batch = leveldb.WriteBatch() | |
for key in keys: | |
batch.Delete(key) | |
self.db.Write(batch, sync = True) | |
def _write(self, K, v): | |
self.delete(K) | |
vtype = type(v) | |
if vtype == type(None): | |
self.db.Put(K, LJS._NONE) | |
elif vtype == bool: | |
self.db.Put(K, LJS._TRUE) if v else self.db.Put(K, LJS._FALSE) | |
elif vtype == unicode: | |
self.db.Put(K, LJS._STR + v.encode('utf-8')) | |
elif vtype == str: | |
self.db.Put(K, LJS._STR + v) | |
elif vtype == int: | |
self.db.Put(K, LJS._INT + i2s(v)) | |
elif vtype == float: | |
self.db.Put(K, LJS._DBL + f2s(v)) | |
elif vtype in (list, tuple): | |
self.db.Put(K, LJS._LIST) | |
for e, x in enumerate(v): | |
nK = K + '.' + '%08d'%e | |
self._write(nK, x) | |
elif vtype == dict: | |
self.db.Put(K, LJS._DICT) | |
for k in v: | |
nK = K + '.' + k | |
self._write(nK, v[k]) | |
def write(self, k, v): | |
assert(self.check(v)) | |
assert(type(k)==str) | |
K = '.'.join([self.cursor, k]) if k else self.cursor | |
self._write(K, v) | |
def _rread(self, K): | |
Kcomps = K.split('.') | |
cd = len(Kcomps) | |
print Kcomps | |
tstack = [5] | |
res = {} | |
collptrstack = [res] | |
for _k, v in self.db.RangeIter(key_from = K, key_to = K + chr(47) ): | |
kcomps = _k.split('.') | |
print kcomps | |
#print res | |
kd = len(kcomps) | |
while kd < cd + len(tstack) - 1 : | |
if chr(tstack.pop()) in (LJS._LIST, LJS._DICT): | |
collptrstack.pop() | |
tbit = v[0] | |
supertype = tstack[-1] # must be collection | |
tstack.append(ord(tbit)) | |
thistype = tstack[-1] | |
#print supertype, thistype, tbit, tbit == LJS._DICT | |
if tbit == LJS._STR: arg = v[1:] | |
elif tbit == LJS._STR: arg = v[1:] | |
elif tbit == LJS._INT: arg = s2i(v[1:]) | |
elif tbit == LJS._DBL: arg = s2f(v[1:]) | |
elif tbit == LJS._LIST: arg = [] | |
elif tbit == LJS._DICT: arg = {} | |
elif tbit == LJS._NONE: arg = None | |
elif tbit == LJS._TRUE: arg = True | |
elif tbit == LJS._FALSE: arg = False | |
#print arg | |
if supertype == ord(LJS._LIST): | |
#print collptrstack, arg | |
collptrstack[-1].append(arg) | |
if thistype in (ord(LJS._LIST), ord(LJS._DICT)): | |
collptrstack.append(collptrstack[-1][len(collptrstack[-1])-1]) | |
elif supertype == ord(LJS._DICT): | |
#print collptrstack, arg | |
collptrstack[-1][kcomps[-1]] = arg | |
if thistype in (ord(LJS._LIST), ord(LJS._DICT)): | |
collptrstack.append(collptrstack[-1][kcomps[-1]]) | |
else: | |
print "ERRRRROR" | |
return res[Kcomps[-1]] | |
def read(self, k = None): | |
assert(type(k)==str) | |
if k == None: | |
K = self.cursor | |
else: | |
K = '.'.join([self.cursor, k]) | |
return self._rread(K) | |
def set_cursor(self, k): | |
self.cursor = '.'.join([self.root, k]) | |
if __name__ == '__main__': | |
store = LJS('test', 'ROOT') | |
smpl = {'none':None, 'int':123, 'true':True, 'false':False, 'float':123.123, 'string':'abc', 'list':[None,True,False,[],{},'2',1,1.1], 'dict':{'a':1,'b':2}, 'emptydic':{}, 'emptylist':[]} | |
store.write('jsom', smpl) | |
store.write('json', smpl) | |
store.write('jsona', smpl) | |
store.write('json2', smpl) | |
store.write('json.false',[1,2,3.333]) | |
store.write('json.falsee',[4,5,6]) | |
store.write('json.falseee',[77777, None]) | |
rsmpl = store.read('json') | |
print smpl | |
print rsmpl | |
print smpl == rsmpl | |
#for k,v in store.db.RangeIter(): | |
# print k | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment