import py import fcntl _escape_mapping = ( ('_', '_under_'), ('/', '_slash_'), ('\\', '_bslash_'), #('$', '_dollar_'), #('?', '_question_'), #('[', '_lbrack_'), #(']', '_rbrack_'), #('"', '_quot_'), #("'", '_apos_'), ('\t', '_tab_'), ('\n', '_lf_'), ('\r', '_cr_'), ) def escape_key(key, inenc='UTF-8', fsenc='UTF-8'): if inenc != fsenc: key = unicode(key, inenc, 'replace').encode(fsenc, 'replace') for char, replace in _escape_mapping: key = key.replace(char, replace) return key class ezdb(object): """ very simple flat-file db stores objects using some id as filename (may be optimized into a dir structure) """ def __init__(self, dbpath, inenc='UTF-8', fsenc='UTF-8', indexer=None): self.dbpath = py.path.local(dbpath) self.dbpath.ensure(dir=True) self.inenc = inenc self.fsenc = fsenc self.indexer = indexer def __getitem__(self, key): if not isinstance(key, str): raise ValueError('only string keys allowed') fpath = self.dbpath.join(escape_key(key, self.inenc, self.fsenc)) if not fpath.check(file=True): raise KeyError(key) fp = fpath.open() try: fcntl.flock(fp, fcntl.LOCK_EX) try: return self._deserialize(fpath.read()) finally: fcntl.flock(fp, fcntl.LOCK_UN) finally: fp.close() def __setitem__(self, key, value): if not isinstance(value, dict): raise ValueError('can only store dicts') if not isinstance(key, str): raise ValueError('only string keys allowed') fpath = self.dbpath.join(escape_key(key, self.inenc, self.fsenc)) fpath.ensure(file=True) fp = fpath.open() try: fcntl.flock(fp, fcntl.LOCK_EX) try: fpath.write(self._serialize(value)) finally: fcntl.flock(fp, fcntl.LOCK_UN) finally: fp.close() if self.indexer: self.indexer.unindex(key) self.indexer.index(key, value) def __delitem__(self, key): if not isinstance(key, str): raise ValueError('only string keys allowed') fpath = self.dbpath.join(escape_key(key, self.inenc, self.fsenc)) if not fpath.check(file=True): raise KeyError(key) fp = fpath.open() try: fcntl.flock(fp, fcntl.LOCK_EX) try: fpath.remove() finally: fcntl.flock(fp, fcntl.LOCK_UN) finally: fp.close() if self.indexer: self.indexer.unindex(key) def _deserialize(self, data): from decimal import Decimal import datetime return eval(data) def _serialize(self, data): return repr(data) def __len__(self): return len(self.dbpath.listdir()) def __iter__(self): for path in self.dbpath.listdir(): if not path.check(file=True): continue yield path.basename def iteritems(self): for path in self.dbpath.listdir(): if not path.check(file=True): continue yield (path.basename, self._deserialize(path.read()))