Louis Vézina
5 years ago
parent
34f34130c6
commit
32a467cec5
@ -1,85 +0,0 @@
|
|||||||
"""Generic interface to all dbm clones.
|
|
||||||
|
|
||||||
Instead of
|
|
||||||
|
|
||||||
import dbm
|
|
||||||
d = dbm.open(file, 'w', 0666)
|
|
||||||
|
|
||||||
use
|
|
||||||
|
|
||||||
import anydbm
|
|
||||||
d = anydbm.open(file, 'w')
|
|
||||||
|
|
||||||
The returned object is a dbhash, gdbm, dbm or dumbdbm object,
|
|
||||||
dependent on the type of database being opened (determined by whichdb
|
|
||||||
module) in the case of an existing dbm. If the dbm does not exist and
|
|
||||||
the create or new flag ('c' or 'n') was specified, the dbm type will
|
|
||||||
be determined by the availability of the modules (tested in the above
|
|
||||||
order).
|
|
||||||
|
|
||||||
It has the following interface (key and data are strings):
|
|
||||||
|
|
||||||
d[key] = data # store data at key (may override data at
|
|
||||||
# existing key)
|
|
||||||
data = d[key] # retrieve data at key (raise KeyError if no
|
|
||||||
# such key)
|
|
||||||
del d[key] # delete data stored at key (raises KeyError
|
|
||||||
# if no such key)
|
|
||||||
flag = key in d # true if the key exists
|
|
||||||
list = d.keys() # return a list of all existing keys (slow!)
|
|
||||||
|
|
||||||
Future versions may change the order in which implementations are
|
|
||||||
tested for existence, and add interfaces to other dbm-like
|
|
||||||
implementations.
|
|
||||||
"""
|
|
||||||
|
|
||||||
class error(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
_names = ['dbhash', 'gdbm', 'dbm', 'dumbdbm']
|
|
||||||
_errors = [error]
|
|
||||||
_defaultmod = None
|
|
||||||
|
|
||||||
for _name in _names:
|
|
||||||
try:
|
|
||||||
_mod = __import__(_name)
|
|
||||||
except ImportError:
|
|
||||||
continue
|
|
||||||
if not _defaultmod:
|
|
||||||
_defaultmod = _mod
|
|
||||||
_errors.append(_mod.error)
|
|
||||||
|
|
||||||
if not _defaultmod:
|
|
||||||
raise ImportError, "no dbm clone found; tried %s" % _names
|
|
||||||
|
|
||||||
error = tuple(_errors)
|
|
||||||
|
|
||||||
def open(file, flag='r', mode=0666):
|
|
||||||
"""Open or create database at path given by *file*.
|
|
||||||
|
|
||||||
Optional argument *flag* can be 'r' (default) for read-only access, 'w'
|
|
||||||
for read-write access of an existing database, 'c' for read-write access
|
|
||||||
to a new or existing database, and 'n' for read-write access to a new
|
|
||||||
database.
|
|
||||||
|
|
||||||
Note: 'r' and 'w' fail if the database doesn't exist; 'c' creates it
|
|
||||||
only if it doesn't exist; and 'n' always creates a new database.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# guess the type of an existing database
|
|
||||||
from whichdb import whichdb
|
|
||||||
result=whichdb(file)
|
|
||||||
if result is None:
|
|
||||||
# db doesn't exist
|
|
||||||
if 'c' in flag or 'n' in flag:
|
|
||||||
# file doesn't exist and the new
|
|
||||||
# flag was used so use default type
|
|
||||||
mod = _defaultmod
|
|
||||||
else:
|
|
||||||
raise error, "need 'c' or 'n' flag to open new db"
|
|
||||||
elif result == "":
|
|
||||||
# db type cannot be determined
|
|
||||||
raise error, "db type could not be determined"
|
|
||||||
else:
|
|
||||||
mod = __import__(result)
|
|
||||||
return mod.open(file, flag, mode)
|
|
||||||
@ -1,18 +0,0 @@
|
|||||||
"""Provide a (g)dbm-compatible interface to bsddb.hashopen."""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
import warnings
|
|
||||||
warnings.warnpy3k("in 3.x, the dbhash module has been removed", stacklevel=2)
|
|
||||||
try:
|
|
||||||
import bsddb
|
|
||||||
except ImportError:
|
|
||||||
# prevent a second import of this module from spuriously succeeding
|
|
||||||
del sys.modules[__name__]
|
|
||||||
raise
|
|
||||||
|
|
||||||
__all__ = ["error","open"]
|
|
||||||
|
|
||||||
error = bsddb.error # Exported for anydbm
|
|
||||||
|
|
||||||
def open(file, flag = 'r', mode=0666):
|
|
||||||
return bsddb.hashopen(file, flag, mode)
|
|
||||||
@ -1,249 +0,0 @@
|
|||||||
"""A dumb and slow but simple dbm clone.
|
|
||||||
|
|
||||||
For database spam, spam.dir contains the index (a text file),
|
|
||||||
spam.bak *may* contain a backup of the index (also a text file),
|
|
||||||
while spam.dat contains the data (a binary file).
|
|
||||||
|
|
||||||
XXX TO DO:
|
|
||||||
|
|
||||||
- seems to contain a bug when updating...
|
|
||||||
|
|
||||||
- reclaim free space (currently, space once occupied by deleted or expanded
|
|
||||||
items is never reused)
|
|
||||||
|
|
||||||
- support concurrent access (currently, if two processes take turns making
|
|
||||||
updates, they can mess up the index)
|
|
||||||
|
|
||||||
- support efficient access to large databases (currently, the whole index
|
|
||||||
is read when the database is opened, and some updates rewrite the whole index)
|
|
||||||
|
|
||||||
- support opening for read-only (flag = 'm')
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
import ast as _ast
|
|
||||||
import os as _os
|
|
||||||
import __builtin__
|
|
||||||
import UserDict
|
|
||||||
|
|
||||||
_open = __builtin__.open
|
|
||||||
|
|
||||||
_BLOCKSIZE = 512
|
|
||||||
|
|
||||||
error = IOError # For anydbm
|
|
||||||
|
|
||||||
class _Database(UserDict.DictMixin):
|
|
||||||
|
|
||||||
# The on-disk directory and data files can remain in mutually
|
|
||||||
# inconsistent states for an arbitrarily long time (see comments
|
|
||||||
# at the end of __setitem__). This is only repaired when _commit()
|
|
||||||
# gets called. One place _commit() gets called is from __del__(),
|
|
||||||
# and if that occurs at program shutdown time, module globals may
|
|
||||||
# already have gotten rebound to None. Since it's crucial that
|
|
||||||
# _commit() finish successfully, we can't ignore shutdown races
|
|
||||||
# here, and _commit() must not reference any globals.
|
|
||||||
_os = _os # for _commit()
|
|
||||||
_open = _open # for _commit()
|
|
||||||
|
|
||||||
def __init__(self, filebasename, mode):
|
|
||||||
self._mode = mode
|
|
||||||
|
|
||||||
# The directory file is a text file. Each line looks like
|
|
||||||
# "%r, (%d, %d)\n" % (key, pos, siz)
|
|
||||||
# where key is the string key, pos is the offset into the dat
|
|
||||||
# file of the associated value's first byte, and siz is the number
|
|
||||||
# of bytes in the associated value.
|
|
||||||
self._dirfile = filebasename + _os.extsep + 'dir'
|
|
||||||
|
|
||||||
# The data file is a binary file pointed into by the directory
|
|
||||||
# file, and holds the values associated with keys. Each value
|
|
||||||
# begins at a _BLOCKSIZE-aligned byte offset, and is a raw
|
|
||||||
# binary 8-bit string value.
|
|
||||||
self._datfile = filebasename + _os.extsep + 'dat'
|
|
||||||
self._bakfile = filebasename + _os.extsep + 'bak'
|
|
||||||
|
|
||||||
# The index is an in-memory dict, mirroring the directory file.
|
|
||||||
self._index = None # maps keys to (pos, siz) pairs
|
|
||||||
|
|
||||||
# Mod by Jack: create data file if needed
|
|
||||||
try:
|
|
||||||
f = _open(self._datfile, 'r')
|
|
||||||
except IOError:
|
|
||||||
with _open(self._datfile, 'w') as f:
|
|
||||||
self._chmod(self._datfile)
|
|
||||||
else:
|
|
||||||
f.close()
|
|
||||||
self._update()
|
|
||||||
|
|
||||||
# Read directory file into the in-memory index dict.
|
|
||||||
def _update(self):
|
|
||||||
self._index = {}
|
|
||||||
try:
|
|
||||||
f = _open(self._dirfile)
|
|
||||||
except IOError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
with f:
|
|
||||||
for line in f:
|
|
||||||
line = line.rstrip()
|
|
||||||
key, pos_and_siz_pair = _ast.literal_eval(line)
|
|
||||||
self._index[key] = pos_and_siz_pair
|
|
||||||
|
|
||||||
# Write the index dict to the directory file. The original directory
|
|
||||||
# file (if any) is renamed with a .bak extension first. If a .bak
|
|
||||||
# file currently exists, it's deleted.
|
|
||||||
def _commit(self):
|
|
||||||
# CAUTION: It's vital that _commit() succeed, and _commit() can
|
|
||||||
# be called from __del__(). Therefore we must never reference a
|
|
||||||
# global in this routine.
|
|
||||||
if self._index is None:
|
|
||||||
return # nothing to do
|
|
||||||
|
|
||||||
try:
|
|
||||||
self._os.unlink(self._bakfile)
|
|
||||||
except self._os.error:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
self._os.rename(self._dirfile, self._bakfile)
|
|
||||||
except self._os.error:
|
|
||||||
pass
|
|
||||||
|
|
||||||
with self._open(self._dirfile, 'w') as f:
|
|
||||||
self._chmod(self._dirfile)
|
|
||||||
for key, pos_and_siz_pair in self._index.iteritems():
|
|
||||||
f.write("%r, %r\n" % (key, pos_and_siz_pair))
|
|
||||||
|
|
||||||
sync = _commit
|
|
||||||
|
|
||||||
def __getitem__(self, key):
|
|
||||||
pos, siz = self._index[key] # may raise KeyError
|
|
||||||
with _open(self._datfile, 'rb') as f:
|
|
||||||
f.seek(pos)
|
|
||||||
dat = f.read(siz)
|
|
||||||
return dat
|
|
||||||
|
|
||||||
# Append val to the data file, starting at a _BLOCKSIZE-aligned
|
|
||||||
# offset. The data file is first padded with NUL bytes (if needed)
|
|
||||||
# to get to an aligned offset. Return pair
|
|
||||||
# (starting offset of val, len(val))
|
|
||||||
def _addval(self, val):
|
|
||||||
with _open(self._datfile, 'rb+') as f:
|
|
||||||
f.seek(0, 2)
|
|
||||||
pos = int(f.tell())
|
|
||||||
npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
|
|
||||||
f.write('\0'*(npos-pos))
|
|
||||||
pos = npos
|
|
||||||
f.write(val)
|
|
||||||
return (pos, len(val))
|
|
||||||
|
|
||||||
# Write val to the data file, starting at offset pos. The caller
|
|
||||||
# is responsible for ensuring that there's enough room starting at
|
|
||||||
# pos to hold val, without overwriting some other value. Return
|
|
||||||
# pair (pos, len(val)).
|
|
||||||
def _setval(self, pos, val):
|
|
||||||
with _open(self._datfile, 'rb+') as f:
|
|
||||||
f.seek(pos)
|
|
||||||
f.write(val)
|
|
||||||
return (pos, len(val))
|
|
||||||
|
|
||||||
# key is a new key whose associated value starts in the data file
|
|
||||||
# at offset pos and with length siz. Add an index record to
|
|
||||||
# the in-memory index dict, and append one to the directory file.
|
|
||||||
def _addkey(self, key, pos_and_siz_pair):
|
|
||||||
self._index[key] = pos_and_siz_pair
|
|
||||||
with _open(self._dirfile, 'a') as f:
|
|
||||||
self._chmod(self._dirfile)
|
|
||||||
f.write("%r, %r\n" % (key, pos_and_siz_pair))
|
|
||||||
|
|
||||||
def __setitem__(self, key, val):
|
|
||||||
if not type(key) == type('') == type(val):
|
|
||||||
raise TypeError, "keys and values must be strings"
|
|
||||||
if key not in self._index:
|
|
||||||
self._addkey(key, self._addval(val))
|
|
||||||
else:
|
|
||||||
# See whether the new value is small enough to fit in the
|
|
||||||
# (padded) space currently occupied by the old value.
|
|
||||||
pos, siz = self._index[key]
|
|
||||||
oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
|
|
||||||
newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
|
|
||||||
if newblocks <= oldblocks:
|
|
||||||
self._index[key] = self._setval(pos, val)
|
|
||||||
else:
|
|
||||||
# The new value doesn't fit in the (padded) space used
|
|
||||||
# by the old value. The blocks used by the old value are
|
|
||||||
# forever lost.
|
|
||||||
self._index[key] = self._addval(val)
|
|
||||||
|
|
||||||
# Note that _index may be out of synch with the directory
|
|
||||||
# file now: _setval() and _addval() don't update the directory
|
|
||||||
# file. This also means that the on-disk directory and data
|
|
||||||
# files are in a mutually inconsistent state, and they'll
|
|
||||||
# remain that way until _commit() is called. Note that this
|
|
||||||
# is a disaster (for the database) if the program crashes
|
|
||||||
# (so that _commit() never gets called).
|
|
||||||
|
|
||||||
def __delitem__(self, key):
|
|
||||||
# The blocks used by the associated value are lost.
|
|
||||||
del self._index[key]
|
|
||||||
# XXX It's unclear why we do a _commit() here (the code always
|
|
||||||
# XXX has, so I'm not changing it). _setitem__ doesn't try to
|
|
||||||
# XXX keep the directory file in synch. Why should we? Or
|
|
||||||
# XXX why shouldn't __setitem__?
|
|
||||||
self._commit()
|
|
||||||
|
|
||||||
def keys(self):
|
|
||||||
return self._index.keys()
|
|
||||||
|
|
||||||
def has_key(self, key):
|
|
||||||
return key in self._index
|
|
||||||
|
|
||||||
def __contains__(self, key):
|
|
||||||
return key in self._index
|
|
||||||
|
|
||||||
def iterkeys(self):
|
|
||||||
return self._index.iterkeys()
|
|
||||||
__iter__ = iterkeys
|
|
||||||
|
|
||||||
def __len__(self):
|
|
||||||
return len(self._index)
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
try:
|
|
||||||
self._commit()
|
|
||||||
finally:
|
|
||||||
self._index = self._datfile = self._dirfile = self._bakfile = None
|
|
||||||
|
|
||||||
__del__ = close
|
|
||||||
|
|
||||||
def _chmod (self, file):
|
|
||||||
if hasattr(self._os, 'chmod'):
|
|
||||||
self._os.chmod(file, self._mode)
|
|
||||||
|
|
||||||
|
|
||||||
def open(file, flag=None, mode=0666):
|
|
||||||
"""Open the database file, filename, and return corresponding object.
|
|
||||||
|
|
||||||
The flag argument, used to control how the database is opened in the
|
|
||||||
other DBM implementations, is ignored in the dumbdbm module; the
|
|
||||||
database is always opened for update, and will be created if it does
|
|
||||||
not exist.
|
|
||||||
|
|
||||||
The optional mode argument is the UNIX mode of the file, used only when
|
|
||||||
the database has to be created. It defaults to octal code 0666 (and
|
|
||||||
will be modified by the prevailing umask).
|
|
||||||
|
|
||||||
"""
|
|
||||||
# flag argument is currently ignored
|
|
||||||
|
|
||||||
# Modify mode depending on the umask
|
|
||||||
try:
|
|
||||||
um = _os.umask(0)
|
|
||||||
_os.umask(um)
|
|
||||||
except AttributeError:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
# Turn off any bits that are set in the umask
|
|
||||||
mode = mode & (~um)
|
|
||||||
|
|
||||||
return _Database(file, mode)
|
|
||||||
Loading…
Reference in new issue