Use SQLite as a key:value store

Question

As suggested in comments from Key: value store in Python for possibly 100 GB of data, without client/server and in other questions, SQLite could totally be used as a persist

Answer 1

Even if there exists modules that do this (see other answer), I tried to write one simple, self-contained version. Here is a class KeyValueStore (key and value are strings) that works like this:

Demo

from sqlitekeyvaluestore import KeyValueStore

kv = KeyValueStore('test.db')  # uses SQLite

print(len(kv))                 # 0 item
kv['hello1'] = 'you1'
kv['hello2'] = 'you2'
kv['hello3'] = 'you3'
print(kv['hello1'])            # you1
print(len(kv))                 # 3 items

del kv['hello1']
print(len(kv))                 # 2 items remaining

print('hello1' in kv)          # False, it has just been deleted!
print('hello3' in kv)          # True

kv['hello3'] = 'newvalue'      # redefine an already present key/value
print(kv['hello3'])            # newvalue

print(kv.keys())               # ['hello2', 'hello3']
print(kv.values())             # ['you2', 'newvalue']
print(kv.items())              # [('hello2', 'you2'), ('hello3', 'newvalue')]

for k in kv:
    print(k, kv[k])

kv.close()                     # important to commit

---

Code: sqlitekeyvaluestore.py

import sqlite3

class KeyValueStore(dict):
    def __init__(self, filename=None):
        self.conn = sqlite3.connect(filename)
        self.conn.execute("CREATE TABLE IF NOT EXISTS kv (key text unique, value text)")

    def close(self):
        self.conn.commit()
        self.conn.close()

    def __len__(self):
        rows = self.conn.execute('SELECT COUNT(*) FROM kv').fetchone()[0]
        return rows if rows is not None else 0

    def iterkeys(self):
        c = self.conn.cursor()
        for row in self.conn.execute('SELECT key FROM kv'):
            yield row[0]

    def itervalues(self):
        c = self.conn.cursor()
        for row in c.execute('SELECT value FROM kv'):
            yield row[0]

    def iteritems(self):
        c = self.conn.cursor()
        for row in c.execute('SELECT key, value FROM kv'):
            yield row[0], row[1]

    def keys(self):
        return list(self.iterkeys())

    def values(self):
        return list(self.itervalues())

    def items(self):
        return list(self.iteritems())

    def __contains__(self, key):
        return self.conn.execute('SELECT 1 FROM kv WHERE key = ?', (key,)).fetchone() is not None

    def __getitem__(self, key):
        item = self.conn.execute('SELECT value FROM kv WHERE key = ?', (key,)).fetchone()
        if item is None:
            raise KeyError(key)
        return item[0]

    def __setitem__(self, key, value):
        self.conn.execute('REPLACE INTO kv (key, value) VALUES (?,?)', (key, value))

    def __delitem__(self, key):
        if key not in self:
            raise KeyError(key)
        self.conn.execute('DELETE FROM kv WHERE key = ?', (key,))

    def __iter__(self):
        return self.iterkeys()

Answer 2

There is already sqlitedict which appears to meet all your needs.

From the documentation:

>>> from sqlitedict import SqliteDict
>>> mydict = SqliteDict('./my_db.sqlite', autocommit=True)
>>> mydict['some_key'] = any_picklable_object
>>> print mydict['some_key']  # prints the new value
>>> for key, value in mydict.iteritems():
>>>     print key, value
>>> print len(mydict) # etc... all dict functions work
>>> mydict.close()