provide access to SQLite, with functions and variables for db-specific variants

.gitignore

@@ -10,3 +10,4 @@
 *#*#
 *.#*
 __pycache__
+var

scopes/storage/common.py

@@ -4,19 +4,28 @@
 
 import base64
 from sqlalchemy import create_engine, MetaData, text
-from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy import Integer
+from sqlalchemy.dialects.sqlite import JSON
 import threading
-import zope.sqlalchemy
 
 
-def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
-    return create_engine('%s://%s:%s@%s:%s/%s' % (
-        dbtype, user, pw, host, port, dbname), **kw)
+# predefined db-specific definitions, usable for SQLite;
+# may be overriden by import of ``scopes.storage.db.<dbname>``
 
 def sessionFactory(engine):
-    Session = scoped_session(sessionmaker(bind=engine, twophase=True))
-    zope.sqlalchemy.register(Session)
-    return Session
+    return engine.connect
+
+def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
+    return create_engine('%s:///%s' % (dbtype, dbname), **kw)
+
+def mark_changed(session):
+    pass
+
+def commit(conn):
+    conn.commit()
+
+IdType = Integer
+JsonType = JSON
 
 # put something like this in code before first creating a Storage object
 #engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret')
@@ -57,7 +66,8 @@ class Storage(object):
 
     def dropTable(self, tableName):
         with self.engine.begin() as conn:
-            conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName)))
+            prefix = self.schema and self.schema + '.' or ''
+            conn.execute(text('drop table if exists %s%s' % (prefix, tableName)))
 
     def resetSequence(self, tableName, colName, v):
         sq = ('alter sequence %s.%s_%s_seq restart %i' % 

scopes/storage/db/__init__.py

@@ -0,0 +1 @@
+"""scopes.storage.db"""

scopes/storage/db/postgres.py

@@ -0,0 +1,23 @@
+# scopes.storage.db.postgres
+
+"""Database-related code specific for PostgreSQL."""
+
+from sqlalchemy import create_engine
+from sqlalchemy import BigInteger, JSONB
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import scoped_session, sessionmaker
+import transaction
+from zope.sqlalchemy import register, mark_changed
+
+
+def sessionFactory(engine):
+    Session = scoped_session(sessionmaker(bind=engine, twophase=True))
+    register(Session)
+    return Session
+
+def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
+    return create_engine('%s://%s:%s@%s:%s/%s' % (
+        dbtype, user, pw, host, port, dbname), **kw)
+
+def commit(conn):
+    transaction.commit()

scopes/storage/tracking.py

@@ -9,12 +9,10 @@ data (payload) represented as a dict.
 import base64
 from datetime import datetime
 from sqlalchemy import Table, Column, Index
-from sqlalchemy import BigInteger, DateTime, Text, func
+from sqlalchemy import DateTime, Text, func
 from sqlalchemy import and_
-from sqlalchemy.dialects.postgresql import JSONB
-import transaction
-from zope.sqlalchemy import register, mark_changed
 
+from scopes.storage.common import commit, IdType, JsonType, mark_changed
 from scopes.storage.common import registerContainerClass
 
 
@@ -177,7 +175,7 @@ class Container(object):
 
 def createTable(storage, tableName, headcols, indexes=None):
     metadata = storage.metadata
-    cols = [Column('trackid', BigInteger, primary_key=True)]
+    cols = [Column('trackid', IdType, primary_key=True)]
     idxs = []
     for ix, f in enumerate(headcols):
         cols.append(Column(f.lower(), Text, nullable=False, server_default=''))
@@ -187,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None):
         indexName = 'idx_%s_%d' % (tableName, (ix + 1))
         idxs.append(Index(indexName, *idef))
     idxs.append(Index('idx_%s_ts' % tableName, 'timestamp'))
-    cols.append(Column('data', JSONB, nullable=False, server_default='{}'))
+    cols.append(Column('data', JsonType, nullable=False, server_default='{}'))
     table = Table(tableName, metadata, *(cols+idxs), extend_existing=True)
     metadata.create_all(storage.engine)
     return table

tests/config.py

@@ -7,8 +7,16 @@ server_port = '8999'
 app = zope_app
 
 # storage settings
+
+# PostgreSQL
 dbengine = 'postgresql+psycopg'
 dbname = 'testdb'
 dbuser = 'testuser'
 dbpassword = 'secret'
+dbschema = 'testing'
+
+# SQLite
+dbengine = 'sqlite'
+dbname = 'var/test.db'
+dbschema = None
 

tests/test_storage.py

@@ -2,27 +2,30 @@
 
 """Tests for the 'scopes.storage' package."""
 
+import config
+
 from datetime import datetime
 import transaction
 import unittest
 
 import scopes.storage.common
-from scopes.storage.common import Storage, getEngine, sessionFactory
+from scopes.storage.common import commit, Storage, getEngine, sessionFactory
 from scopes.storage import proxy
 from scopes.storage import folder, tracking
 
-import config
-engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword)
+engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) 
 scopes.storage.common.engine = engine
 scopes.storage.common.Session = sessionFactory(engine)
 
-storage = Storage(schema='testing')
+#storage = Storage(schema='testing')
+#storage = Storage(schema=config.dbschema)
+storage = Storage()
 
 
 class Test(unittest.TestCase):
     "Basic tests for the cco.storage package."
 
-    def testTracking(self):
+    def test_001_tracking(self):
         storage.dropTable('tracks')
         tracks = storage.create(tracking.Container)
 
@@ -68,9 +71,9 @@ class Test(unittest.TestCase):
         self.assertEqual(n, 1)
         self.assertEqual(tracks.get(31), None)
 
-        transaction.commit()
+        commit(storage.session)
 
-    def testFolder(self):
+    def test_002_folder(self):
         storage.dropTable('folders')
         root = folder.Root(storage)
         self.assertEqual(list(root.keys()), [])
@@ -83,6 +86,9 @@ class Test(unittest.TestCase):
         self.assertEqual(ch1.parent, top.rid)
         assert list(top.keys()) == ['child1']
 
+        #transaction.commit()
+        storage.session.commit()
+
 def suite():
     return unittest.TestSuite((
         unittest.TestLoader().loadTestsFromTestCase(Test),