diff --git a/.gitignore b/.gitignore index 4e355e3..f625fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ *#*# *.#* __pycache__ +var diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 6d04ec1..9e6142e 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -4,19 +4,28 @@ import base64 from sqlalchemy import create_engine, MetaData, text -from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy import Integer +from sqlalchemy.dialects.sqlite import JSON import threading -import zope.sqlalchemy -def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): - return create_engine('%s://%s:%s@%s:%s/%s' % ( - dbtype, user, pw, host, port, dbname), **kw) +# predefined db-specific definitions, usable for SQLite; +# may be overriden by import of ``scopes.storage.db.`` def sessionFactory(engine): - Session = scoped_session(sessionmaker(bind=engine, twophase=True)) - zope.sqlalchemy.register(Session) - return Session + return engine.connect + +def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s:///%s' % (dbtype, dbname), **kw) + +def mark_changed(session): + pass + +def commit(conn): + conn.commit() + +IdType = Integer +JsonType = JSON # put something like this in code before first creating a Storage object #engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') @@ -57,7 +66,8 @@ class Storage(object): def dropTable(self, tableName): with self.engine.begin() as conn: - conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName))) + prefix = self.schema and self.schema + '.' or '' + conn.execute(text('drop table if exists %s%s' % (prefix, tableName))) def resetSequence(self, tableName, colName, v): sq = ('alter sequence %s.%s_%s_seq restart %i' % diff --git a/scopes/storage/db/__init__.py b/scopes/storage/db/__init__.py new file mode 100644 index 0000000..467122c --- /dev/null +++ b/scopes/storage/db/__init__.py @@ -0,0 +1 @@ +"""scopes.storage.db""" diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py new file mode 100644 index 0000000..5fd0773 --- /dev/null +++ b/scopes/storage/db/postgres.py @@ -0,0 +1,23 @@ +# scopes.storage.db.postgres + +"""Database-related code specific for PostgreSQL.""" + +from sqlalchemy import create_engine +from sqlalchemy import BigInteger, JSONB +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import scoped_session, sessionmaker +import transaction +from zope.sqlalchemy import register, mark_changed + + +def sessionFactory(engine): + Session = scoped_session(sessionmaker(bind=engine, twophase=True)) + register(Session) + return Session + +def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s://%s:%s@%s:%s/%s' % ( + dbtype, user, pw, host, port, dbname), **kw) + +def commit(conn): + transaction.commit() diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 9a8d8f3..76b7cf5 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -9,12 +9,10 @@ data (payload) represented as a dict. import base64 from datetime import datetime from sqlalchemy import Table, Column, Index -from sqlalchemy import BigInteger, DateTime, Text, func +from sqlalchemy import DateTime, Text, func from sqlalchemy import and_ -from sqlalchemy.dialects.postgresql import JSONB -import transaction -from zope.sqlalchemy import register, mark_changed +from scopes.storage.common import commit, IdType, JsonType, mark_changed from scopes.storage.common import registerContainerClass @@ -177,7 +175,7 @@ class Container(object): def createTable(storage, tableName, headcols, indexes=None): metadata = storage.metadata - cols = [Column('trackid', BigInteger, primary_key=True)] + cols = [Column('trackid', IdType, primary_key=True)] idxs = [] for ix, f in enumerate(headcols): cols.append(Column(f.lower(), Text, nullable=False, server_default='')) @@ -187,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None): indexName = 'idx_%s_%d' % (tableName, (ix + 1)) idxs.append(Index(indexName, *idef)) idxs.append(Index('idx_%s_ts' % tableName, 'timestamp')) - cols.append(Column('data', JSONB, nullable=False, server_default='{}')) + cols.append(Column('data', JsonType, nullable=False, server_default='{}')) table = Table(tableName, metadata, *(cols+idxs), extend_existing=True) metadata.create_all(storage.engine) return table diff --git a/tests/config.py b/tests/config.py index 2fe27db..5ed3d8a 100644 --- a/tests/config.py +++ b/tests/config.py @@ -7,8 +7,16 @@ server_port = '8999' app = zope_app # storage settings + +# PostgreSQL dbengine = 'postgresql+psycopg' dbname = 'testdb' dbuser = 'testuser' dbpassword = 'secret' +dbschema = 'testing' + +# SQLite +dbengine = 'sqlite' +dbname = 'var/test.db' +dbschema = None diff --git a/tests/test_storage.py b/tests/test_storage.py index d61f459..8b2fbb6 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -2,27 +2,30 @@ """Tests for the 'scopes.storage' package.""" +import config + from datetime import datetime import transaction import unittest import scopes.storage.common -from scopes.storage.common import Storage, getEngine, sessionFactory +from scopes.storage.common import commit, Storage, getEngine, sessionFactory from scopes.storage import proxy from scopes.storage import folder, tracking -import config -engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) +engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) scopes.storage.common.engine = engine scopes.storage.common.Session = sessionFactory(engine) -storage = Storage(schema='testing') +#storage = Storage(schema='testing') +#storage = Storage(schema=config.dbschema) +storage = Storage() class Test(unittest.TestCase): "Basic tests for the cco.storage package." - def testTracking(self): + def test_001_tracking(self): storage.dropTable('tracks') tracks = storage.create(tracking.Container) @@ -68,9 +71,9 @@ class Test(unittest.TestCase): self.assertEqual(n, 1) self.assertEqual(tracks.get(31), None) - transaction.commit() + commit(storage.session) - def testFolder(self): + def test_002_folder(self): storage.dropTable('folders') root = folder.Root(storage) self.assertEqual(list(root.keys()), []) @@ -83,6 +86,9 @@ class Test(unittest.TestCase): self.assertEqual(ch1.parent, top.rid) assert list(top.keys()) == ['child1'] + #transaction.commit() + storage.session.commit() + def suite(): return unittest.TestSuite(( unittest.TestLoader().loadTestsFromTestCase(Test),