diff --git a/.gitignore b/.gitignore index 4e355e3..f625fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ *#*# *.#* __pycache__ +var diff --git a/scopes/organize/__init__.py b/scopes/organize/__init__.py new file mode 100644 index 0000000..829e828 --- /dev/null +++ b/scopes/organize/__init__.py @@ -0,0 +1 @@ +"""package scopes.organize""" diff --git a/scopes/organize/task.py b/scopes/organize/task.py new file mode 100644 index 0000000..c0a98cc --- /dev/null +++ b/scopes/organize/task.py @@ -0,0 +1,13 @@ +# scopes.organize.task + +"""Task (and corresponding container) implementation.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.concept import Concept + + +class Task(Concept): + + headFields = ['name'] + prefix = 'tsk' + diff --git a/scopes/server/__init__.py b/scopes/server/__init__.py new file mode 100644 index 0000000..cf54a0c --- /dev/null +++ b/scopes/server/__init__.py @@ -0,0 +1 @@ +"""package scopes.server""" diff --git a/scopes/server/app.py b/scopes/server/app.py new file mode 100644 index 0000000..fda9130 --- /dev/null +++ b/scopes/server/app.py @@ -0,0 +1,44 @@ +# scopes.server.app + +from zope.publisher.base import DefaultPublication +from zope.publisher.browser import BrowserRequest +from zope.publisher.publish import publish +from zope.traversing.publicationtraverse import PublicationTraverser + + +def demo_app(environ, start_response): + print(f'*** environ {environ}.') + status = '200 OK' + headers = [("Content-type", "text/plain; charset=utf-8")] + start_response(status, headers) + return ['Hello World'.encode()] + + +def zope_app(environ, start_response): + request = BrowserRequest(environ['wsgi.input'], environ) + request.setPublication(DefaultPublication(AppRoot())) + request = publish(request, False) + response = request.response + start_response(response.getStatusString(), response.getHeaders()) + return response.consumeBodyIter() + + +class AppRoot: + """Zope Demo AppRoot""" + + def __call__(self): + """calling AppRoot""" + return 'At root' + + def __getitem__(self, key): + def child(): + """get child""" + print(f'--- getitem {key}') + return 'getitem' + return child + + def hello(self): + """method hello""" + return 'Hello AppRoot' + + diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 89f6673..6b4f5d3 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -4,35 +4,61 @@ import base64 from sqlalchemy import create_engine, MetaData, text -from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy import Integer +from sqlalchemy.dialects.sqlite import JSON import threading -import zope.sqlalchemy -def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): - return create_engine('%s://%s:%s@%s:%s/%s' % ( - dbtype, user, pw, host, port, dbname), **kw) +class StorageFactory(object): -def sessionFactory(engine): - Session = scoped_session(sessionmaker(bind=engine, twophase=True)) - zope.sqlalchemy.register(Session) - return Session + def sessionFactory(self): + return self.engine.connect -# put something like this in code before first creating a Storage object -#engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') -#scopes.storage.common.engine = engine -#scopes.storage.common.Session = sessionFactory(engine) + @staticmethod + def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s:///%s' % (dbtype, dbname), **kw) + @staticmethod + def mark_changed(session): + pass + + @staticmethod + def commit(conn): + conn.commit() + + IdType = Integer + JsonType = JSON + + def __init__(self, config): + self.engine = self.getEngine(config.dbengine, config.dbname, + config.dbuser, config.dbpassword) + self.Session = self.sessionFactory() + + def __call__(self, schema=None): + return Storage(self, schema=schema) + + +# you may put something like this in your code: +#factory = StorageFactory(config) +# and then call at appropriate places: +#storage = scopes.storage.common.factory(schema=...) class Storage(object): - def __init__(self, schema=None): - self.engine = engine - self.session = Session() + def __init__(self, db, schema=None): + self.db = db + self.engine = db.engine + self.session = db.Session() self.schema = schema self.metadata = MetaData(schema=schema) self.containers = {} + def commit(self): + self.db.commit(self.session) + + def mark_changed(self): + self.db.mark_changed(self.session) + def create(self, cls): container = cls(self) self.add(container) @@ -57,7 +83,8 @@ class Storage(object): def dropTable(self, tableName): with self.engine.begin() as conn: - conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName))) + prefix = self.schema and self.schema + '.' or '' + conn.execute(text('drop table if exists %s%s' % (prefix, tableName))) def resetSequence(self, tableName, colName, v): sq = ('alter sequence %s.%s_%s_seq restart %i' % @@ -71,8 +98,10 @@ class Storage(object): registry = {} def registerContainerClass(cls): - # TODO: error on duplicate key - registry[cls.itemFactory.prefix] = cls + prefix = cls.itemFactory.prefix + if prefix in registry: + raise ValueError("prefix '%s' already registered!" % prefix) + registry[prefix] = cls cls.headCols = cols = tuple(f.lower() for f in cls.itemFactory.headFields) if cls.indexes is None: cls.indexes = [cols[i:] for i in range(len(cols))] diff --git a/scopes/storage/concept.py b/scopes/storage/concept.py new file mode 100644 index 0000000..95ede52 --- /dev/null +++ b/scopes/storage/concept.py @@ -0,0 +1,12 @@ +# scopes.storage.concept + +"""Abstract base classes for concept map application classes.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Concept(Track): + + headFields = ['name'] + diff --git a/scopes/storage/db/__init__.py b/scopes/storage/db/__init__.py new file mode 100644 index 0000000..467122c --- /dev/null +++ b/scopes/storage/db/__init__.py @@ -0,0 +1 @@ +"""scopes.storage.db""" diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py new file mode 100644 index 0000000..ccc835a --- /dev/null +++ b/scopes/storage/db/postgres.py @@ -0,0 +1,37 @@ +# scopes.storage.db.postgres + +"""Database-related code specific for PostgreSQL.""" + +from sqlalchemy import create_engine +from sqlalchemy import BigInteger +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import scoped_session, sessionmaker +import transaction +from zope.sqlalchemy import register, mark_changed + +from scopes.storage.common import StorageFactory + + +class StorageFactory(StorageFactory): + + def sessionFactory(self): + Session = scoped_session(sessionmaker(bind=self.engine, twophase=True)) + register(Session) + return Session + + @staticmethod + def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s://%s:%s@%s:%s/%s' % ( + dbtype, user, pw, host, port, dbname), **kw) + + @staticmethod + def mark_changed(session): + return mark_changed(session) + + @staticmethod + def commit(conn): + transaction.commit() + + IdType = BigInteger + JsonType = JSONB + diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py new file mode 100644 index 0000000..9ac89be --- /dev/null +++ b/scopes/storage/folder.py @@ -0,0 +1,51 @@ +# scopes.storage.folder + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Folder(Track): + + headFields = ['parent', 'name', 'ref'] + prefix = 'fldr' + + def keys(self): + for f in self.container.query(parent=self.rid): + yield f.name + + def get(self, key, default=None): + value = self.container.queryLast(parent=self.rid, name=key) + if value is None: + return default + return value + + def __getitem__(self, key): + value = self.container.queryLast(parent=self.rid, name=key) + if value is None: + raise KeyError(key) + return value + + def __setitem__(self, key, value): + value.set('parent', self.rid) + value.set('name', key) + self.container.save(value) + + +class Root(Folder): + """A dummy (virtual) root folder for creating real folders + using the Folder API.""" + + def __init__(self, storage): + cont = storage.create(Folders) + super(Root, self).__init__(container=cont) + + uid = '' + + +@registerContainerClass +class Folders(Container): + + itemFactory = Folder + indexes = [('parent', 'name'), ('ref',)] + tableName = 'folders' + insertOnChange = False diff --git a/scopes/storage/relation.py b/scopes/storage/relation.py new file mode 100644 index 0000000..2d2bd5a --- /dev/null +++ b/scopes/storage/relation.py @@ -0,0 +1,13 @@ +# scopes.storage.relation + +"""An SQL-based relationship engine using RDF-like triples.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Triple(Track): + + headFields = ['first', 'second', 'pred'] + prefix = 'rel' + diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 96ce3f9..9939dda 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -9,11 +9,8 @@ data (payload) represented as a dict. import base64 from datetime import datetime from sqlalchemy import Table, Column, Index -from sqlalchemy import BigInteger, DateTime, Text, func +from sqlalchemy import DateTime, Text, func from sqlalchemy import and_ -from sqlalchemy.dialects.postgresql import JSONB -import transaction -from zope.sqlalchemy import register, mark_changed from scopes.storage.common import registerContainerClass @@ -36,6 +33,15 @@ class Track(object): self.trackId = kw.get('trackId') self.container = kw.get('container') + def set(self, attr, value): + if attr in self.headFields: + if value is None: + value = '' + self.head[attr] = value + setattr(self, attr, value) + else: + raise AttributeError(attr) + def update(self, data, overwrite=False): if data is None: return @@ -50,6 +56,12 @@ class Track(object): return None return '%s-%d' % (self.prefix, self.trackId) + @property + def rid(self): + if self.trackId is None: + return '' + return str(self.trackId) + @registerContainerClass class Container(object): @@ -64,6 +76,7 @@ class Container(object): def __init__(self, storage): self.storage = storage + self.db = storage.db self.session = storage.session self.table = self.getTable() @@ -100,7 +113,7 @@ class Container(object): values = self.setupValues(track, withTrackId) stmt = t.insert().values(**values).returning(t.c.trackid) trackId = self.session.execute(stmt).first()[0] - mark_changed(self.session) + self.storage.mark_changed() return trackId def update(self, track): @@ -111,7 +124,7 @@ class Container(object): stmt = t.update().values(**values).where(t.c.trackid == track.trackId) n = self.session.execute(stmt).rowcount if n > 0: - mark_changed(self.session) + self.storage.mark_changed() return n def upsert(self, track): @@ -129,7 +142,7 @@ class Container(object): stmt = self.table.delete().where(self.table.c.trackid == trackId) n = self.session.execute(stmt).rowcount if n > 0: - mark_changed(self.session) + self.storage.mark_changed() return n def makeTrack(self, r): @@ -162,7 +175,7 @@ class Container(object): def createTable(storage, tableName, headcols, indexes=None): metadata = storage.metadata - cols = [Column('trackid', BigInteger, primary_key=True)] + cols = [Column('trackid', storage.db.IdType, primary_key=True)] idxs = [] for ix, f in enumerate(headcols): cols.append(Column(f.lower(), Text, nullable=False, server_default='')) @@ -172,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None): indexName = 'idx_%s_%d' % (tableName, (ix + 1)) idxs.append(Index(indexName, *idef)) idxs.append(Index('idx_%s_ts' % tableName, 'timestamp')) - cols.append(Column('data', JSONB, nullable=False, server_default='{}')) + cols.append(Column('data', storage.db.JsonType, nullable=False, server_default='{}')) table = Table(tableName, metadata, *(cols+idxs), extend_existing=True) metadata.create_all(storage.engine) return table diff --git a/scopes/tests.py b/scopes/tests.py index 89ee6f4..88029cb 100644 --- a/scopes/tests.py +++ b/scopes/tests.py @@ -1,27 +1,28 @@ -#! /usr/bin/python - -"""Tests for the 'scopes.storage' package.""" +"""The real test implementations""" from datetime import datetime -import transaction import unittest -import scopes.storage.common -from scopes.storage.common import Storage, getEngine, sessionFactory -from scopes.storage import proxy -from scopes.storage import tracking +from scopes.storage import folder, tracking -engine = getEngine('postgresql', 'ccotest', 'ccotest', 'cco') -scopes.storage.common.engine = engine -scopes.storage.common.Session = sessionFactory(engine) +#import config +class Config(object): pass -storage = Storage(schema='testing') +config = Config() +config.dbengine = 'postgresql' +config.dbname = 'ccotest' +config.dbuser = 'ccotest' +config.dbpassword = 'cco' + +# PostgreSQL-specific settings +from scopes.storage.db.postgres import StorageFactory +factory = StorageFactory(config) +storage = factory(schema='testing') class Test(unittest.TestCase): - "Basic tests for the cco.storage package." - def testBasicStuff(self): + def test_001_tracking(self): storage.dropTable('tracks') tracks = storage.create(tracking.Container) @@ -67,7 +68,22 @@ class Test(unittest.TestCase): self.assertEqual(n, 1) self.assertEqual(tracks.get(31), None) - transaction.commit() + storage.commit() + + def test_002_folder(self): + storage.dropTable('folders') + root = folder.Root(storage) + self.assertEqual(list(root.keys()), []) + root['top'] = folder.Folder() + self.assertEqual(list(root.keys()), ['top']) + top = root['top'] + top['child1'] = folder.Folder(data=dict(title='First Child')) + self.assertEqual(list(top.keys()), ['child1']) + ch1 = top['child1'] + self.assertEqual(ch1.parent, top.rid) + self.assertEqual(list(top.keys()), ['child1']) + + storage.commit() def suite():