merge branch master, fix tests for using zope testrunner, with postgres only

.gitignore

@@ -10,3 +10,4 @@
 *#*#
 *.#*
 __pycache__
+var

scopes/organize/__init__.py

@@ -0,0 +1 @@
+"""package scopes.organize"""

scopes/organize/task.py

@@ -0,0 +1,13 @@
+# scopes.organize.task
+
+"""Task (and corresponding container) implementation."""
+
+from scopes.storage.common import registerContainerClass
+from scopes.storage.concept import Concept
+
+
+class Task(Concept):
+
+    headFields = ['name']
+    prefix = 'tsk' 
+

scopes/server/__init__.py

@@ -0,0 +1 @@
+"""package scopes.server"""

scopes/server/app.py

@@ -0,0 +1,44 @@
+# scopes.server.app
+
+from zope.publisher.base import DefaultPublication
+from zope.publisher.browser import BrowserRequest
+from zope.publisher.publish import publish
+from zope.traversing.publicationtraverse import PublicationTraverser
+
+
+def demo_app(environ, start_response):
+    print(f'*** environ {environ}.')
+    status = '200 OK'
+    headers = [("Content-type", "text/plain; charset=utf-8")]
+    start_response(status, headers)
+    return ['Hello World'.encode()]
+
+
+def zope_app(environ, start_response):
+    request = BrowserRequest(environ['wsgi.input'], environ)
+    request.setPublication(DefaultPublication(AppRoot()))
+    request = publish(request, False)
+    response = request.response
+    start_response(response.getStatusString(), response.getHeaders())
+    return response.consumeBodyIter()
+
+
+class AppRoot:
+    """Zope Demo AppRoot"""
+
+    def __call__(self):
+        """calling AppRoot"""
+        return 'At root'
+
+    def __getitem__(self, key):
+        def child():
+            """get child"""
+            print(f'--- getitem {key}')
+            return 'getitem'
+        return child
+
+    def hello(self): 
+        """method hello"""
+        return 'Hello AppRoot'
+
+

scopes/storage/common.py

@@ -4,35 +4,61 @@
 
 import base64
 from sqlalchemy import create_engine, MetaData, text
-from sqlalchemy.orm import scoped_session, sessionmaker
+from sqlalchemy import Integer
+from sqlalchemy.dialects.sqlite import JSON
 import threading
-import zope.sqlalchemy
 
 
-def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
+class StorageFactory(object):
-    return create_engine('%s://%s:%s@%s:%s/%s' % (
-        dbtype, user, pw, host, port, dbname), **kw)
 
-def sessionFactory(engine):
+    def sessionFactory(self):
-    Session = scoped_session(sessionmaker(bind=engine, twophase=True))
+         return self.engine.connect
-    zope.sqlalchemy.register(Session)
-    return Session
 
-# put something like this in code before first creating a Storage object
+    @staticmethod
-#engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret')
+    def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
-#scopes.storage.common.engine = engine
+        return create_engine('%s:///%s' % (dbtype, dbname), **kw)
-#scopes.storage.common.Session = sessionFactory(engine)
 
+    @staticmethod
+    def mark_changed(session):
+        pass
+
+    @staticmethod
+    def commit(conn):
+        conn.commit()
+
+    IdType = Integer
+    JsonType = JSON
+
+    def __init__(self, config):
+        self.engine = self.getEngine(config.dbengine, config.dbname, 
+                                     config.dbuser, config.dbpassword) 
+        self.Session = self.sessionFactory()
+
+    def __call__(self, schema=None):
+        return Storage(self, schema=schema)
+
+
+# you may put something like this in your code:
+#factory = StorageFactory(config)
+# and then call at appropriate places:
+#storage = scopes.storage.common.factory(schema=...)
 
 class Storage(object):
 
-    def __init__(self, schema=None):
+    def __init__(self, db, schema=None):
-        self.engine = engine
+        self.db = db
-        self.session = Session()
+        self.engine = db.engine
+        self.session = db.Session()
         self.schema = schema
         self.metadata = MetaData(schema=schema)
         self.containers = {}
 
+    def commit(self):
+        self.db.commit(self.session)
+
+    def mark_changed(self):
+        self.db.mark_changed(self.session)
+
     def create(self, cls):
         container = cls(self)
         self.add(container)
@@ -57,7 +83,8 @@ class Storage(object):
 
     def dropTable(self, tableName):
         with self.engine.begin() as conn:
-            conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName)))
+            prefix = self.schema and self.schema + '.' or ''
+            conn.execute(text('drop table if exists %s%s' % (prefix, tableName)))
 
     def resetSequence(self, tableName, colName, v):
         sq = ('alter sequence %s.%s_%s_seq restart %i' % 
@@ -71,8 +98,10 @@ class Storage(object):
 registry = {}
 
 def registerContainerClass(cls):
-    # TODO: error on duplicate key
+    prefix = cls.itemFactory.prefix
-    registry[cls.itemFactory.prefix] = cls
+    if prefix in registry:
+        raise ValueError("prefix '%s' already registered!" % prefix)
+    registry[prefix] = cls
     cls.headCols = cols = tuple(f.lower() for f in cls.itemFactory.headFields)
     if cls.indexes is None:
         cls.indexes = [cols[i:] for i in range(len(cols))]

scopes/storage/concept.py

@@ -0,0 +1,12 @@
+# scopes.storage.concept
+
+"""Abstract base classes for concept map application classes."""
+
+from scopes.storage.common import registerContainerClass
+from scopes.storage.tracking import Container, Track
+
+
+class Concept(Track):
+
+    headFields = ['name']
+ 

scopes/storage/db/__init__.py

@@ -0,0 +1 @@
+"""scopes.storage.db"""

scopes/storage/db/postgres.py

@@ -0,0 +1,37 @@
+# scopes.storage.db.postgres
+
+"""Database-related code specific for PostgreSQL."""
+
+from sqlalchemy import create_engine
+from sqlalchemy import BigInteger
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import scoped_session, sessionmaker
+import transaction
+from zope.sqlalchemy import register, mark_changed
+
+from scopes.storage.common import StorageFactory
+
+
+class StorageFactory(StorageFactory):
+
+    def sessionFactory(self):
+        Session = scoped_session(sessionmaker(bind=self.engine, twophase=True))
+        register(Session)
+        return Session
+
+    @staticmethod
+    def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
+        return create_engine('%s://%s:%s@%s:%s/%s' % (
+            dbtype, user, pw, host, port, dbname), **kw)
+
+    @staticmethod
+    def mark_changed(session):
+        return mark_changed(session)
+
+    @staticmethod
+    def commit(conn):
+        transaction.commit()
+
+    IdType = BigInteger
+    JsonType = JSONB
+

scopes/storage/folder.py

@@ -0,0 +1,51 @@
+# scopes.storage.folder
+
+from scopes.storage.common import registerContainerClass
+from scopes.storage.tracking import Container, Track
+
+
+class Folder(Track):
+
+    headFields = ['parent', 'name', 'ref']
+    prefix = 'fldr'
+
+    def keys(self):
+        for f in self.container.query(parent=self.rid):
+            yield f.name
+
+    def get(self, key, default=None):
+        value = self.container.queryLast(parent=self.rid, name=key)
+        if value is None:
+            return default
+        return value
+
+    def __getitem__(self, key):
+        value = self.container.queryLast(parent=self.rid, name=key)
+        if value is None:
+            raise KeyError(key)
+        return value
+
+    def __setitem__(self, key, value):
+        value.set('parent', self.rid)
+        value.set('name', key)
+        self.container.save(value)
+
+
+class Root(Folder):
+    """A dummy (virtual) root folder for creating real folders
+       using the Folder API."""
+
+    def __init__(self, storage):
+        cont = storage.create(Folders)
+        super(Root, self).__init__(container=cont)
+
+    uid = ''
+
+
+@registerContainerClass
+class Folders(Container):
+
+    itemFactory = Folder
+    indexes = [('parent', 'name'), ('ref',)]
+    tableName = 'folders'
+    insertOnChange = False

scopes/storage/relation.py

@@ -0,0 +1,13 @@
+# scopes.storage.relation
+
+"""An SQL-based relationship engine using RDF-like triples."""
+
+from scopes.storage.common import registerContainerClass
+from scopes.storage.tracking import Container, Track
+
+
+class Triple(Track):
+
+    headFields = ['first', 'second', 'pred']
+    prefix = 'rel'
+ 

scopes/storage/tracking.py

@@ -9,11 +9,8 @@ data (payload) represented as a dict.
 import base64
 from datetime import datetime
 from sqlalchemy import Table, Column, Index
-from sqlalchemy import BigInteger, DateTime, Text, func
+from sqlalchemy import DateTime, Text, func
 from sqlalchemy import and_
-from sqlalchemy.dialects.postgresql import JSONB
-import transaction
-from zope.sqlalchemy import register, mark_changed
 
 from scopes.storage.common import registerContainerClass
 
@@ -36,6 +33,15 @@ class Track(object):
         self.trackId = kw.get('trackId')
         self.container = kw.get('container')
 
+    def set(self, attr, value):
+        if attr in self.headFields:
+            if value is None:
+                value = ''
+            self.head[attr] = value
+            setattr(self, attr, value)
+        else:
+            raise AttributeError(attr)
+
     def update(self, data, overwrite=False):
         if data is None:
             return
@@ -50,6 +56,12 @@ class Track(object):
             return None
         return '%s-%d' % (self.prefix, self.trackId)
 
+    @property
+    def rid(self):
+        if self.trackId is None:
+            return ''
+        return str(self.trackId)
+
 
 @registerContainerClass
 class Container(object):
@@ -64,6 +76,7 @@ class Container(object):
 
     def __init__(self, storage):
         self.storage = storage
+        self.db = storage.db
         self.session = storage.session
         self.table = self.getTable()
 
@@ -100,7 +113,7 @@ class Container(object):
         values = self.setupValues(track, withTrackId)
         stmt = t.insert().values(**values).returning(t.c.trackid)
         trackId = self.session.execute(stmt).first()[0]
-        mark_changed(self.session)
+        self.storage.mark_changed()
         return trackId
 
     def update(self, track):
@@ -111,7 +124,7 @@ class Container(object):
         stmt = t.update().values(**values).where(t.c.trackid == track.trackId)
         n = self.session.execute(stmt).rowcount
         if n > 0:
-            mark_changed(self.session)
+            self.storage.mark_changed()
         return n
 
     def upsert(self, track):
@@ -129,7 +142,7 @@ class Container(object):
         stmt = self.table.delete().where(self.table.c.trackid == trackId)
         n = self.session.execute(stmt).rowcount
         if n > 0:
-            mark_changed(self.session)
+            self.storage.mark_changed()
         return n
 
     def makeTrack(self, r):
@@ -162,7 +175,7 @@ class Container(object):
 
 def createTable(storage, tableName, headcols, indexes=None):
     metadata = storage.metadata
-    cols = [Column('trackid', BigInteger, primary_key=True)]
+    cols = [Column('trackid', storage.db.IdType, primary_key=True)]
     idxs = []
     for ix, f in enumerate(headcols):
         cols.append(Column(f.lower(), Text, nullable=False, server_default=''))
@@ -172,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None):
         indexName = 'idx_%s_%d' % (tableName, (ix + 1))
         idxs.append(Index(indexName, *idef))
     idxs.append(Index('idx_%s_ts' % tableName, 'timestamp'))
-    cols.append(Column('data', JSONB, nullable=False, server_default='{}'))
+    cols.append(Column('data', storage.db.JsonType, nullable=False, server_default='{}'))
     table = Table(tableName, metadata, *(cols+idxs), extend_existing=True)
     metadata.create_all(storage.engine)
     return table

scopes/tests.py

@@ -1,27 +1,28 @@
-#! /usr/bin/python
+"""The real test implementations"""
-
-"""Tests for the 'scopes.storage' package."""
 
 from datetime import datetime
-import transaction
 import unittest
 
-import scopes.storage.common
+from scopes.storage import folder, tracking
-from scopes.storage.common import Storage, getEngine, sessionFactory
-from scopes.storage import proxy
-from scopes.storage import tracking
 
-engine = getEngine('postgresql', 'ccotest', 'ccotest', 'cco')
+#import config
-scopes.storage.common.engine = engine
+class Config(object): pass
-scopes.storage.common.Session = sessionFactory(engine)
 
-storage = Storage(schema='testing')
+config = Config()
+config.dbengine = 'postgresql'
+config.dbname = 'ccotest'
+config.dbuser = 'ccotest'
+config.dbpassword = 'cco'
+
+# PostgreSQL-specific settings
+from scopes.storage.db.postgres import StorageFactory 
+factory = StorageFactory(config)
+storage = factory(schema='testing')
 
 
 class Test(unittest.TestCase):
-    "Basic tests for the cco.storage package."
 
-    def testBasicStuff(self):
+    def test_001_tracking(self):
         storage.dropTable('tracks')
         tracks = storage.create(tracking.Container)
 
@@ -67,7 +68,22 @@ class Test(unittest.TestCase):
         self.assertEqual(n, 1)
         self.assertEqual(tracks.get(31), None)
 
-        transaction.commit()
+        storage.commit()
+
+    def test_002_folder(self):
+        storage.dropTable('folders')
+        root = folder.Root(storage)
+        self.assertEqual(list(root.keys()), [])
+        root['top'] = folder.Folder()
+        self.assertEqual(list(root.keys()), ['top'])
+        top = root['top']
+        top['child1'] = folder.Folder(data=dict(title='First Child'))
+        self.assertEqual(list(top.keys()), ['child1'])
+        ch1 = top['child1']
+        self.assertEqual(ch1.parent, top.rid)
+        self.assertEqual(list(top.keys()), ['child1'])
+
+        storage.commit()
 
 
 def suite():