initial import

.gitignore

@@ -0,0 +1,12 @@
+*.pyc
+*.pyo
+*.egg-info
+*.project
+*.swp
+*.pydevproject
+*.sublime-project
+*.sublime-workspace
+*.ropeproject
+*#*#
+*.#*
+__pycache__

CHANGES.txt

@@ -0,0 +1,7 @@
+Changelog
+=========
+
+3.0.1 (unreleased)
+------------------
+
+- Package created from cco.storage (2024-02-14)

CONTRIBUTORS.txt

@@ -0,0 +1 @@
+- Helmut Merz, Original Author

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT) 
+
+Copyright (C) 2024 cyberconcepts.org team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,17 @@
+# py-scopes
+
+The 'py-scopes' package is a re-implementation of a similar module in Go,
+called 'go-scopes'. It processes application data focussing on changes
+instead of objects or state.
+
+The first sub-package (scopes.storage) deals with storing application data 
+(as records or tracks, messages, or more specific kinds of entities)
+in a SQL database, using some header columns for indexing and direct access and 
+a jsonb column for the real data (payload).
+
+Status: implementation started
+
+Project website: https://www.cyberconcepts.org
+
+License: MIT, see LICENSE file
+

docs/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT) 
+
+Copyright (C) 2017 cyberconcepts.org team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "py-scopes"
+version = "3.0.1"
+description = "Implementation of the strange 'scopes' paradigma in Python"
+readme = "README.md"
+license = {text = "MIT"}
+keywords = ["scopes"]
+authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}]
+
+dependencies = [
+	"transaction",
+	"psycopg[binary]",
+	"SQLAlchemy",
+	"zope.sqlalchemy",
+]
+
+[project.optional-dependencies]
+test = ["pytest"]

requirements.txt

@@ -0,0 +1,6 @@
+setuptools
+psycopg[binary]
+SQLAlchemy
+transaction
+zope.sqlalchemy
+

scopes/__init__.py

@@ -0,0 +1 @@
+"""package scopes"""

scopes/storage/README.rst

@@ -0,0 +1,37 @@
+========================================================
+SQL-based Storage for Records (Tracks) and Other Objects
+========================================================
+
+Test Prerequisite: PostgreSQL database ccotest (user ccotest with password cco).
+
+  >>> from cco.storage.common import getEngine, sessionFactory
+  >>> from cco.storage.tracking import record
+
+  >>> record.engine = getEngine('postgresql+psycopg', 'ccotest', 'ccotest', 'cco')
+  >>> record.Session = sessionFactory(record.engine)
+
+
+Tracking Storage
+================
+
+  >>> storage = record.Storage(doCommit=True)
+
+  >>> tr01 = record.Track('t01', 'john')
+  >>> tr01.head
+  {'taskId': 't01', 'userName': 'john'}
+
+  >>> storage.getTable()
+  Table(...)
+
+  >>> trackId = storage.save(tr01)
+  >>> trackId > 0
+  True
+
+  >>> tr01a = storage.get(trackId)
+  >>> tr01a.head
+
+ Fin
+ ===
+
+  >>> storage.conn.close()
+

scopes/storage/__init__.py

@@ -0,0 +1 @@
+"""package scopes.storage"""

scopes/storage/common.py

@@ -0,0 +1,79 @@
+# scopes.storage.common
+
+"""Common utility stuff for the scopes.storage package."""
+
+from sqlalchemy import create_engine, MetaData, text
+from sqlalchemy.orm import scoped_session, sessionmaker
+import threading
+import zope.sqlalchemy
+
+
+def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw):
+    return create_engine('%s://%s:%s@%s:%s/%s' % (
+        dbtype, user, pw, host, port, dbname), **kw)
+
+def sessionFactory(engine):
+    Session = scoped_session(sessionmaker(bind=engine, twophase=True))
+    zope.sqlalchemy.register(Session)
+    return Session
+
+# put something like this in code before first creating a Storage object
+#engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret')
+#scopes.storage.common.engine = engine
+#scopes.storage.common.Session = sessionFactory(engine)
+
+
+class Storage(object):
+
+    def __init__(self, schema=None):
+        self.engine = engine
+        self.session = Session()
+        self.schema = schema
+        self.metadata = MetaData(schema=schema)
+        self.containers = {}
+
+    def create(self, cls):
+        container = cls(self)
+        self.add(container)
+        return container
+
+    def add(self, container):
+        self.containers[container.itemFactory.prefix] = container
+
+    def getItem(self, uid):
+        prefix, id = uid.split('-')
+        id = int(id)
+        container = self.containers.get(prefix)
+        if container is None:
+            container = self.create(registry[prefix])
+        return container.get(id)
+
+    def getExistingTable(self, tableName):
+        metadata = self.metadata
+        schema = self.schema
+        metadata.reflect(self.engine)
+        return metadata.tables.get((schema and schema + '.' or '') + tableName)
+
+    def dropTable(self, tableName):
+        with self.engine.begin() as conn:
+            conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName)))
+
+    def resetSequence(self, tableName, colName, v):
+        sq = ('alter sequence %s.%s_%s_seq restart %i' % 
+                (self.schema, tableName, colName, v))
+        with self.engine.begin() as conn:
+            conn.execute(text(sq))
+
+
+# store information about container implementations, identified by a uid prefix.
+
+registry = {}
+
+def registerContainerClass(cls):
+    # TODO: error on duplicate key
+    registry[cls.itemFactory.prefix] = cls
+    cls.headCols = cols = tuple(f.lower() for f in cls.itemFactory.headFields)
+    if cls.indexes is None:
+        cls.indexes = [cols[i:] for i in range(len(cols))]
+    return cls
+

scopes/storage/proxy.py

@@ -0,0 +1,50 @@
+# scopes.storage.proxy
+
+"""Core classes and helper functions for creating proxy and adapter objects
+in order to store attribute values in a SQL database.
+
+This is currently in concept and exploration state.
+"""
+
+import transaction
+
+_not_found = object()
+
+
+def loadData(obj):
+    print ('getData ***', obj.context.__name__, obj.context.__parent__.__name__)
+    return dict(dummy='dummy')
+
+
+def storeData(obj, data):
+    print ('storeData ***', obj.context.__name__, obj.context.__parent__.__name__, data)
+
+
+class AdapterBase(object):
+
+    _old_data = None
+    _cont = None
+    _id = None
+
+    def __init__(self, context):
+        super(AdapterBase, self).__init__(context)
+        object.__setattr__(self, '_new_data', {})
+
+    def __getattr__(self, attr):
+        value = self._new_data.get(attr, _not_found)
+        if value is _not_found:
+            if self._old_data is None:
+                object.__setattr__(self, '_old_data', loadData(self))
+            value = self._old_data.get(attr, _not_found)
+        if value is _not_found:
+            return super(AdapterBase, self).__getattr__(attr)
+        return value
+
+    def __setattr__(self, attr, value):
+        super(AdapterBase, self).__setattr__(attr, value)
+        if attr.startswith('__') or attr in self._adapterAttributes:
+            return
+        if not self._new_data:
+            tr = transaction.manager.get()
+            tr.addBeforeCommitHook(storeData, [self, self._new_data], {})
+        self._new_data[attr] = value

scopes/storage/tracking.py

@@ -0,0 +1,178 @@
+# scopes.storage.tracking
+
+"""SQL-based storage for simple tracks (records). 
+
+A track consists of a head (index data, metadata)  with a fixed set of fields and 
+data (payload) represented as a dict.
+"""
+
+from datetime import datetime
+from sqlalchemy import Table, Column, Index
+from sqlalchemy import BigInteger, DateTime, Text, func
+from sqlalchemy import and_
+from sqlalchemy.dialects.postgresql import JSONB
+import transaction
+from zope.sqlalchemy import register, mark_changed
+
+from scopes.storage.common import registerContainerClass
+
+
+class Track(object):
+    
+    headFields = ['taskId', 'userName']
+    prefix = 'rec'
+
+    def __init__(self, *keys, data=None, timeStamp=None, trackId=None, container=None):
+        self.head = {}
+        for ix, k in enumerate(keys):
+            self.head[self.headFields[ix]] = k
+        for k in self.headFields:
+            if self.head.get(k) is None:
+                self.heaad[k] = ''
+            setattr(self, k, self.head[k])
+        self.data = data or {}
+        self.timeStamp = timeStamp
+        self.trackId = trackId
+        self.container = container
+
+    def update(self, data, overwrite=False):
+        if data is None:
+            return
+        if overwrite:
+            self.data = data
+        else:
+            self.data.update(data)
+
+    @property
+    def uid(self):
+        if self.trackId is None:
+            return None
+        return '%s-%d' % (self.prefix, self.trackId)
+
+
+@registerContainerClass
+class Container(object):
+
+    itemFactory = Track
+    tableName = 'tracks'
+    insertOnChange = True # always insert new track when data are changed
+    indexes = None  # default, will be overwritten by registerContainerClass()
+    #indexes = [('username',), ('taskid', 'username')] # or put explicitly in class
+
+    table = None
+
+    def __init__(self, storage):
+        self.storage = storage
+        self.session = storage.session
+        self.table = self.getTable()
+
+    def get(self, trackId):
+        stmt = self.table.select().where(self.table.c.trackid == trackId)
+        return self.makeTrack(self.session.execute(stmt).first())
+
+    def query(self, **crit):
+        stmt = self.table.select().where(
+                and_(*self.setupWhere(crit))).order_by(self.table.c.trackid)
+        for r in self.session.execute(stmt):
+            yield self.makeTrack(r)
+
+    def queryLast(self, **crit):
+        stmt = (self.table.select().where(and_(*self.setupWhere(crit))).
+                order_by(self.table.c.trackid.desc()).limit(1))
+        return self.makeTrack(self.session.execute(stmt).first())
+
+    def save(self, track):
+        crit = dict((hf, track.head[hf]) for hf in track.headFields)
+        found = self.queryLast(**crit)
+        if found is None:
+            return self.insert(track)
+        if self.insertOnChange and found.data != track.data:
+            return self.insert(track)
+        if found.data != track.data or found.timeStamp != track.timeStamp:
+            found.update(track.data)
+            found.timeStamp = track.timeStamp
+            self.update(found)
+        return found.trackId
+
+    def insert(self, track, withTrackId=False):
+        t = self.table
+        values = self.setupValues(track, withTrackId)
+        stmt = t.insert().values(**values).returning(t.c.trackid)
+        trackId = self.session.execute(stmt).first()[0]
+        mark_changed(self.session)
+        return trackId
+
+    def update(self, track):
+        t = self.table
+        values = self.setupValues(track)
+        if track.timeStamp is None:
+            values['timestamp'] = datetime.now()
+        stmt = t.update().values(**values).where(t.c.trackid == track.trackId)
+        n = self.session.execute(stmt).rowcount
+        if n > 0:
+            mark_changed(self.session)
+        return n
+
+    def upsert(self, track):
+        """Try to update the record identified by the trackId given with ``track``. 
+        If not found insert new record without generating a new trackId.
+        Use this method for migration and other bulk insert/update tasks.
+        Don't forget to update the trackid sequence afterwards: 
+        ``select setval('<schema>.tracks_trackid_seq', <max>);``"""
+        if track.trackId is not None:
+            if self.update(track) > 0:
+                return track.trackId
+        return self.insert(track, withTrackId=True)
+
+    def remove(self, trackId):
+        stmt = self.table.delete().where(self.table.c.trackid == trackId)
+        n = self.session.execute(stmt).rowcount
+        if n > 0:
+            mark_changed(self.session)
+        return n
+
+    def makeTrack(self, r):
+        if r is None:
+            return None
+        return self.itemFactory(
+                *r[1:-2], trackId=r[0],timeStamp=r[-2], data=r[-1], container=self)
+
+    def setupWhere(self, crit):
+        return [self.table.c[k.lower()] == v for k, v in crit.items()]
+
+    def setupValues(self, track, withTrackId=False):
+        values = {}
+        hf = self.itemFactory.headFields
+        for i, c in enumerate(self.headCols):
+            values[c] = track.head[hf[i]]
+        values['data'] = track.data
+        if track.timeStamp is not None:
+            values['timestamp'] = track.timeStamp
+        if withTrackId and track.trackId is not None:
+            values['trackid'] = track.trackId
+        return values
+
+    def getTable(self):
+        #table = self.storage.getExistingTable(self.tableName)
+        #if table is None:
+        return createTable(self.storage, self.tableName, self.headCols, 
+                           indexes=self.indexes)
+
+
+def createTable(storage, tableName, headcols, indexes=None):
+    metadata = storage.metadata
+    cols = [Column('trackid', BigInteger, primary_key=True)]
+    idxs = []
+    for ix, f in enumerate(headcols):
+        cols.append(Column(f.lower(), Text, nullable=False, server_default=''))
+    cols.append(Column('timestamp', DateTime(timezone=True), 
+                       nullable=False, server_default=func.now()))
+    for ix, idef in enumerate(indexes):
+        indexName = 'idx_%s_%d' % (tableName, (ix + 1))
+        idxs.append(Index(indexName, *idef))
+    idxs.append(Index('idx_%s_ts' % tableName, 'timestamp'))
+    cols.append(Column('data', JSONB, nullable=False, server_default='{}'))
+    table = Table(tableName, metadata, *(cols+idxs), extend_existing=True)
+    metadata.create_all(storage.engine)
+    return table
+

setup.py

@@ -0,0 +1,4 @@
+from setuptools import setup
+
+setup()
+

tests/test_storage.py

@@ -0,0 +1,79 @@
+#! /usr/bin/python
+
+"""Tests for the 'scopes.storage' package."""
+
+from datetime import datetime
+import transaction
+import unittest
+
+import scopes.storage.common
+from scopes.storage.common import Storage, getEngine, sessionFactory
+from scopes.storage import proxy
+from scopes.storage import tracking
+
+engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret')
+scopes.storage.common.engine = engine
+scopes.storage.common.Session = sessionFactory(engine)
+
+storage = Storage(schema='testing')
+
+
+class Test(unittest.TestCase):
+    "Basic tests for the cco.storage package."
+
+    def testBasicStuff(self):
+        storage.dropTable('tracks')
+        tracks = storage.create(tracking.Container)
+
+        tr01 = tracking.Track('t01', 'john')
+        tr01.update(dict(activity='testing'))
+        self.assertEqual(tr01.head, {'taskId': 't01', 'userName': 'john'})
+        self.assertEqual(tr01.taskId, 't01')
+        self.assertEqual(tr01.userName, 'john')
+
+        self.assertTrue(tracks.getTable() is not None)
+
+        trid01 = tracks.save(tr01)
+        self.assertTrue(trid01 > 0)
+
+        tr01a = tracks.get(trid01)
+        self.assertEqual(tr01a.head, tr01.head)
+        self.assertEqual(tr01a.trackId, trid01)
+        self.assertEqual(tr01a.data.get('activity'), 'testing')
+
+        tr01a.update(dict(text='Set up unit tests.'))
+        tr01a.timeStamp = None
+        self.assertTrue(tracks.save(tr01a) > 0)
+
+        tr01b = tracks.queryLast(taskId='t01')
+        self.assertEqual(tr01b.head, tr01.head)
+        self.assertNotEqual(tr01b.trackId, trid01)
+        self.assertEqual(tr01b.data.get('activity'), 'testing')
+
+        tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30),
+                            data=dict(activity='concept'))
+        trid02 = tracks.upsert(tr02)
+        self.assertEqual(trid02, 31)
+        self.assertEqual(tr02.uid, 'rec-31')
+        tr02.trackId = trid01
+        trid021 = tracks.upsert(tr02)
+        self.assertEqual(trid021, trid01)
+        self.assertEqual(tr02.uid, 'rec-' + str(trid01))
+
+        tr03 = storage.getItem('rec-31')
+        self.assertEqual(tr03.trackId, 31)
+
+        n = tracks.remove(31)
+        self.assertEqual(n, 1)
+        self.assertEqual(tracks.get(31), None)
+
+        transaction.commit()
+
+
+def suite():
+    return unittest.TestSuite((
+        unittest.TestLoader().loadTestsFromTestCase(Test),
+    ))
+
+if __name__ == '__main__':
+    unittest.main(defaultTest='suite')