commit 02a56bf94d39f2eb9ff53b93d41d077ecfebb787 Author: Helmut Merz Date: Wed Feb 14 18:15:48 2024 +0100 initial import diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e355e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +*.pyc +*.pyo +*.egg-info +*.project +*.swp +*.pydevproject +*.sublime-project +*.sublime-workspace +*.ropeproject +*#*# +*.#* +__pycache__ diff --git a/CHANGES.txt b/CHANGES.txt new file mode 100644 index 0000000..366e742 --- /dev/null +++ b/CHANGES.txt @@ -0,0 +1,7 @@ +Changelog +========= + +3.0.1 (unreleased) +------------------ + +- Package created from cco.storage (2024-02-14) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt new file mode 100644 index 0000000..149473f --- /dev/null +++ b/CONTRIBUTORS.txt @@ -0,0 +1 @@ +- Helmut Merz, Original Author diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..71fa02e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (C) 2024 cyberconcepts.org team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f4c243 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# py-scopes + +The 'py-scopes' package is a re-implementation of a similar module in Go, +called 'go-scopes'. It processes application data focussing on changes +instead of objects or state. + +The first sub-package (scopes.storage) deals with storing application data +(as records or tracks, messages, or more specific kinds of entities) +in a SQL database, using some header columns for indexing and direct access and +a jsonb column for the real data (payload). + +Status: implementation started + +Project website: https://www.cyberconcepts.org + +License: MIT, see LICENSE file + diff --git a/docs/LICENSE.txt b/docs/LICENSE.txt new file mode 100644 index 0000000..e3ff287 --- /dev/null +++ b/docs/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (C) 2017 cyberconcepts.org team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..226e1b5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "py-scopes" +version = "3.0.1" +description = "Implementation of the strange 'scopes' paradigma in Python" +readme = "README.md" +license = {text = "MIT"} +keywords = ["scopes"] +authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}] + +dependencies = [ + "transaction", + "psycopg[binary]", + "SQLAlchemy", + "zope.sqlalchemy", +] + +[project.optional-dependencies] +test = ["pytest"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..61e3eb0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +setuptools +psycopg[binary] +SQLAlchemy +transaction +zope.sqlalchemy + diff --git a/scopes/__init__.py b/scopes/__init__.py new file mode 100644 index 0000000..c9b856f --- /dev/null +++ b/scopes/__init__.py @@ -0,0 +1 @@ +"""package scopes""" diff --git a/scopes/storage/README.rst b/scopes/storage/README.rst new file mode 100644 index 0000000..9cb82fb --- /dev/null +++ b/scopes/storage/README.rst @@ -0,0 +1,37 @@ +======================================================== +SQL-based Storage for Records (Tracks) and Other Objects +======================================================== + +Test Prerequisite: PostgreSQL database ccotest (user ccotest with password cco). + + >>> from cco.storage.common import getEngine, sessionFactory + >>> from cco.storage.tracking import record + + >>> record.engine = getEngine('postgresql+psycopg', 'ccotest', 'ccotest', 'cco') + >>> record.Session = sessionFactory(record.engine) + + +Tracking Storage +================ + + >>> storage = record.Storage(doCommit=True) + + >>> tr01 = record.Track('t01', 'john') + >>> tr01.head + {'taskId': 't01', 'userName': 'john'} + + >>> storage.getTable() + Table(...) + + >>> trackId = storage.save(tr01) + >>> trackId > 0 + True + + >>> tr01a = storage.get(trackId) + >>> tr01a.head + + Fin + === + + >>> storage.conn.close() + diff --git a/scopes/storage/__init__.py b/scopes/storage/__init__.py new file mode 100644 index 0000000..00738cf --- /dev/null +++ b/scopes/storage/__init__.py @@ -0,0 +1 @@ +"""package scopes.storage""" diff --git a/scopes/storage/common.py b/scopes/storage/common.py new file mode 100644 index 0000000..969cb7c --- /dev/null +++ b/scopes/storage/common.py @@ -0,0 +1,79 @@ +# scopes.storage.common + +"""Common utility stuff for the scopes.storage package.""" + +from sqlalchemy import create_engine, MetaData, text +from sqlalchemy.orm import scoped_session, sessionmaker +import threading +import zope.sqlalchemy + + +def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s://%s:%s@%s:%s/%s' % ( + dbtype, user, pw, host, port, dbname), **kw) + +def sessionFactory(engine): + Session = scoped_session(sessionmaker(bind=engine, twophase=True)) + zope.sqlalchemy.register(Session) + return Session + +# put something like this in code before first creating a Storage object +#engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') +#scopes.storage.common.engine = engine +#scopes.storage.common.Session = sessionFactory(engine) + + +class Storage(object): + + def __init__(self, schema=None): + self.engine = engine + self.session = Session() + self.schema = schema + self.metadata = MetaData(schema=schema) + self.containers = {} + + def create(self, cls): + container = cls(self) + self.add(container) + return container + + def add(self, container): + self.containers[container.itemFactory.prefix] = container + + def getItem(self, uid): + prefix, id = uid.split('-') + id = int(id) + container = self.containers.get(prefix) + if container is None: + container = self.create(registry[prefix]) + return container.get(id) + + def getExistingTable(self, tableName): + metadata = self.metadata + schema = self.schema + metadata.reflect(self.engine) + return metadata.tables.get((schema and schema + '.' or '') + tableName) + + def dropTable(self, tableName): + with self.engine.begin() as conn: + conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName))) + + def resetSequence(self, tableName, colName, v): + sq = ('alter sequence %s.%s_%s_seq restart %i' % + (self.schema, tableName, colName, v)) + with self.engine.begin() as conn: + conn.execute(text(sq)) + + +# store information about container implementations, identified by a uid prefix. + +registry = {} + +def registerContainerClass(cls): + # TODO: error on duplicate key + registry[cls.itemFactory.prefix] = cls + cls.headCols = cols = tuple(f.lower() for f in cls.itemFactory.headFields) + if cls.indexes is None: + cls.indexes = [cols[i:] for i in range(len(cols))] + return cls + diff --git a/scopes/storage/proxy.py b/scopes/storage/proxy.py new file mode 100644 index 0000000..7281294 --- /dev/null +++ b/scopes/storage/proxy.py @@ -0,0 +1,50 @@ +# scopes.storage.proxy + +"""Core classes and helper functions for creating proxy and adapter objects +in order to store attribute values in a SQL database. + +This is currently in concept and exploration state. +""" + +import transaction + +_not_found = object() + + +def loadData(obj): + print ('getData ***', obj.context.__name__, obj.context.__parent__.__name__) + return dict(dummy='dummy') + + +def storeData(obj, data): + print ('storeData ***', obj.context.__name__, obj.context.__parent__.__name__, data) + + +class AdapterBase(object): + + _old_data = None + _cont = None + _id = None + + def __init__(self, context): + super(AdapterBase, self).__init__(context) + object.__setattr__(self, '_new_data', {}) + + def __getattr__(self, attr): + value = self._new_data.get(attr, _not_found) + if value is _not_found: + if self._old_data is None: + object.__setattr__(self, '_old_data', loadData(self)) + value = self._old_data.get(attr, _not_found) + if value is _not_found: + return super(AdapterBase, self).__getattr__(attr) + return value + + def __setattr__(self, attr, value): + super(AdapterBase, self).__setattr__(attr, value) + if attr.startswith('__') or attr in self._adapterAttributes: + return + if not self._new_data: + tr = transaction.manager.get() + tr.addBeforeCommitHook(storeData, [self, self._new_data], {}) + self._new_data[attr] = value diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py new file mode 100644 index 0000000..dc74605 --- /dev/null +++ b/scopes/storage/tracking.py @@ -0,0 +1,178 @@ +# scopes.storage.tracking + +"""SQL-based storage for simple tracks (records). + +A track consists of a head (index data, metadata) with a fixed set of fields and +data (payload) represented as a dict. +""" + +from datetime import datetime +from sqlalchemy import Table, Column, Index +from sqlalchemy import BigInteger, DateTime, Text, func +from sqlalchemy import and_ +from sqlalchemy.dialects.postgresql import JSONB +import transaction +from zope.sqlalchemy import register, mark_changed + +from scopes.storage.common import registerContainerClass + + +class Track(object): + + headFields = ['taskId', 'userName'] + prefix = 'rec' + + def __init__(self, *keys, data=None, timeStamp=None, trackId=None, container=None): + self.head = {} + for ix, k in enumerate(keys): + self.head[self.headFields[ix]] = k + for k in self.headFields: + if self.head.get(k) is None: + self.heaad[k] = '' + setattr(self, k, self.head[k]) + self.data = data or {} + self.timeStamp = timeStamp + self.trackId = trackId + self.container = container + + def update(self, data, overwrite=False): + if data is None: + return + if overwrite: + self.data = data + else: + self.data.update(data) + + @property + def uid(self): + if self.trackId is None: + return None + return '%s-%d' % (self.prefix, self.trackId) + + +@registerContainerClass +class Container(object): + + itemFactory = Track + tableName = 'tracks' + insertOnChange = True # always insert new track when data are changed + indexes = None # default, will be overwritten by registerContainerClass() + #indexes = [('username',), ('taskid', 'username')] # or put explicitly in class + + table = None + + def __init__(self, storage): + self.storage = storage + self.session = storage.session + self.table = self.getTable() + + def get(self, trackId): + stmt = self.table.select().where(self.table.c.trackid == trackId) + return self.makeTrack(self.session.execute(stmt).first()) + + def query(self, **crit): + stmt = self.table.select().where( + and_(*self.setupWhere(crit))).order_by(self.table.c.trackid) + for r in self.session.execute(stmt): + yield self.makeTrack(r) + + def queryLast(self, **crit): + stmt = (self.table.select().where(and_(*self.setupWhere(crit))). + order_by(self.table.c.trackid.desc()).limit(1)) + return self.makeTrack(self.session.execute(stmt).first()) + + def save(self, track): + crit = dict((hf, track.head[hf]) for hf in track.headFields) + found = self.queryLast(**crit) + if found is None: + return self.insert(track) + if self.insertOnChange and found.data != track.data: + return self.insert(track) + if found.data != track.data or found.timeStamp != track.timeStamp: + found.update(track.data) + found.timeStamp = track.timeStamp + self.update(found) + return found.trackId + + def insert(self, track, withTrackId=False): + t = self.table + values = self.setupValues(track, withTrackId) + stmt = t.insert().values(**values).returning(t.c.trackid) + trackId = self.session.execute(stmt).first()[0] + mark_changed(self.session) + return trackId + + def update(self, track): + t = self.table + values = self.setupValues(track) + if track.timeStamp is None: + values['timestamp'] = datetime.now() + stmt = t.update().values(**values).where(t.c.trackid == track.trackId) + n = self.session.execute(stmt).rowcount + if n > 0: + mark_changed(self.session) + return n + + def upsert(self, track): + """Try to update the record identified by the trackId given with ``track``. + If not found insert new record without generating a new trackId. + Use this method for migration and other bulk insert/update tasks. + Don't forget to update the trackid sequence afterwards: + ``select setval('.tracks_trackid_seq', );``""" + if track.trackId is not None: + if self.update(track) > 0: + return track.trackId + return self.insert(track, withTrackId=True) + + def remove(self, trackId): + stmt = self.table.delete().where(self.table.c.trackid == trackId) + n = self.session.execute(stmt).rowcount + if n > 0: + mark_changed(self.session) + return n + + def makeTrack(self, r): + if r is None: + return None + return self.itemFactory( + *r[1:-2], trackId=r[0],timeStamp=r[-2], data=r[-1], container=self) + + def setupWhere(self, crit): + return [self.table.c[k.lower()] == v for k, v in crit.items()] + + def setupValues(self, track, withTrackId=False): + values = {} + hf = self.itemFactory.headFields + for i, c in enumerate(self.headCols): + values[c] = track.head[hf[i]] + values['data'] = track.data + if track.timeStamp is not None: + values['timestamp'] = track.timeStamp + if withTrackId and track.trackId is not None: + values['trackid'] = track.trackId + return values + + def getTable(self): + #table = self.storage.getExistingTable(self.tableName) + #if table is None: + return createTable(self.storage, self.tableName, self.headCols, + indexes=self.indexes) + + +def createTable(storage, tableName, headcols, indexes=None): + metadata = storage.metadata + cols = [Column('trackid', BigInteger, primary_key=True)] + idxs = [] + for ix, f in enumerate(headcols): + cols.append(Column(f.lower(), Text, nullable=False, server_default='')) + cols.append(Column('timestamp', DateTime(timezone=True), + nullable=False, server_default=func.now())) + for ix, idef in enumerate(indexes): + indexName = 'idx_%s_%d' % (tableName, (ix + 1)) + idxs.append(Index(indexName, *idef)) + idxs.append(Index('idx_%s_ts' % tableName, 'timestamp')) + cols.append(Column('data', JSONB, nullable=False, server_default='{}')) + table = Table(tableName, metadata, *(cols+idxs), extend_existing=True) + metadata.create_all(storage.engine) + return table + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..ecd37c5 --- /dev/null +++ b/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +setup() + diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..97390e5 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,79 @@ +#! /usr/bin/python + +"""Tests for the 'scopes.storage' package.""" + +from datetime import datetime +import transaction +import unittest + +import scopes.storage.common +from scopes.storage.common import Storage, getEngine, sessionFactory +from scopes.storage import proxy +from scopes.storage import tracking + +engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') +scopes.storage.common.engine = engine +scopes.storage.common.Session = sessionFactory(engine) + +storage = Storage(schema='testing') + + +class Test(unittest.TestCase): + "Basic tests for the cco.storage package." + + def testBasicStuff(self): + storage.dropTable('tracks') + tracks = storage.create(tracking.Container) + + tr01 = tracking.Track('t01', 'john') + tr01.update(dict(activity='testing')) + self.assertEqual(tr01.head, {'taskId': 't01', 'userName': 'john'}) + self.assertEqual(tr01.taskId, 't01') + self.assertEqual(tr01.userName, 'john') + + self.assertTrue(tracks.getTable() is not None) + + trid01 = tracks.save(tr01) + self.assertTrue(trid01 > 0) + + tr01a = tracks.get(trid01) + self.assertEqual(tr01a.head, tr01.head) + self.assertEqual(tr01a.trackId, trid01) + self.assertEqual(tr01a.data.get('activity'), 'testing') + + tr01a.update(dict(text='Set up unit tests.')) + tr01a.timeStamp = None + self.assertTrue(tracks.save(tr01a) > 0) + + tr01b = tracks.queryLast(taskId='t01') + self.assertEqual(tr01b.head, tr01.head) + self.assertNotEqual(tr01b.trackId, trid01) + self.assertEqual(tr01b.data.get('activity'), 'testing') + + tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30), + data=dict(activity='concept')) + trid02 = tracks.upsert(tr02) + self.assertEqual(trid02, 31) + self.assertEqual(tr02.uid, 'rec-31') + tr02.trackId = trid01 + trid021 = tracks.upsert(tr02) + self.assertEqual(trid021, trid01) + self.assertEqual(tr02.uid, 'rec-' + str(trid01)) + + tr03 = storage.getItem('rec-31') + self.assertEqual(tr03.trackId, 31) + + n = tracks.remove(31) + self.assertEqual(n, 1) + self.assertEqual(tracks.get(31), None) + + transaction.commit() + + +def suite(): + return unittest.TestSuite(( + unittest.TestLoader().loadTestsFromTestCase(Test), + )) + +if __name__ == '__main__': + unittest.main(defaultTest='suite')