From c4459866c8f2c9c73d1ae8eb83abb7ad9d14c255 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Sun, 18 Feb 2024 18:48:43 +0100 Subject: [PATCH 01/17] revert encoding stuff for UIDs, just use plain - --- scopes/storage/common.py | 1 - scopes/storage/tracking.py | 4 +--- tests/test_storage.py | 9 +++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 6e32844..89f6673 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -42,7 +42,6 @@ class Storage(object): self.containers[container.itemFactory.prefix] = container def getItem(self, uid): - uid = base64.urlsafe_b64decode(uid[1:]).decode() prefix, id = uid.split('-') id = int(id) container = self.containers.get(prefix) diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 184c21a..4c0ea38 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -48,9 +48,7 @@ class Track(object): def uid(self): if self.trackId is None: return None - raw = ('%s-%d' % (self.prefix, self.trackId)).encode() - return 'b' + base64.urlsafe_b64encode(raw).decode() - #return '%s-%d' % (self.prefix, self.trackId) + return '%s-%d' % (self.prefix, self.trackId) @registerContainerClass diff --git a/tests/test_storage.py b/tests/test_storage.py index 452345d..97390e5 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -54,16 +54,13 @@ class Test(unittest.TestCase): data=dict(activity='concept')) trid02 = tracks.upsert(tr02) self.assertEqual(trid02, 31) - self.assertEqual(tr02.uid, 'bcmVjLTMx') - #self.assertEqual(tr02.uid, 'rec-31') + self.assertEqual(tr02.uid, 'rec-31') tr02.trackId = trid01 trid021 = tracks.upsert(tr02) self.assertEqual(trid021, trid01) - self.assertEqual(tr02.uid, 'bcmVjLTE=') - #self.assertEqual(tr02.uid, 'rec-' + str(trid01)) + self.assertEqual(tr02.uid, 'rec-' + str(trid01)) - #tr03 = storage.getItem(b'bcmVjLTE=') - tr03 = storage.getItem('bcmVjLTMx') + tr03 = storage.getItem('rec-31') self.assertEqual(tr03.trackId, 31) n = tracks.remove(31) From 265048e76afe041db74e4a55dca056539d9107aa Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Tue, 20 Feb 2024 16:10:26 +0100 Subject: [PATCH 02/17] add demo wsgi project (server + app) --- demo/app.py | 8 ++++++++ demo/config.py | 9 +++++++++ demo/main.py | 9 +++++++++ demo/server.py | 12 ++++++++++++ pyproject.toml | 5 +++++ 5 files changed, 43 insertions(+) create mode 100644 demo/app.py create mode 100644 demo/config.py create mode 100644 demo/main.py create mode 100644 demo/server.py diff --git a/demo/app.py b/demo/app.py new file mode 100644 index 0000000..2f44926 --- /dev/null +++ b/demo/app.py @@ -0,0 +1,8 @@ +# py-scopes/demo/app.py + +def demo_app(environ, start_response): + status = '200 OK' + headers = [("Content-type", "text/plain; charset=utf-8")] + start_response(status, headers) + return ['Hello World'.encode()] + diff --git a/demo/config.py b/demo/config.py new file mode 100644 index 0000000..c4f2668 --- /dev/null +++ b/demo/config.py @@ -0,0 +1,9 @@ +# py-scopes/demo/config.py + +from dotenv import load_dotenv +from os import getenv + +load_dotenv() + +server_port = getenv('SERVER_PORT', '8999') + diff --git a/demo/main.py b/demo/main.py new file mode 100644 index 0000000..4eef565 --- /dev/null +++ b/demo/main.py @@ -0,0 +1,9 @@ +# py-scopes/demo/main.py + +import config + +from app import demo_app +import server + +server.run(demo_app, config) + diff --git a/demo/server.py b/demo/server.py new file mode 100644 index 0000000..aabed33 --- /dev/null +++ b/demo/server.py @@ -0,0 +1,12 @@ +# py-scopes/demo/server.py + +from wsgiref.simple_server import make_server + +def run(app, config): + port = int(config.server_port) + with make_server('', port, app) as httpd: + print(f'Serving on port {port}.') + try: + httpd.serve_forever() + except KeyboardInterrupt: + print('Shutting down.') diff --git a/pyproject.toml b/pyproject.toml index 226e1b5..02a3fe6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,4 +19,9 @@ dependencies = [ ] [project.optional-dependencies] +demo = ["python-dotenv"] test = ["pytest"] + +[tool.setuptools] +packages = ["scopes"] + From 2986d426bb7fcaee03f7cc3beeef7b15b7e9e2c9 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Wed, 21 Feb 2024 22:45:54 +0100 Subject: [PATCH 03/17] move app to scopes.server; rename / clean-up --- demo/app.py | 8 ------ demo/config.py | 2 ++ demo/{server.py => demo_server.py} | 8 +++++- demo/main.py | 9 ------- pyproject.toml | 2 +- scopes/server/__init__.py | 1 + scopes/server/app.py | 43 ++++++++++++++++++++++++++++++ 7 files changed, 54 insertions(+), 19 deletions(-) delete mode 100644 demo/app.py rename demo/{server.py => demo_server.py} (74%) delete mode 100644 demo/main.py create mode 100644 scopes/server/__init__.py create mode 100644 scopes/server/app.py diff --git a/demo/app.py b/demo/app.py deleted file mode 100644 index 2f44926..0000000 --- a/demo/app.py +++ /dev/null @@ -1,8 +0,0 @@ -# py-scopes/demo/app.py - -def demo_app(environ, start_response): - status = '200 OK' - headers = [("Content-type", "text/plain; charset=utf-8")] - start_response(status, headers) - return ['Hello World'.encode()] - diff --git a/demo/config.py b/demo/config.py index c4f2668..464dff2 100644 --- a/demo/config.py +++ b/demo/config.py @@ -2,8 +2,10 @@ from dotenv import load_dotenv from os import getenv +from scopes.server.app import demo_app, zope_app load_dotenv() server_port = getenv('SERVER_PORT', '8999') +app = zope_app diff --git a/demo/server.py b/demo/demo_server.py similarity index 74% rename from demo/server.py rename to demo/demo_server.py index aabed33..219d9bb 100644 --- a/demo/server.py +++ b/demo/demo_server.py @@ -1,4 +1,4 @@ -# py-scopes/demo/server.py +# py-scopes/demo/demo_server.py from wsgiref.simple_server import make_server @@ -10,3 +10,9 @@ def run(app, config): httpd.serve_forever() except KeyboardInterrupt: print('Shutting down.') + + +if __name__ == '__main__': + import config + run(config.app, config) + diff --git a/demo/main.py b/demo/main.py deleted file mode 100644 index 4eef565..0000000 --- a/demo/main.py +++ /dev/null @@ -1,9 +0,0 @@ -# py-scopes/demo/main.py - -import config - -from app import demo_app -import server - -server.run(demo_app, config) - diff --git a/pyproject.toml b/pyproject.toml index 02a3fe6..405c74e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ ] [project.optional-dependencies] -demo = ["python-dotenv"] +demo = ["python-dotenv", "zope.publisher"] test = ["pytest"] [tool.setuptools] diff --git a/scopes/server/__init__.py b/scopes/server/__init__.py new file mode 100644 index 0000000..cf54a0c --- /dev/null +++ b/scopes/server/__init__.py @@ -0,0 +1 @@ +"""package scopes.server""" diff --git a/scopes/server/app.py b/scopes/server/app.py new file mode 100644 index 0000000..675a789 --- /dev/null +++ b/scopes/server/app.py @@ -0,0 +1,43 @@ +# scopes.server.app + +from zope.publisher.base import DefaultPublication +from zope.publisher.browser import BrowserRequest +from zope.publisher.publish import publish + + +def demo_app(environ, start_response): + print(f'*** environ {environ}.') + status = '200 OK' + headers = [("Content-type", "text/plain; charset=utf-8")] + start_response(status, headers) + return ['Hello World'.encode()] + + +def zope_app(environ, start_response): + request = BrowserRequest(environ['wsgi.input'], environ) + request.setPublication(DefaultPublication(AppRoot())) + request = publish(request, False) + response = request.response + start_response(response.getStatusString(), response.getHeaders()) + return response.consumeBodyIter() + + +class AppRoot: + """Zope Demo AppRoot""" + + def __call__(self): + """calling AppRoot""" + return 'At root' + + def __getitem__(self, key): + def child(): + """get child""" + print(f'--- getitem {key}') + return 'getitem' + return child + + def hello(self): + """method hello""" + return 'Hello AppRoot' + + From 90da2aafcd910c3bec5ff542ad474012c24aa3a3 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Mon, 26 Feb 2024 08:34:56 +0100 Subject: [PATCH 04/17] provide config.py in tests folder --- tests/config.py | 14 ++++++++++++++ tests/test_storage.py | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/config.py diff --git a/tests/config.py b/tests/config.py new file mode 100644 index 0000000..2fe27db --- /dev/null +++ b/tests/config.py @@ -0,0 +1,14 @@ +# py-scopes/tests/config.py + +from scopes.server.app import demo_app, zope_app + +# server / app settings +server_port = '8999' +app = zope_app + +# storage settings +dbengine = 'postgresql+psycopg' +dbname = 'testdb' +dbuser = 'testuser' +dbpassword = 'secret' + diff --git a/tests/test_storage.py b/tests/test_storage.py index 97390e5..aff5d0d 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -11,7 +11,8 @@ from scopes.storage.common import Storage, getEngine, sessionFactory from scopes.storage import proxy from scopes.storage import tracking -engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') +import config +engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) scopes.storage.common.engine = engine scopes.storage.common.Session = sessionFactory(engine) @@ -21,7 +22,7 @@ storage = Storage(schema='testing') class Test(unittest.TestCase): "Basic tests for the cco.storage package." - def testBasicStuff(self): + def testTracking(self): storage.dropTable('tracks') tracks = storage.create(tracking.Container) From 1280f8b15ab47cc73b2233dc46897bb5d27ff8aa Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Mon, 26 Feb 2024 09:22:27 +0100 Subject: [PATCH 05/17] work in progress: storage.folder implementation --- demo/demo_server.py | 3 ++- scopes/storage/folder.py | 28 ++++++++++++++++++++++++++++ tests/test_storage.py | 5 ++++- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 scopes/storage/folder.py diff --git a/demo/demo_server.py b/demo/demo_server.py index 219d9bb..b585fac 100644 --- a/demo/demo_server.py +++ b/demo/demo_server.py @@ -15,4 +15,5 @@ def run(app, config): if __name__ == '__main__': import config run(config.app, config) - + #run(config.app_factory(), config) + # see zope.app.wsgi.getWSGIApplication diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py new file mode 100644 index 0000000..a2c85d6 --- /dev/null +++ b/scopes/storage/folder.py @@ -0,0 +1,28 @@ +# scopes.storage.folder + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Folder(Track): + + headFields = ['parent', 'name'] + prefix = 'fldr' + + def keys(self): + return [] + + def get(self, key, default=None): + return default + + def __getitem__(self, key): + raise KeyError + + +@registerContainerClass +class Folders(Container): + + itemFactory = Folder + indexes = [('parent', 'name')] + tableName = 'folders' + insertOnChange = False diff --git a/tests/test_storage.py b/tests/test_storage.py index aff5d0d..9180e3b 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -9,7 +9,7 @@ import unittest import scopes.storage.common from scopes.storage.common import Storage, getEngine, sessionFactory from scopes.storage import proxy -from scopes.storage import tracking +from scopes.storage import folder, tracking import config engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) @@ -70,6 +70,9 @@ class Test(unittest.TestCase): transaction.commit() + def testFolder(self): + storage.dropTable('folders') + folders = storage.create(folder.Folders) def suite(): return unittest.TestSuite(( From 77f5abc7bffc76cc53396a61593e8d63624b7e90 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Tue, 27 Feb 2024 17:40:59 +0100 Subject: [PATCH 06/17] basic folder implementation with tests --- demo/config.py | 9 ++++++++- pyproject.toml | 2 +- scopes/server/app.py | 1 + scopes/storage/folder.py | 22 +++++++++++++++++----- tests/test_storage.py | 13 ++++++++++++- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/demo/config.py b/demo/config.py index 464dff2..80d834c 100644 --- a/demo/config.py +++ b/demo/config.py @@ -6,6 +6,13 @@ from scopes.server.app import demo_app, zope_app load_dotenv() -server_port = getenv('SERVER_PORT', '8999') +server_port = getenv('SERVER_PORT', '8099') app = zope_app + +# storage settings +dbengine = 'postgresql+psycopg' +dbname = getenv('DBNAME', 'demo') +dbuser = getenv('DBUSER', 'demo') +dbpassword = getenv('DBPASSWORD', 'secret') + diff --git a/pyproject.toml b/pyproject.toml index 405c74e..7b09479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ ] [project.optional-dependencies] -demo = ["python-dotenv", "zope.publisher"] +app = ["python-dotenv", "zope.publisher", "zope.traversing"] test = ["pytest"] [tool.setuptools] diff --git a/scopes/server/app.py b/scopes/server/app.py index 675a789..fda9130 100644 --- a/scopes/server/app.py +++ b/scopes/server/app.py @@ -3,6 +3,7 @@ from zope.publisher.base import DefaultPublication from zope.publisher.browser import BrowserRequest from zope.publisher.publish import publish +from zope.traversing.publicationtraverse import PublicationTraverser def demo_app(environ, start_response): diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py index a2c85d6..287b5b7 100644 --- a/scopes/storage/folder.py +++ b/scopes/storage/folder.py @@ -6,23 +6,35 @@ from scopes.storage.tracking import Container, Track class Folder(Track): - headFields = ['parent', 'name'] + headFields = ['parent', 'name', 'ref'] prefix = 'fldr' def keys(self): - return [] + for f in self.container.query(parent=self.uid): + yield f.name def get(self, key, default=None): - return default + value = self.container.queryLast(parent=self.uid, name=key) + if value is None: + return default + return value def __getitem__(self, key): - raise KeyError + value = self.container.queryLast(parent=self.uid, name=key) + if value is None: + raise KeyError + return value + + def __setitem__(self, key, value): + value.head['parent'] = self.uid + value.head['name']= key + self.container.save(value) @registerContainerClass class Folders(Container): itemFactory = Folder - indexes = [('parent', 'name')] + indexes = [('parent', 'name'), ('ref',)] tableName = 'folders' insertOnChange = False diff --git a/tests/test_storage.py b/tests/test_storage.py index 9180e3b..ef8f248 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -72,7 +72,18 @@ class Test(unittest.TestCase): def testFolder(self): storage.dropTable('folders') - folders = storage.create(folder.Folders) + cont = storage.create(folder.Folders) + self.assertEqual(list(cont.query(parent='')), []) + cont.save(folder.Folder('', 'root')) + folders = list(cont.query(parent='')) + self.assertEqual(len(folders), 1) + root = folders[0] + root['child1'] = folder.Folder(data=dict(title='First Child')) + folders = list(cont.query(parent=root.uid)) + self.assertEqual(len(folders), 1) + ch1 = root['child1'] + self.assertEqual(ch1.parent, root.uid) + assert list(root.keys()) == ['child1'] def suite(): return unittest.TestSuite(( From 2382abf129ccaafbe524230a054a36fd42cf0a93 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Tue, 27 Feb 2024 19:07:29 +0100 Subject: [PATCH 07/17] provide a virtual / dummy folder root object for top-down folder creation and access --- scopes/storage/folder.py | 17 ++++++++++++++--- scopes/storage/tracking.py | 9 +++++++++ tests/test_storage.py | 22 ++++++++++------------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py index 287b5b7..6a0eb88 100644 --- a/scopes/storage/folder.py +++ b/scopes/storage/folder.py @@ -22,15 +22,26 @@ class Folder(Track): def __getitem__(self, key): value = self.container.queryLast(parent=self.uid, name=key) if value is None: - raise KeyError + raise KeyError(key) return value def __setitem__(self, key, value): - value.head['parent'] = self.uid - value.head['name']= key + value.set('parent', self.uid) + value.set('name', key) self.container.save(value) +class Root(Folder): + """A dummy (virtual) root folder for creating real folders + using the Folder API.""" + + def __init__(self, storage): + cont = storage.create(Folders) + super(Root, self).__init__(container=cont) + + uid = '' + + @registerContainerClass class Folders(Container): diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 4c0ea38..ba0d5c0 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -36,6 +36,15 @@ class Track(object): self.trackId = trackId self.container = container + def set(self, attr, value): + if attr in self.headFields: + if value is None: + value = '' + self.head[attr] = value + setattr(self, attr, value) + else: + raise AttributeError(attr) + def update(self, data, overwrite=False): if data is None: return diff --git a/tests/test_storage.py b/tests/test_storage.py index ef8f248..ed49eec 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -72,18 +72,16 @@ class Test(unittest.TestCase): def testFolder(self): storage.dropTable('folders') - cont = storage.create(folder.Folders) - self.assertEqual(list(cont.query(parent='')), []) - cont.save(folder.Folder('', 'root')) - folders = list(cont.query(parent='')) - self.assertEqual(len(folders), 1) - root = folders[0] - root['child1'] = folder.Folder(data=dict(title='First Child')) - folders = list(cont.query(parent=root.uid)) - self.assertEqual(len(folders), 1) - ch1 = root['child1'] - self.assertEqual(ch1.parent, root.uid) - assert list(root.keys()) == ['child1'] + root = folder.Root(storage) + self.assertEqual(list(root.keys()), []) + root['top'] = folder.Folder() + self.assertEqual(list(root.keys()), ['top']) + top = root['top'] + top['child1'] = folder.Folder(data=dict(title='First Child')) + self.assertEqual(list(top.keys()), ['child1']) + ch1 = top['child1'] + self.assertEqual(ch1.parent, top.uid) + assert list(top.keys()) == ['child1'] def suite(): return unittest.TestSuite(( From e2df59247cf2b89d3b47e070aacd8afd8d490f83 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Tue, 5 Mar 2024 15:18:38 +0100 Subject: [PATCH 08/17] work in progress: concepts, relations, ...; folder: use name as value for parent --- scopes/organize/__init__.py | 1 + scopes/storage/common.py | 6 ++++-- scopes/storage/concept.py | 12 ++++++++++++ scopes/storage/folder.py | 8 ++++---- scopes/storage/relation.py | 3 +++ tests/test_storage.py | 2 +- 6 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 scopes/organize/__init__.py create mode 100644 scopes/storage/concept.py create mode 100644 scopes/storage/relation.py diff --git a/scopes/organize/__init__.py b/scopes/organize/__init__.py new file mode 100644 index 0000000..829e828 --- /dev/null +++ b/scopes/organize/__init__.py @@ -0,0 +1 @@ +"""package scopes.organize""" diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 89f6673..6d04ec1 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -71,8 +71,10 @@ class Storage(object): registry = {} def registerContainerClass(cls): - # TODO: error on duplicate key - registry[cls.itemFactory.prefix] = cls + prefix = cls.itemFactory.prefix + if prefix in registry: + raise ValueError("prefix '%s' already registered!" % prefix) + registry[prefix] = cls cls.headCols = cols = tuple(f.lower() for f in cls.itemFactory.headFields) if cls.indexes is None: cls.indexes = [cols[i:] for i in range(len(cols))] diff --git a/scopes/storage/concept.py b/scopes/storage/concept.py new file mode 100644 index 0000000..967f3a4 --- /dev/null +++ b/scopes/storage/concept.py @@ -0,0 +1,12 @@ +# scopes.storage.concept + +"""Abstract base classes for concept map application classes.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Concept(Track): + + headFields = ['parent', 'name'] + diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py index 6a0eb88..c5e80d5 100644 --- a/scopes/storage/folder.py +++ b/scopes/storage/folder.py @@ -10,23 +10,23 @@ class Folder(Track): prefix = 'fldr' def keys(self): - for f in self.container.query(parent=self.uid): + for f in self.container.query(parent=self.name): yield f.name def get(self, key, default=None): - value = self.container.queryLast(parent=self.uid, name=key) + value = self.container.queryLast(parent=self.name, name=key) if value is None: return default return value def __getitem__(self, key): - value = self.container.queryLast(parent=self.uid, name=key) + value = self.container.queryLast(parent=self.name, name=key) if value is None: raise KeyError(key) return value def __setitem__(self, key, value): - value.set('parent', self.uid) + value.set('parent', self.name) value.set('name', key) self.container.save(value) diff --git a/scopes/storage/relation.py b/scopes/storage/relation.py new file mode 100644 index 0000000..100528e --- /dev/null +++ b/scopes/storage/relation.py @@ -0,0 +1,3 @@ +# scopes.storage.relation + +"""An SQL-based relationship engine using RDF-like triples.""" diff --git a/tests/test_storage.py b/tests/test_storage.py index ed49eec..ab234ad 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -80,7 +80,7 @@ class Test(unittest.TestCase): top['child1'] = folder.Folder(data=dict(title='First Child')) self.assertEqual(list(top.keys()), ['child1']) ch1 = top['child1'] - self.assertEqual(ch1.parent, top.uid) + self.assertEqual(ch1.parent, top.name) assert list(top.keys()) == ['child1'] def suite(): From 83071842c8b38973404bdb20d23e3b4d5c220d40 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Tue, 5 Mar 2024 21:03:36 +0100 Subject: [PATCH 09/17] folder: use rid for parent; more on concept / organize --- scopes/organize/task.py | 13 +++++++++++++ scopes/storage/concept.py | 2 +- scopes/storage/folder.py | 8 ++++---- scopes/storage/relation.py | 10 ++++++++++ scopes/storage/tracking.py | 6 ++++++ tests/test_storage.py | 2 +- 6 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 scopes/organize/task.py diff --git a/scopes/organize/task.py b/scopes/organize/task.py new file mode 100644 index 0000000..c0a98cc --- /dev/null +++ b/scopes/organize/task.py @@ -0,0 +1,13 @@ +# scopes.organize.task + +"""Task (and corresponding container) implementation.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.concept import Concept + + +class Task(Concept): + + headFields = ['name'] + prefix = 'tsk' + diff --git a/scopes/storage/concept.py b/scopes/storage/concept.py index 967f3a4..95ede52 100644 --- a/scopes/storage/concept.py +++ b/scopes/storage/concept.py @@ -8,5 +8,5 @@ from scopes.storage.tracking import Container, Track class Concept(Track): - headFields = ['parent', 'name'] + headFields = ['name'] diff --git a/scopes/storage/folder.py b/scopes/storage/folder.py index c5e80d5..9ac89be 100644 --- a/scopes/storage/folder.py +++ b/scopes/storage/folder.py @@ -10,23 +10,23 @@ class Folder(Track): prefix = 'fldr' def keys(self): - for f in self.container.query(parent=self.name): + for f in self.container.query(parent=self.rid): yield f.name def get(self, key, default=None): - value = self.container.queryLast(parent=self.name, name=key) + value = self.container.queryLast(parent=self.rid, name=key) if value is None: return default return value def __getitem__(self, key): - value = self.container.queryLast(parent=self.name, name=key) + value = self.container.queryLast(parent=self.rid, name=key) if value is None: raise KeyError(key) return value def __setitem__(self, key, value): - value.set('parent', self.name) + value.set('parent', self.rid) value.set('name', key) self.container.save(value) diff --git a/scopes/storage/relation.py b/scopes/storage/relation.py index 100528e..2d2bd5a 100644 --- a/scopes/storage/relation.py +++ b/scopes/storage/relation.py @@ -1,3 +1,13 @@ # scopes.storage.relation """An SQL-based relationship engine using RDF-like triples.""" + +from scopes.storage.common import registerContainerClass +from scopes.storage.tracking import Container, Track + + +class Triple(Track): + + headFields = ['first', 'second', 'pred'] + prefix = 'rel' + diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index ba0d5c0..9a8d8f3 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -59,6 +59,12 @@ class Track(object): return None return '%s-%d' % (self.prefix, self.trackId) + @property + def rid(self): + if self.trackId is None: + return '' + return str(self.trackId) + @registerContainerClass class Container(object): diff --git a/tests/test_storage.py b/tests/test_storage.py index ab234ad..d61f459 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -80,7 +80,7 @@ class Test(unittest.TestCase): top['child1'] = folder.Folder(data=dict(title='First Child')) self.assertEqual(list(top.keys()), ['child1']) ch1 = top['child1'] - self.assertEqual(ch1.parent, top.name) + self.assertEqual(ch1.parent, top.rid) assert list(top.keys()) == ['child1'] def suite(): From 592e65356166450cd058d074af5a7f6c35fb9871 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Wed, 6 Mar 2024 22:45:59 +0100 Subject: [PATCH 10/17] provide access to SQLite, with functions and variables for db-specific variants --- .gitignore | 1 + scopes/storage/common.py | 28 +++++++++++++++++++--------- scopes/storage/db/__init__.py | 1 + scopes/storage/db/postgres.py | 23 +++++++++++++++++++++++ scopes/storage/tracking.py | 10 ++++------ tests/config.py | 8 ++++++++ tests/test_storage.py | 20 +++++++++++++------- 7 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 scopes/storage/db/__init__.py create mode 100644 scopes/storage/db/postgres.py diff --git a/.gitignore b/.gitignore index 4e355e3..f625fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ *#*# *.#* __pycache__ +var diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 6d04ec1..9e6142e 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -4,19 +4,28 @@ import base64 from sqlalchemy import create_engine, MetaData, text -from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy import Integer +from sqlalchemy.dialects.sqlite import JSON import threading -import zope.sqlalchemy -def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): - return create_engine('%s://%s:%s@%s:%s/%s' % ( - dbtype, user, pw, host, port, dbname), **kw) +# predefined db-specific definitions, usable for SQLite; +# may be overriden by import of ``scopes.storage.db.`` def sessionFactory(engine): - Session = scoped_session(sessionmaker(bind=engine, twophase=True)) - zope.sqlalchemy.register(Session) - return Session + return engine.connect + +def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s:///%s' % (dbtype, dbname), **kw) + +def mark_changed(session): + pass + +def commit(conn): + conn.commit() + +IdType = Integer +JsonType = JSON # put something like this in code before first creating a Storage object #engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') @@ -57,7 +66,8 @@ class Storage(object): def dropTable(self, tableName): with self.engine.begin() as conn: - conn.execute(text('drop table if exists %s.%s' % (self.schema, tableName))) + prefix = self.schema and self.schema + '.' or '' + conn.execute(text('drop table if exists %s%s' % (prefix, tableName))) def resetSequence(self, tableName, colName, v): sq = ('alter sequence %s.%s_%s_seq restart %i' % diff --git a/scopes/storage/db/__init__.py b/scopes/storage/db/__init__.py new file mode 100644 index 0000000..467122c --- /dev/null +++ b/scopes/storage/db/__init__.py @@ -0,0 +1 @@ +"""scopes.storage.db""" diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py new file mode 100644 index 0000000..5fd0773 --- /dev/null +++ b/scopes/storage/db/postgres.py @@ -0,0 +1,23 @@ +# scopes.storage.db.postgres + +"""Database-related code specific for PostgreSQL.""" + +from sqlalchemy import create_engine +from sqlalchemy import BigInteger, JSONB +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import scoped_session, sessionmaker +import transaction +from zope.sqlalchemy import register, mark_changed + + +def sessionFactory(engine): + Session = scoped_session(sessionmaker(bind=engine, twophase=True)) + register(Session) + return Session + +def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s://%s:%s@%s:%s/%s' % ( + dbtype, user, pw, host, port, dbname), **kw) + +def commit(conn): + transaction.commit() diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 9a8d8f3..76b7cf5 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -9,12 +9,10 @@ data (payload) represented as a dict. import base64 from datetime import datetime from sqlalchemy import Table, Column, Index -from sqlalchemy import BigInteger, DateTime, Text, func +from sqlalchemy import DateTime, Text, func from sqlalchemy import and_ -from sqlalchemy.dialects.postgresql import JSONB -import transaction -from zope.sqlalchemy import register, mark_changed +from scopes.storage.common import commit, IdType, JsonType, mark_changed from scopes.storage.common import registerContainerClass @@ -177,7 +175,7 @@ class Container(object): def createTable(storage, tableName, headcols, indexes=None): metadata = storage.metadata - cols = [Column('trackid', BigInteger, primary_key=True)] + cols = [Column('trackid', IdType, primary_key=True)] idxs = [] for ix, f in enumerate(headcols): cols.append(Column(f.lower(), Text, nullable=False, server_default='')) @@ -187,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None): indexName = 'idx_%s_%d' % (tableName, (ix + 1)) idxs.append(Index(indexName, *idef)) idxs.append(Index('idx_%s_ts' % tableName, 'timestamp')) - cols.append(Column('data', JSONB, nullable=False, server_default='{}')) + cols.append(Column('data', JsonType, nullable=False, server_default='{}')) table = Table(tableName, metadata, *(cols+idxs), extend_existing=True) metadata.create_all(storage.engine) return table diff --git a/tests/config.py b/tests/config.py index 2fe27db..5ed3d8a 100644 --- a/tests/config.py +++ b/tests/config.py @@ -7,8 +7,16 @@ server_port = '8999' app = zope_app # storage settings + +# PostgreSQL dbengine = 'postgresql+psycopg' dbname = 'testdb' dbuser = 'testuser' dbpassword = 'secret' +dbschema = 'testing' + +# SQLite +dbengine = 'sqlite' +dbname = 'var/test.db' +dbschema = None diff --git a/tests/test_storage.py b/tests/test_storage.py index d61f459..8b2fbb6 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -2,27 +2,30 @@ """Tests for the 'scopes.storage' package.""" +import config + from datetime import datetime import transaction import unittest import scopes.storage.common -from scopes.storage.common import Storage, getEngine, sessionFactory +from scopes.storage.common import commit, Storage, getEngine, sessionFactory from scopes.storage import proxy from scopes.storage import folder, tracking -import config -engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) +engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) scopes.storage.common.engine = engine scopes.storage.common.Session = sessionFactory(engine) -storage = Storage(schema='testing') +#storage = Storage(schema='testing') +#storage = Storage(schema=config.dbschema) +storage = Storage() class Test(unittest.TestCase): "Basic tests for the cco.storage package." - def testTracking(self): + def test_001_tracking(self): storage.dropTable('tracks') tracks = storage.create(tracking.Container) @@ -68,9 +71,9 @@ class Test(unittest.TestCase): self.assertEqual(n, 1) self.assertEqual(tracks.get(31), None) - transaction.commit() + commit(storage.session) - def testFolder(self): + def test_002_folder(self): storage.dropTable('folders') root = folder.Root(storage) self.assertEqual(list(root.keys()), []) @@ -83,6 +86,9 @@ class Test(unittest.TestCase): self.assertEqual(ch1.parent, top.rid) assert list(top.keys()) == ['child1'] + #transaction.commit() + storage.session.commit() + def suite(): return unittest.TestSuite(( unittest.TestLoader().loadTestsFromTestCase(Test), From dba278fc025e6abe40da6c5f17a3e8cba2bf73aa Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Thu, 7 Mar 2024 09:02:59 +0100 Subject: [PATCH 11/17] ... and back to PostgreSQL --- pyproject.toml | 8 +++++--- scopes/storage/db/postgres.py | 11 ++++++++++- tests/config.py | 7 ++++--- tests/test_storage.py | 7 ++----- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7b09479..28154e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,20 +5,22 @@ build-backend = "setuptools.build_meta" [project] name = "py-scopes" version = "3.0.1" -description = "Implementation of the strange 'scopes' paradigma in Python" +description = "Implementation of the unknown 'scopes' paradigm in Python" readme = "README.md" license = {text = "MIT"} keywords = ["scopes"] authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}] dependencies = [ +] + +[project.optional-dependencies] +postgres = [ "transaction", "psycopg[binary]", "SQLAlchemy", "zope.sqlalchemy", ] - -[project.optional-dependencies] app = ["python-dotenv", "zope.publisher", "zope.traversing"] test = ["pytest"] diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py index 5fd0773..55e49b8 100644 --- a/scopes/storage/db/postgres.py +++ b/scopes/storage/db/postgres.py @@ -3,7 +3,7 @@ """Database-related code specific for PostgreSQL.""" from sqlalchemy import create_engine -from sqlalchemy import BigInteger, JSONB +from sqlalchemy import BigInteger from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import scoped_session, sessionmaker import transaction @@ -21,3 +21,12 @@ def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): def commit(conn): transaction.commit() + +import scopes.storage.common +scopes.storage.common.IdType = BigInteger +scopes.storage.common.JsonType = JSONB +scopes.storage.common.sessionFactory = sessionFactory +scopes.storage.common.getEngine = getEngine +scopes.storage.common.mark_changed = mark_changed +scopes.storage.common.commit = commit + diff --git a/tests/config.py b/tests/config.py index 5ed3d8a..881dd13 100644 --- a/tests/config.py +++ b/tests/config.py @@ -9,6 +9,7 @@ app = zope_app # storage settings # PostgreSQL +import scopes.storage.db.postgres dbengine = 'postgresql+psycopg' dbname = 'testdb' dbuser = 'testuser' @@ -16,7 +17,7 @@ dbpassword = 'secret' dbschema = 'testing' # SQLite -dbengine = 'sqlite' -dbname = 'var/test.db' -dbschema = None +#dbengine = 'sqlite' +#dbname = 'var/test.db' +#dbschema = None diff --git a/tests/test_storage.py b/tests/test_storage.py index 8b2fbb6..59598bd 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -17,9 +17,7 @@ engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassw scopes.storage.common.engine = engine scopes.storage.common.Session = sessionFactory(engine) -#storage = Storage(schema='testing') -#storage = Storage(schema=config.dbschema) -storage = Storage() +storage = Storage(schema=config.dbschema) class Test(unittest.TestCase): @@ -86,8 +84,7 @@ class Test(unittest.TestCase): self.assertEqual(ch1.parent, top.rid) assert list(top.keys()) == ['child1'] - #transaction.commit() - storage.session.commit() + commit(storage.session) def suite(): return unittest.TestSuite(( From 406672048636221037288153f049ca6ffa78cbf3 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Thu, 7 Mar 2024 09:38:55 +0100 Subject: [PATCH 12/17] move real testing code to separate 'tlib' module --- pyproject.toml | 4 +- scopes/storage/db/postgres.py | 1 + tests/test_storage.py | 63 ++---------------------------- tests/tlib.py | 72 +++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 62 deletions(-) create mode 100644 tests/tlib.py diff --git a/pyproject.toml b/pyproject.toml index 28154e6..6f9040d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,13 +12,13 @@ keywords = ["scopes"] authors = [{name = "Helmut Merz", email = "helmutm@cy55.de"}] dependencies = [ + "SQLAlchemy", ] [project.optional-dependencies] postgres = [ - "transaction", "psycopg[binary]", - "SQLAlchemy", + "transaction", "zope.sqlalchemy", ] app = ["python-dotenv", "zope.publisher", "zope.traversing"] diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py index 55e49b8..beaa34a 100644 --- a/scopes/storage/db/postgres.py +++ b/scopes/storage/db/postgres.py @@ -22,6 +22,7 @@ def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): def commit(conn): transaction.commit() +# patch `common` module import scopes.storage.common scopes.storage.common.IdType = BigInteger scopes.storage.common.JsonType = JSONB diff --git a/tests/test_storage.py b/tests/test_storage.py index 59598bd..bea9d1f 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -5,7 +5,6 @@ import config from datetime import datetime -import transaction import unittest import scopes.storage.common @@ -19,72 +18,16 @@ scopes.storage.common.Session = sessionFactory(engine) storage = Storage(schema=config.dbschema) +import tlib class Test(unittest.TestCase): "Basic tests for the cco.storage package." def test_001_tracking(self): - storage.dropTable('tracks') - tracks = storage.create(tracking.Container) - - tr01 = tracking.Track('t01', 'john') - tr01.update(dict(activity='testing')) - self.assertEqual(tr01.head, {'taskId': 't01', 'userName': 'john'}) - self.assertEqual(tr01.taskId, 't01') - self.assertEqual(tr01.userName, 'john') - - self.assertTrue(tracks.getTable() is not None) - - trid01 = tracks.save(tr01) - self.assertTrue(trid01 > 0) - - tr01a = tracks.get(trid01) - self.assertEqual(tr01a.head, tr01.head) - self.assertEqual(tr01a.trackId, trid01) - self.assertEqual(tr01a.data.get('activity'), 'testing') - - tr01a.update(dict(text='Set up unit tests.')) - tr01a.timeStamp = None - self.assertTrue(tracks.save(tr01a) > 0) - - tr01b = tracks.queryLast(taskId='t01') - self.assertEqual(tr01b.head, tr01.head) - self.assertNotEqual(tr01b.trackId, trid01) - self.assertEqual(tr01b.data.get('activity'), 'testing') - - tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30), - data=dict(activity='concept')) - trid02 = tracks.upsert(tr02) - self.assertEqual(trid02, 31) - self.assertEqual(tr02.uid, 'rec-31') - tr02.trackId = trid01 - trid021 = tracks.upsert(tr02) - self.assertEqual(trid021, trid01) - self.assertEqual(tr02.uid, 'rec-' + str(trid01)) - - tr03 = storage.getItem('rec-31') - self.assertEqual(tr03.trackId, 31) - - n = tracks.remove(31) - self.assertEqual(n, 1) - self.assertEqual(tracks.get(31), None) - - commit(storage.session) + tlib.test_tracking(self, storage) def test_002_folder(self): - storage.dropTable('folders') - root = folder.Root(storage) - self.assertEqual(list(root.keys()), []) - root['top'] = folder.Folder() - self.assertEqual(list(root.keys()), ['top']) - top = root['top'] - top['child1'] = folder.Folder(data=dict(title='First Child')) - self.assertEqual(list(top.keys()), ['child1']) - ch1 = top['child1'] - self.assertEqual(ch1.parent, top.rid) - assert list(top.keys()) == ['child1'] - - commit(storage.session) + tlib.test_folder(self, storage) def suite(): return unittest.TestSuite(( diff --git a/tests/tlib.py b/tests/tlib.py new file mode 100644 index 0000000..00fa6d7 --- /dev/null +++ b/tests/tlib.py @@ -0,0 +1,72 @@ +"""The real test implementations""" + + +from datetime import datetime +from scopes.storage import folder, tracking +from scopes.storage.common import commit + + +def test_tracking(self, storage): + storage.dropTable('tracks') + tracks = storage.create(tracking.Container) + + tr01 = tracking.Track('t01', 'john') + tr01.update(dict(activity='testing')) + self.assertEqual(tr01.head, {'taskId': 't01', 'userName': 'john'}) + self.assertEqual(tr01.taskId, 't01') + self.assertEqual(tr01.userName, 'john') + + self.assertTrue(tracks.getTable() is not None) + + trid01 = tracks.save(tr01) + self.assertTrue(trid01 > 0) + + tr01a = tracks.get(trid01) + self.assertEqual(tr01a.head, tr01.head) + self.assertEqual(tr01a.trackId, trid01) + self.assertEqual(tr01a.data.get('activity'), 'testing') + + tr01a.update(dict(text='Set up unit tests.')) + tr01a.timeStamp = None + self.assertTrue(tracks.save(tr01a) > 0) + + tr01b = tracks.queryLast(taskId='t01') + self.assertEqual(tr01b.head, tr01.head) + self.assertNotEqual(tr01b.trackId, trid01) + self.assertEqual(tr01b.data.get('activity'), 'testing') + + tr02 = tracking.Track('t02', 'jim', trackId=31, timeStamp=datetime(2023, 11, 30), + data=dict(activity='concept')) + trid02 = tracks.upsert(tr02) + self.assertEqual(trid02, 31) + self.assertEqual(tr02.uid, 'rec-31') + tr02.trackId = trid01 + trid021 = tracks.upsert(tr02) + self.assertEqual(trid021, trid01) + self.assertEqual(tr02.uid, 'rec-' + str(trid01)) + + tr03 = storage.getItem('rec-31') + self.assertEqual(tr03.trackId, 31) + + n = tracks.remove(31) + self.assertEqual(n, 1) + self.assertEqual(tracks.get(31), None) + + commit(storage.session) + + +def test_folder(self, storage): + storage.dropTable('folders') + root = folder.Root(storage) + self.assertEqual(list(root.keys()), []) + root['top'] = folder.Folder() + self.assertEqual(list(root.keys()), ['top']) + top = root['top'] + top['child1'] = folder.Folder(data=dict(title='First Child')) + self.assertEqual(list(top.keys()), ['child1']) + ch1 = top['child1'] + self.assertEqual(ch1.parent, top.rid) + self.assertEqual(list(top.keys()), ['child1']) + + commit(storage.session) + From a4158e96d84f53d7847ba74a033d85c3a4595e49 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Thu, 7 Mar 2024 13:34:41 +0100 Subject: [PATCH 13/17] make tests run (with sqlite) on Android phone --- tests/config.py | 12 ++++++------ tests/test_storage.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/config.py b/tests/config.py index 881dd13..c3617c8 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,15 +1,15 @@ # py-scopes/tests/config.py -from scopes.server.app import demo_app, zope_app +#from scopes.server.app import demo_app, zope_app # server / app settings server_port = '8999' -app = zope_app +#app = zope_app # storage settings # PostgreSQL -import scopes.storage.db.postgres +#import scopes.storage.db.postgres dbengine = 'postgresql+psycopg' dbname = 'testdb' dbuser = 'testuser' @@ -17,7 +17,7 @@ dbpassword = 'secret' dbschema = 'testing' # SQLite -#dbengine = 'sqlite' -#dbname = 'var/test.db' -#dbschema = None +dbengine = 'sqlite' +dbname = 'var/test.db' +dbschema = None diff --git a/tests/test_storage.py b/tests/test_storage.py index bea9d1f..cd8be3b 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -9,7 +9,7 @@ import unittest import scopes.storage.common from scopes.storage.common import commit, Storage, getEngine, sessionFactory -from scopes.storage import proxy +#from scopes.storage import proxy from scopes.storage import folder, tracking engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) From 23f9b1c38445c3406723356e284681bd860a908b Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Fri, 8 Mar 2024 10:02:40 +0100 Subject: [PATCH 14/17] always call test files explicitly: tests/standard (=sqlite) or tests/postgres --- tests/config.py | 10 ++-------- tests/postgres.py | 33 +++++++++++++++++++++++++++++++++ tests/standard.py | 24 ++++++++++++++++++++++++ tests/test_storage.py | 38 -------------------------------------- tests/tlib.py | 16 ++++++++++++---- 5 files changed, 71 insertions(+), 50 deletions(-) create mode 100644 tests/postgres.py create mode 100644 tests/standard.py delete mode 100644 tests/test_storage.py diff --git a/tests/config.py b/tests/config.py index c3617c8..4a19f36 100644 --- a/tests/config.py +++ b/tests/config.py @@ -8,16 +8,10 @@ server_port = '8999' # storage settings -# PostgreSQL -#import scopes.storage.db.postgres -dbengine = 'postgresql+psycopg' -dbname = 'testdb' -dbuser = 'testuser' -dbpassword = 'secret' -dbschema = 'testing' - # SQLite dbengine = 'sqlite' dbname = 'var/test.db' +dbuser = None +dbpassword = None dbschema = None diff --git a/tests/postgres.py b/tests/postgres.py new file mode 100644 index 0000000..f7fe524 --- /dev/null +++ b/tests/postgres.py @@ -0,0 +1,33 @@ +#! /usr/bin/python + +"""Tests for the 'scopes.storage' package - using PostgreSQL.""" + +from datetime import datetime +import unittest + +# PostgreSQL-specific settings +import scopes.storage.db.postgres +import config +config.dbengine = 'postgresql+psycopg' +config.dbname = 'testdb' +config.dbuser = 'testuser' +config.dbpassword = 'secret' +config.dbschema = 'testing' + +import tlib + +class Test(unittest.TestCase): + + def test_001_tracking(self): + tlib.test_tracking(self) + + def test_002_folder(self): + tlib.test_folder(self) + +def suite(): + return unittest.TestSuite(( + unittest.TestLoader().loadTestsFromTestCase(Test), + )) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/tests/standard.py b/tests/standard.py new file mode 100644 index 0000000..cffa9fd --- /dev/null +++ b/tests/standard.py @@ -0,0 +1,24 @@ +#! /usr/bin/python + +"""Tests for the 'scopes.storage' package.""" + +from datetime import datetime +import unittest + +import tlib + +class Test(unittest.TestCase): + + def test_001_tracking(self): + tlib.test_tracking(self) + + def test_002_folder(self): + tlib.test_folder(self) + +def suite(): + return unittest.TestSuite(( + unittest.TestLoader().loadTestsFromTestCase(Test), + )) + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/tests/test_storage.py b/tests/test_storage.py deleted file mode 100644 index cd8be3b..0000000 --- a/tests/test_storage.py +++ /dev/null @@ -1,38 +0,0 @@ -#! /usr/bin/python - -"""Tests for the 'scopes.storage' package.""" - -import config - -from datetime import datetime -import unittest - -import scopes.storage.common -from scopes.storage.common import commit, Storage, getEngine, sessionFactory -#from scopes.storage import proxy -from scopes.storage import folder, tracking - -engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) -scopes.storage.common.engine = engine -scopes.storage.common.Session = sessionFactory(engine) - -storage = Storage(schema=config.dbschema) - -import tlib - -class Test(unittest.TestCase): - "Basic tests for the cco.storage package." - - def test_001_tracking(self): - tlib.test_tracking(self, storage) - - def test_002_folder(self): - tlib.test_folder(self, storage) - -def suite(): - return unittest.TestSuite(( - unittest.TestLoader().loadTestsFromTestCase(Test), - )) - -if __name__ == '__main__': - unittest.main(defaultTest='suite') diff --git a/tests/tlib.py b/tests/tlib.py index 00fa6d7..dac6db3 100644 --- a/tests/tlib.py +++ b/tests/tlib.py @@ -1,12 +1,20 @@ """The real test implementations""" - +import config from datetime import datetime from scopes.storage import folder, tracking -from scopes.storage.common import commit + +import scopes.storage.common +from scopes.storage.common import commit, Storage, getEngine, sessionFactory + +engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) +scopes.storage.common.engine = engine +scopes.storage.common.Session = sessionFactory(engine) + +storage = Storage(schema=config.dbschema) -def test_tracking(self, storage): +def test_tracking(self): storage.dropTable('tracks') tracks = storage.create(tracking.Container) @@ -55,7 +63,7 @@ def test_tracking(self, storage): commit(storage.session) -def test_folder(self, storage): +def test_folder(self): storage.dropTable('folders') root = folder.Root(storage) self.assertEqual(list(root.keys()), []) From fbe8d99d74a0f14bf820ee5b48f6cc207c2f8f14 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Sat, 9 Mar 2024 08:03:53 +0100 Subject: [PATCH 15/17] work in progress: separate settings for different types of databases --- scopes/storage/common.py | 31 +++++++++++++++++++++---- scopes/storage/db/postgres.py | 13 ++++++----- tests/postgres.py | 10 +++++--- tests/{standard.py => test_standard.py} | 6 ++++- tests/tlib.py | 12 +++++----- 5 files changed, 52 insertions(+), 20 deletions(-) rename tests/{standard.py => test_standard.py} (74%) diff --git a/scopes/storage/common.py b/scopes/storage/common.py index 9e6142e..cd8a70c 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -27,10 +27,33 @@ def commit(conn): IdType = Integer JsonType = JSON -# put something like this in code before first creating a Storage object -#engine = getEngine('postgresql+psycopg', 'testdb', 'testuser', 'secret') -#scopes.storage.common.engine = engine -#scopes.storage.common.Session = sessionFactory(engine) + +class StorageFactory(object): + + engine = Session = None + + sessionFactory = sessionFactory + getEngine = getEngine + mark_changed = mark_changed + commit = commit + IdType = IdType + JsonType = JsonType + + def __call__(self, schema=None): + st = Storage(schema=schema) + st.setup(self) + return st + + def setup(self, config): + self.engine = self.getEngine(config.dbengine, config.dbname, + config.dbuser, config.dbpassword) + self.Session = self.sessionFactory + + +# you may put something like this in your code: +#scopes.storage.common.factory = StorageFactory(config) +# and then call at appropriate places: +#storage = scopes.storage.common.factory(schema=...) class Storage(object): diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py index beaa34a..c8a6e5d 100644 --- a/scopes/storage/db/postgres.py +++ b/scopes/storage/db/postgres.py @@ -24,10 +24,11 @@ def commit(conn): # patch `common` module import scopes.storage.common -scopes.storage.common.IdType = BigInteger -scopes.storage.common.JsonType = JSONB -scopes.storage.common.sessionFactory = sessionFactory -scopes.storage.common.getEngine = getEngine -scopes.storage.common.mark_changed = mark_changed -scopes.storage.common.commit = commit +def init(): + scopes.storage.common.IdType = BigInteger + scopes.storage.common.JsonType = JSONB + scopes.storage.common.sessionFactory = sessionFactory + scopes.storage.common.getEngine = getEngine + scopes.storage.common.mark_changed = mark_changed + scopes.storage.common.commit = commit diff --git a/tests/postgres.py b/tests/postgres.py index f7fe524..c6e261e 100644 --- a/tests/postgres.py +++ b/tests/postgres.py @@ -2,11 +2,8 @@ """Tests for the 'scopes.storage' package - using PostgreSQL.""" -from datetime import datetime import unittest -# PostgreSQL-specific settings -import scopes.storage.db.postgres import config config.dbengine = 'postgresql+psycopg' config.dbname = 'testdb' @@ -14,7 +11,14 @@ config.dbuser = 'testuser' config.dbpassword = 'secret' config.dbschema = 'testing' +# PostgreSQL-specific settings +from scopes.storage.db import postgres +postgres.init() + import tlib +tlib.init(config) +#factory = postgres.StorageFactory(config) +#storage = factory(schema='testing') class Test(unittest.TestCase): diff --git a/tests/standard.py b/tests/test_standard.py similarity index 74% rename from tests/standard.py rename to tests/test_standard.py index cffa9fd..cb8554a 100644 --- a/tests/standard.py +++ b/tests/test_standard.py @@ -2,10 +2,14 @@ """Tests for the 'scopes.storage' package.""" -from datetime import datetime import unittest +import config import tlib +tlib.init(config) +#from scopes.storage.common import StorageFactory +#factory = StorageFactory(config) +#storage = factory(schema=None) class Test(unittest.TestCase): diff --git a/tests/tlib.py b/tests/tlib.py index dac6db3..ebd1bcd 100644 --- a/tests/tlib.py +++ b/tests/tlib.py @@ -1,17 +1,17 @@ """The real test implementations""" -import config from datetime import datetime from scopes.storage import folder, tracking import scopes.storage.common from scopes.storage.common import commit, Storage, getEngine, sessionFactory -engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) -scopes.storage.common.engine = engine -scopes.storage.common.Session = sessionFactory(engine) - -storage = Storage(schema=config.dbschema) +def init(config): + global storage + engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) + scopes.storage.common.engine = engine + scopes.storage.common.Session = sessionFactory(engine) + storage = Storage(schema=config.dbschema) def test_tracking(self): From e3bb5b288016b761883030c1eeb1eb6323168e6b Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Sat, 9 Mar 2024 08:57:49 +0100 Subject: [PATCH 16/17] use separate StorageFactory objects for different database / database types --- scopes/storage/common.py | 62 ++++++++++++++--------------------- scopes/storage/db/postgres.py | 39 ++++++++++++---------- scopes/storage/tracking.py | 12 +++---- tests/postgres.py | 12 +++---- tests/test_standard.py | 15 +++++---- tests/tlib.py | 18 +++------- 6 files changed, 70 insertions(+), 88 deletions(-) diff --git a/scopes/storage/common.py b/scopes/storage/common.py index cd8a70c..e26e4b1 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -9,58 +9,46 @@ from sqlalchemy.dialects.sqlite import JSON import threading -# predefined db-specific definitions, usable for SQLite; -# may be overriden by import of ``scopes.storage.db.`` - -def sessionFactory(engine): - return engine.connect - -def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): - return create_engine('%s:///%s' % (dbtype, dbname), **kw) - -def mark_changed(session): - pass - -def commit(conn): - conn.commit() - -IdType = Integer -JsonType = JSON - - class StorageFactory(object): - engine = Session = None + def sessionFactory(self): + return self.engine.connect - sessionFactory = sessionFactory - getEngine = getEngine - mark_changed = mark_changed - commit = commit - IdType = IdType - JsonType = JsonType + @staticmethod + def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s:///%s' % (dbtype, dbname), **kw) - def __call__(self, schema=None): - st = Storage(schema=schema) - st.setup(self) - return st + @staticmethod + def mark_changed(session): + pass - def setup(self, config): + @staticmethod + def commit(conn): + conn.commit() + + IdType = Integer + JsonType = JSON + + def __init__(self, config): self.engine = self.getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) - self.Session = self.sessionFactory + self.Session = self.sessionFactory() + + def __call__(self, schema=None): + return Storage(self, schema=schema) # you may put something like this in your code: -#scopes.storage.common.factory = StorageFactory(config) +#factory = StorageFactory(config) # and then call at appropriate places: #storage = scopes.storage.common.factory(schema=...) - class Storage(object): - def __init__(self, schema=None): - self.engine = engine - self.session = Session() + def __init__(self, db, schema=None): + self.db = db + self.engine = db.engine + self.session = db.Session() self.schema = schema self.metadata = MetaData(schema=schema) self.containers = {} diff --git a/scopes/storage/db/postgres.py b/scopes/storage/db/postgres.py index c8a6e5d..ccc835a 100644 --- a/scopes/storage/db/postgres.py +++ b/scopes/storage/db/postgres.py @@ -9,26 +9,29 @@ from sqlalchemy.orm import scoped_session, sessionmaker import transaction from zope.sqlalchemy import register, mark_changed +from scopes.storage.common import StorageFactory -def sessionFactory(engine): - Session = scoped_session(sessionmaker(bind=engine, twophase=True)) - register(Session) - return Session -def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): - return create_engine('%s://%s:%s@%s:%s/%s' % ( - dbtype, user, pw, host, port, dbname), **kw) +class StorageFactory(StorageFactory): -def commit(conn): - transaction.commit() + def sessionFactory(self): + Session = scoped_session(sessionmaker(bind=self.engine, twophase=True)) + register(Session) + return Session -# patch `common` module -import scopes.storage.common -def init(): - scopes.storage.common.IdType = BigInteger - scopes.storage.common.JsonType = JSONB - scopes.storage.common.sessionFactory = sessionFactory - scopes.storage.common.getEngine = getEngine - scopes.storage.common.mark_changed = mark_changed - scopes.storage.common.commit = commit + @staticmethod + def getEngine(dbtype, dbname, user, pw, host='localhost', port=5432, **kw): + return create_engine('%s://%s:%s@%s:%s/%s' % ( + dbtype, user, pw, host, port, dbname), **kw) + + @staticmethod + def mark_changed(session): + return mark_changed(session) + + @staticmethod + def commit(conn): + transaction.commit() + + IdType = BigInteger + JsonType = JSONB diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 76b7cf5..8e07aa3 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -12,7 +12,6 @@ from sqlalchemy import Table, Column, Index from sqlalchemy import DateTime, Text, func from sqlalchemy import and_ -from scopes.storage.common import commit, IdType, JsonType, mark_changed from scopes.storage.common import registerContainerClass @@ -77,6 +76,7 @@ class Container(object): def __init__(self, storage): self.storage = storage + self.db = storage.db self.session = storage.session self.table = self.getTable() @@ -113,7 +113,7 @@ class Container(object): values = self.setupValues(track, withTrackId) stmt = t.insert().values(**values).returning(t.c.trackid) trackId = self.session.execute(stmt).first()[0] - mark_changed(self.session) + self.db.mark_changed(self.session) return trackId def update(self, track): @@ -124,7 +124,7 @@ class Container(object): stmt = t.update().values(**values).where(t.c.trackid == track.trackId) n = self.session.execute(stmt).rowcount if n > 0: - mark_changed(self.session) + self.db.mark_changed(self.session) return n def upsert(self, track): @@ -142,7 +142,7 @@ class Container(object): stmt = self.table.delete().where(self.table.c.trackid == trackId) n = self.session.execute(stmt).rowcount if n > 0: - mark_changed(self.session) + self.db.mark_changed(self.session) return n def makeTrack(self, r): @@ -175,7 +175,7 @@ class Container(object): def createTable(storage, tableName, headcols, indexes=None): metadata = storage.metadata - cols = [Column('trackid', IdType, primary_key=True)] + cols = [Column('trackid', storage.db.IdType, primary_key=True)] idxs = [] for ix, f in enumerate(headcols): cols.append(Column(f.lower(), Text, nullable=False, server_default='')) @@ -185,7 +185,7 @@ def createTable(storage, tableName, headcols, indexes=None): indexName = 'idx_%s_%d' % (tableName, (ix + 1)) idxs.append(Index(indexName, *idef)) idxs.append(Index('idx_%s_ts' % tableName, 'timestamp')) - cols.append(Column('data', JsonType, nullable=False, server_default='{}')) + cols.append(Column('data', storage.db.JsonType, nullable=False, server_default='{}')) table = Table(tableName, metadata, *(cols+idxs), extend_existing=True) metadata.create_all(storage.engine) return table diff --git a/tests/postgres.py b/tests/postgres.py index c6e261e..f3a6309 100644 --- a/tests/postgres.py +++ b/tests/postgres.py @@ -12,21 +12,19 @@ config.dbpassword = 'secret' config.dbschema = 'testing' # PostgreSQL-specific settings -from scopes.storage.db import postgres -postgres.init() +from scopes.storage.db.postgres import StorageFactory +factory = StorageFactory(config) +storage = factory(schema='testing') import tlib -tlib.init(config) -#factory = postgres.StorageFactory(config) -#storage = factory(schema='testing') class Test(unittest.TestCase): def test_001_tracking(self): - tlib.test_tracking(self) + tlib.test_tracking(self, storage) def test_002_folder(self): - tlib.test_folder(self) + tlib.test_folder(self, storage) def suite(): return unittest.TestSuite(( diff --git a/tests/test_standard.py b/tests/test_standard.py index cb8554a..6ff4a7e 100644 --- a/tests/test_standard.py +++ b/tests/test_standard.py @@ -5,19 +5,22 @@ import unittest import config +config.dbengine = 'sqlite' +config.dbname = 'var/test.db' + +from scopes.storage.common import StorageFactory +factory = StorageFactory(config) +storage = factory(schema=None) + import tlib -tlib.init(config) -#from scopes.storage.common import StorageFactory -#factory = StorageFactory(config) -#storage = factory(schema=None) class Test(unittest.TestCase): def test_001_tracking(self): - tlib.test_tracking(self) + tlib.test_tracking(self, storage) def test_002_folder(self): - tlib.test_folder(self) + tlib.test_folder(self, storage) def suite(): return unittest.TestSuite(( diff --git a/tests/tlib.py b/tests/tlib.py index ebd1bcd..692e070 100644 --- a/tests/tlib.py +++ b/tests/tlib.py @@ -3,18 +3,8 @@ from datetime import datetime from scopes.storage import folder, tracking -import scopes.storage.common -from scopes.storage.common import commit, Storage, getEngine, sessionFactory -def init(config): - global storage - engine = getEngine(config.dbengine, config.dbname, config.dbuser, config.dbpassword) - scopes.storage.common.engine = engine - scopes.storage.common.Session = sessionFactory(engine) - storage = Storage(schema=config.dbschema) - - -def test_tracking(self): +def test_tracking(self, storage): storage.dropTable('tracks') tracks = storage.create(tracking.Container) @@ -60,10 +50,10 @@ def test_tracking(self): self.assertEqual(n, 1) self.assertEqual(tracks.get(31), None) - commit(storage.session) + storage.db.commit(storage.session) -def test_folder(self): +def test_folder(self, storage): storage.dropTable('folders') root = folder.Root(storage) self.assertEqual(list(root.keys()), []) @@ -76,5 +66,5 @@ def test_folder(self): self.assertEqual(ch1.parent, top.rid) self.assertEqual(list(top.keys()), ['child1']) - commit(storage.session) + storage.db.commit(storage.session) From 9e67fe08e666a84920ef7ea27d3d86f523c7a8b6 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Sat, 9 Mar 2024 09:38:52 +0100 Subject: [PATCH 17/17] provide mark_changed() and commit() as Storage methods --- scopes/storage/common.py | 6 ++++++ scopes/storage/tracking.py | 6 +++--- tests/postgres.py | 1 - tests/tlib.py | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/scopes/storage/common.py b/scopes/storage/common.py index e26e4b1..6b4f5d3 100644 --- a/scopes/storage/common.py +++ b/scopes/storage/common.py @@ -53,6 +53,12 @@ class Storage(object): self.metadata = MetaData(schema=schema) self.containers = {} + def commit(self): + self.db.commit(self.session) + + def mark_changed(self): + self.db.mark_changed(self.session) + def create(self, cls): container = cls(self) self.add(container) diff --git a/scopes/storage/tracking.py b/scopes/storage/tracking.py index 8e07aa3..dd805f3 100644 --- a/scopes/storage/tracking.py +++ b/scopes/storage/tracking.py @@ -113,7 +113,7 @@ class Container(object): values = self.setupValues(track, withTrackId) stmt = t.insert().values(**values).returning(t.c.trackid) trackId = self.session.execute(stmt).first()[0] - self.db.mark_changed(self.session) + self.storage.mark_changed() return trackId def update(self, track): @@ -124,7 +124,7 @@ class Container(object): stmt = t.update().values(**values).where(t.c.trackid == track.trackId) n = self.session.execute(stmt).rowcount if n > 0: - self.db.mark_changed(self.session) + self.storage.mark_changed() return n def upsert(self, track): @@ -142,7 +142,7 @@ class Container(object): stmt = self.table.delete().where(self.table.c.trackid == trackId) n = self.session.execute(stmt).rowcount if n > 0: - self.db.mark_changed(self.session) + self.storage.mark_changed() return n def makeTrack(self, r): diff --git a/tests/postgres.py b/tests/postgres.py index f3a6309..a94df90 100644 --- a/tests/postgres.py +++ b/tests/postgres.py @@ -9,7 +9,6 @@ config.dbengine = 'postgresql+psycopg' config.dbname = 'testdb' config.dbuser = 'testuser' config.dbpassword = 'secret' -config.dbschema = 'testing' # PostgreSQL-specific settings from scopes.storage.db.postgres import StorageFactory diff --git a/tests/tlib.py b/tests/tlib.py index 692e070..a774356 100644 --- a/tests/tlib.py +++ b/tests/tlib.py @@ -50,7 +50,7 @@ def test_tracking(self, storage): self.assertEqual(n, 1) self.assertEqual(tracks.get(31), None) - storage.db.commit(storage.session) + storage.commit() def test_folder(self, storage): @@ -66,5 +66,5 @@ def test_folder(self, storage): self.assertEqual(ch1.parent, top.rid) self.assertEqual(list(top.keys()), ['child1']) - storage.db.commit(storage.session) + storage.commit()