From 382ffe2e28d9ece6ad44eae2de56abb5e49b00cb Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Mon, 11 Dec 2023 08:24:54 +0100 Subject: [PATCH 1/8] add MIT license file as default for new or changed modules --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1ae14e4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (C) 2023 cyberconcepts.org team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d6666c166f45e6a88783583bb98b54fe3016bc24 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Mon, 11 Dec 2023 09:46:27 +0100 Subject: [PATCH 2/8] first migration (with favorites) basically working --- .gitignore | 1 + tracking/migration.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tracking/migration.py diff --git a/.gitignore b/.gitignore index 41ef9e2..7ff38ab 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ajax/dojo/* build/ dist/ +*.swp *.egg-info *.project *.pydevproject diff --git a/tracking/migration.py b/tracking/migration.py new file mode 100644 index 0000000..2f02ea5 --- /dev/null +++ b/tracking/migration.py @@ -0,0 +1,40 @@ +# cybertools.tracking.migration + +"""Tools for migration ZODB-/BTree-based tracks to SQL-base records.""" + +from datetime import datetime +import time + +import config +from cco.storage.common import Context, getEngine +from cco.storage.tracking import record +from loops.config.base import LoopsOptions + + +def migrate(loopsRoot, recFolderName, storageFactory=record.Storage): + rf = loopsRoot.getRecordManager().get(recFolderName) + if rf is None: + print('*** ERROR: folder %r not found!' % recFolderName) + return + options = LoopsOptions(loopsRoot) + print('*** database:', config.dbname, config.dbuser, config.dbpassword) + schema = options('cco.storage.schema') or None + if schema is not None: + schema = schema[0] + print('*** schema:', schema) + context = Context(getEngine(config.dbengine, config.dbname, + config.dbuser, config.dbpassword, + host=config.dbhost, port=config.dbport), + schema=schema) + storage = storageFactory(context) + for id, inTrack in rf.items(): + #ts = time.mktime(inTrack.timeStamp.timetuple()) + ts = datetime.fromtimestamp(inTrack.timeStamp) + print('*** in:', id, inTrack) + head = [inTrack.metadata[k] for k in storage.trackFactory.headFields] + print('*** out:', head, ts) + track = storage.trackFactory(*head, trackId=int(id), + timeStamp=ts, data=inTrack.data) + storage.upsert(track) + + From 6a52601060c80f14d2f69c7f1a80614225aec2ed Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Fri, 15 Dec 2023 14:03:35 +0100 Subject: [PATCH 3/8] move tracking migration to loops --- tracking/migration.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 tracking/migration.py diff --git a/tracking/migration.py b/tracking/migration.py deleted file mode 100644 index 2f02ea5..0000000 --- a/tracking/migration.py +++ /dev/null @@ -1,40 +0,0 @@ -# cybertools.tracking.migration - -"""Tools for migration ZODB-/BTree-based tracks to SQL-base records.""" - -from datetime import datetime -import time - -import config -from cco.storage.common import Context, getEngine -from cco.storage.tracking import record -from loops.config.base import LoopsOptions - - -def migrate(loopsRoot, recFolderName, storageFactory=record.Storage): - rf = loopsRoot.getRecordManager().get(recFolderName) - if rf is None: - print('*** ERROR: folder %r not found!' % recFolderName) - return - options = LoopsOptions(loopsRoot) - print('*** database:', config.dbname, config.dbuser, config.dbpassword) - schema = options('cco.storage.schema') or None - if schema is not None: - schema = schema[0] - print('*** schema:', schema) - context = Context(getEngine(config.dbengine, config.dbname, - config.dbuser, config.dbpassword, - host=config.dbhost, port=config.dbport), - schema=schema) - storage = storageFactory(context) - for id, inTrack in rf.items(): - #ts = time.mktime(inTrack.timeStamp.timetuple()) - ts = datetime.fromtimestamp(inTrack.timeStamp) - print('*** in:', id, inTrack) - head = [inTrack.metadata[k] for k in storage.trackFactory.headFields] - print('*** out:', head, ts) - track = storage.trackFactory(*head, trackId=int(id), - timeStamp=ts, data=inTrack.data) - storage.upsert(track) - - From abfd75a782e64082fa218874ba7848d073839160 Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Thu, 21 Dec 2023 08:18:49 +0100 Subject: [PATCH 4/8] date2TimeStamp(): accept also int and float as input --- util/date.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/date.py b/util/date.py index f614d35..99328ab 100644 --- a/util/date.py +++ b/util/date.py @@ -32,6 +32,8 @@ def getTimeStamp(): return int(time.time()) def date2TimeStamp(d): + if isinstance(d, (int, float)): + return int(d) return int(time.mktime(d.timetuple())) From e5f1be9f916a6c5a92e9985b7f381da0228e345d Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Mon, 25 Dec 2023 10:52:45 +0100 Subject: [PATCH 5/8] provide method for automatic conversion from KeyTable to Records field --- composer/schema/grid/field.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/composer/schema/grid/field.py b/composer/schema/grid/field.py index be1c980..e256a1e 100644 --- a/composer/schema/grid/field.py +++ b/composer/schema/grid/field.py @@ -175,6 +175,12 @@ class RecordsFieldInstance(GridFieldInstance): def getRenderer(self, name): return grid_macros.macros.get(name) + def fromKeyTable(self, value): + fis = self.columnFieldInstances + return [dict([(fis[0].name, k)] + + [(fi.name, v[i]) for i, v in enumerate(fis[1:])]) + for k, v in value.items()] + def marshall(self, value): result = [] value = value or [] From 94ce64ef00e449d6d19c2ede79153d1231a9352d Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Thu, 28 Dec 2023 09:09:00 +0100 Subject: [PATCH 6/8] add RecordsTable field - context-based, i.e. the columns are defined in the object or type --- composer/schema/grid/field.py | 7 +++++++ composer/schema/grid/interfaces.py | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/composer/schema/grid/field.py b/composer/schema/grid/field.py index e256a1e..053be8f 100644 --- a/composer/schema/grid/field.py +++ b/composer/schema/grid/field.py @@ -282,6 +282,13 @@ class KeyTableFieldInstance(RecordsFieldInstance): pass +class ContextBasedRecordsFieldInstance(RecordsFieldInstance): + + @Lazy + def columnTypes(self): + obj = self.clientInstance.context + return [Field(name) for name in obj.columnNames] + class ContextBasedKeyTableFieldInstance(KeyTableFieldInstance): @Lazy diff --git a/composer/schema/grid/interfaces.py b/composer/schema/grid/interfaces.py index bc521d9..5fc27e2 100644 --- a/composer/schema/grid/interfaces.py +++ b/composer/schema/grid/interfaces.py @@ -51,6 +51,16 @@ class Records(Grid): instanceName='records',)) +class RecordsTable(Grid): + + __typeInfo__ = ('recordstable', + FieldType('recordstable', 'recordstable', + u'A series of records or rows.', + displayRenderer='display_records', + inputRenderer='input_records', + instanceName='recordstable',)) + + class KeyTable(Grid): __typeInfo__ = ('keytable', From a6fb66319219c811d0c5de40a729b8e571e9d45e Mon Sep 17 00:00:00 2001 From: Helmut Merz Date: Wed, 3 Jan 2024 11:07:55 +0100 Subject: [PATCH 7/8] minor additions for compatibility with new cco.storage implementation --- tracking/btree.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tracking/btree.py b/tracking/btree.py index 3325325..9ad27f5 100644 --- a/tracking/btree.py +++ b/tracking/btree.py @@ -123,6 +123,7 @@ class TrackingStorage(BTreeContainer): runs = None # currently active runs finishedRuns = None # finished runs currentRuns = None # the currently active run for each task + storage = None # compatibility with new cco.storage def __init__(self, *args, **kw): trackFactory = kw.pop('trackFactory', None) @@ -221,6 +222,9 @@ class TrackingStorage(BTreeContainer): self.indexTrack(trackNum, track) return trackId + def setTrackData(self, track, data): + track.data = data # persistent track: data will be stored automatically + def updateTrack(self, track, data, overwrite=False): trackId = str(track.__name__) trackNum = int(trackId) From 35ab24a78a99e9388b9e2267c810ab9497c7bb6f Mon Sep 17 00:00:00 2001 From: helmutm Date: Fri, 3 May 2024 13:08:35 +0200 Subject: [PATCH 8/8] tracking: add updateIndex() method --- tracking/btree.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tracking/btree.py b/tracking/btree.py index 9ad27f5..6a68f65 100644 --- a/tracking/btree.py +++ b/tracking/btree.py @@ -100,6 +100,11 @@ class Track(Persistent): data.update(newData) self.data = data # record change + def updateIndex(self, **kw): + for k, v in kw.items(): + setattr(self, k, v) + getParent(self).indexTrack(0, self, k) + def __repr__(self): md = self.metadata md['timeStamp'] = timeStamp2ISO(md['timeStamp'])