migration: allow source list, source list slicing, commit after step records

storage/migration/tracking.py

@@ -14,11 +14,16 @@ from loops.storage.compat.common import Storage
 from loops import util
 
 
-def migrate(loopsRoot, recFolderName, factory=tracking.Container):
-    rf = loopsRoot.getRecordManager().get(recFolderName)
-    if rf is None:
-        print('*** ERROR: folder %r not found!' % recFolderName)
-        return
+def migrate(loopsRoot, source, factory=tracking.Container,
+            start=0, stop=None, step=10):
+    if isinstance(source, basestring):
+        rf = loopsRoot.getRecordManager().get(source)
+        if rf is None:
+            print('*** ERROR: folder %r not found!' % recFolderName)
+            return
+        items = rf.items()[start:stop]
+    else:
+        items = [(s.__name__, s) for s in source[start:stop]]
     options = LoopsOptions(loopsRoot)
     #print('*** database:', config.dbname, config.dbuser, config.dbpassword)
     schema = options('cco.storage.schema') or None
@@ -27,7 +32,7 @@ def migrate(loopsRoot, recFolderName, factory=tracking.Container):
     #print('*** schema:', schema)
     storage = Storage(schema=schema)
     container = storage.create(factory)
-    for id, inTrack in rf.items():
+    for ix, (id, inTrack) in enumerate(items):
         ts = datetime.fromtimestamp(inTrack.timeStamp)
         #print('*** in:', id, inTrack)
         head = [inTrack.metadata[k] for k in container.itemFactory.headFields]
@@ -37,5 +42,8 @@ def migrate(loopsRoot, recFolderName, factory=tracking.Container):
         container.upsert(track)
         ouid = util.getUidForObject(inTrack)
         storage.storeUid(ouid, track.uid)
+        if divmod(ix+1, step)[1] == 0:
+            print('*** migrated %d' % (ix + 1 + start))
+            transaction.commit()
     transaction.commit()
 

util.py

@@ -37,7 +37,6 @@ engine = getEngine(config.dbengine, config.dbname,
 cco.storage.common.engine = engine
 cco.storage.common.Session = sessionFactory(engine)
 
-
 renderingFactories = {
     'text/plain': 'zope.source.plaintext',
     'text/stx': 'zope.source.stx',
@@ -133,7 +132,7 @@ def records(context, name, factory):
     opts = IOptions(root)
     if name in (opts.cco.storage.records or []):
         schema = (opts.cco.storage.schema or [None])[0]
-        storage = Storage(                          schema=schema)
+        storage = Storage(schema=schema)
         cont = storage.create(factory)
     else:
         cont = root.getRecordManager().get(name)