cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

provide commits and logging during update of external collection

Browse source
This commit is contained in:
Helmut Merz 2015-07-25 11:16:26 +02:00
parent 69e529ce39
commit 08d8993e5a
1 changed files with 15 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
integrator/collection.py
View file

@ -24,6 +24,7 @@ file system.
from datetime import datetime from datetime import datetime
from logging import getLogger from logging import getLogger
import os, re, stat import os, re, stat
import transaction
from zope.app.container.interfaces import INameChooser from zope.app.container.interfaces import INameChooser
from zope.app.container.contained import ObjectRemovedEvent from zope.app.container.contained import ObjectRemovedEvent
@ -67,6 +68,7 @@ class ExternalCollectionAdapter(AdapterBase):
newResources = None newResources = None
updateMessage = None updateMessage = None
logger = getLogger('loops.integrator.collection')
def getExclude(self): def getExclude(self):
return getattr(self.context, '_exclude', None) or [] return getattr(self.context, '_exclude', None) or []
@ -88,6 +90,7 @@ class ExternalCollectionAdapter(AdapterBase):
provider = component.getUtility(IExternalCollectionProvider, provider = component.getUtility(IExternalCollectionProvider,
name=self.providerName or '') name=self.providerName or '')
#print '*** old', old, versions, self.lastUpdated #print '*** old', old, versions, self.lastUpdated
changeCount = 0
for addr, mdate in provider.collect(self): for addr, mdate in provider.collect(self):
#print '***', addr, mdate #print '***', addr, mdate
if addr in versions: if addr in versions:
@ -97,6 +100,7 @@ class ExternalCollectionAdapter(AdapterBase):
# for checking for changes... # for checking for changes...
oldFound.append(addr) oldFound.append(addr)
if self.lastUpdated is None or (mdate and mdate > self.lastUpdated): if self.lastUpdated is None or (mdate and mdate > self.lastUpdated):
changeCount +=1
obj = old[addr] obj = old[addr]
# update settings and regenerate scale variant for media asset # update settings and regenerate scale variant for media asset
adobj = adapted(obj) adobj = adapted(obj)
@ -111,31 +115,40 @@ class ExternalCollectionAdapter(AdapterBase):
self.updateMessage = message self.updateMessage = message
# force reindexing # force reindexing
notify(ObjectModifiedEvent(obj)) notify(ObjectModifiedEvent(obj))
if changeCount % 100 == 0:
self.logger.info('Updated: %i.' % changeCount)
transaction.commit()
else: else:
new.append(addr) new.append(addr)
self.logger.info('%i objects updated.' % changeCount)
transaction.commit()
if new: if new:
self.newResources = provider.createExtFileObjects(self, new) self.newResources = provider.createExtFileObjects(self, new)
for r in self.newResources: for r in self.newResources:
self.context.assignResource(r) self.context.assignResource(r)
self.logger.info('%i objects created.' % len(new))
transaction.commit()
for addr in old: for addr in old:
if str(addr) not in oldFound: if str(addr) not in oldFound:
# not part of the collection any more # not part of the collection any more
# TODO: only remove from collection but keep object? # TODO: only remove from collection but keep object?
self.remove(old[addr]) self.remove(old[addr])
transaction.commit()
for r in self.context.getResources(): for r in self.context.getResources():
adobj = adapted(r) adobj = adapted(r)
if self.metaInfo != adobj.metaInfo and ( if self.metaInfo != adobj.metaInfo and (
not adobj.metaInfo or self.overwriteMetaInfo): not adobj.metaInfo or self.overwriteMetaInfo):
adobj.metaInfo = self.metaInfo adobj.metaInfo = self.metaInfo
self.lastUpdated = datetime.today() self.lastUpdated = datetime.today()
self.logger.info('External collection updated.')
transaction.commit()
def clear(self): def clear(self):
for obj in self.context.getResources(): for obj in self.context.getResources():
self.remove(obj) self.remove(obj)
def remove(self, obj): def remove(self, obj):
getLogger('loops.integrator.collection').info( self.logger.info('Removing object: %s.' % getName(obj))
'Removing object: %s.' % getName(obj))
del self.resourceManager[getName(obj)] del self.resourceManager[getName(obj)]
notify(ObjectRemovedEvent(obj)) notify(ObjectRemovedEvent(obj))