cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

more intermediate commits + logging

Browse source
This commit is contained in:
Helmut Merz 2015-07-25 13:23:10 +02:00
parent 0ec131e1a8
commit 11123b16fa
1 changed files with 15 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
integrator/collection.py
View file

@ -53,6 +53,8 @@ from loops.versioning.interfaces import IVersionable
TypeInterfaceSourceList.typeInterfaces += (IExternalCollection,) TypeInterfaceSourceList.typeInterfaces += (IExternalCollection,)
logger = getLogger('loops.integrator.collection')
class ExternalCollectionAdapter(AdapterBase): class ExternalCollectionAdapter(AdapterBase):
""" A concept adapter for accessing an external collection. """ A concept adapter for accessing an external collection.
@ -68,8 +70,7 @@ class ExternalCollectionAdapter(AdapterBase):
newResources = None newResources = None
updateMessage = None updateMessage = None
logger = getLogger('loops.integrator.collection')
def getExclude(self): def getExclude(self):
return getattr(self.context, '_exclude', None) or [] return getattr(self.context, '_exclude', None) or []
def setExclude(self, value): def setExclude(self, value):
@ -115,18 +116,18 @@ class ExternalCollectionAdapter(AdapterBase):
self.updateMessage = message self.updateMessage = message
# force reindexing # force reindexing
notify(ObjectModifiedEvent(obj)) notify(ObjectModifiedEvent(obj))
if changeCount % 100 == 0: if changeCount % 10 == 0:
self.logger.info('Updated: %i.' % changeCount) logger.info('Updated: %i.' % changeCount)
transaction.commit() transaction.commit()
else: else:
new.append(addr) new.append(addr)
self.logger.info('%i objects updated.' % changeCount) logger.info('%i objects updated.' % changeCount)
transaction.commit() transaction.commit()
if new: if new:
self.newResources = provider.createExtFileObjects(self, new) self.newResources = provider.createExtFileObjects(self, new)
for r in self.newResources: for r in self.newResources:
self.context.assignResource(r) self.context.assignResource(r)
self.logger.info('%i objects created.' % len(new)) logger.info('%i objects created.' % len(new))
transaction.commit() transaction.commit()
for addr in old: for addr in old:
if str(addr) not in oldFound: if str(addr) not in oldFound:
@ -140,7 +141,7 @@ class ExternalCollectionAdapter(AdapterBase):
not adobj.metaInfo or self.overwriteMetaInfo): not adobj.metaInfo or self.overwriteMetaInfo):
adobj.metaInfo = self.metaInfo adobj.metaInfo = self.metaInfo
self.lastUpdated = datetime.today() self.lastUpdated = datetime.today()
self.logger.info('External collection updated.') logger.info('External collection updated.')
transaction.commit() transaction.commit()
def clear(self): def clear(self):
@ -148,7 +149,7 @@ class ExternalCollectionAdapter(AdapterBase):
self.remove(obj) self.remove(obj)
def remove(self, obj): def remove(self, obj):
self.logger.info('Removing object: %s.' % getName(obj)) logger.info('Removing object: %s.' % getName(obj))
del self.resourceManager[getName(obj)] del self.resourceManager[getName(obj)]
notify(ObjectRemovedEvent(obj)) notify(ObjectRemovedEvent(obj))
@ -204,7 +205,7 @@ class DirectoryCollectionProvider(object):
for k, v in self.extFileTypeMapping.items()) for k, v in self.extFileTypeMapping.items())
container = client.context.getLoopsRoot().getResourceManager() container = client.context.getLoopsRoot().getResourceManager()
directory = self.getDirectory(client) directory = self.getDirectory(client)
for addr in addresses: for idx, addr in enumerate(addresses):
name = self.generateName(container, addr) name = self.generateName(container, addr)
title = self.generateTitle(addr) title = self.generateTitle(addr)
contentType = guess_content_type(addr, contentType = guess_content_type(addr,
@ -217,9 +218,8 @@ class DirectoryCollectionProvider(object):
if extFileType is None: if extFileType is None:
extFileType = extFileTypes['image/*'] extFileType = extFileTypes['image/*']
if extFileType is None: if extFileType is None:
getLogger('loops.integrator.collection.DirectoryCollectionProvider' logger.warn('No external file type found for %r, '
).warn('No external file type found for %r, ' 'content type: %r' % (name, contentType))
'content type: %r' % (name, contentType))
obj = addAndConfigureObject( obj = addAndConfigureObject(
container, Resource, name, container, Resource, name,
title=title, title=title,
@ -236,6 +236,9 @@ class DirectoryCollectionProvider(object):
message = client.updateMessage or u'' message = client.updateMessage or u''
message += u'<br />'.join(adobj.processingErrors) message += u'<br />'.join(adobj.processingErrors)
client.updateMessage = message client.updateMessage = message
if idx and idx % 10 == 0:
logger.info('Created: %i.' % idx)
transaction.commit()
yield obj yield obj
def getDirectory(self, client): def getDirectory(self, client):