make job processing and export more informative

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@3653 fd906abe-77d9-0310-91a1-e0d9ade77398

external/base.py

@@ -92,6 +92,8 @@ class Extractor(Base):
 
     implements(IExtractor)
 
+    count = 0
+
     def extract(self):
         return itertools.chain(self.extractTypes(),
                                self.extractConcepts(),
@@ -107,25 +109,30 @@ class Extractor(Base):
             data = self.getObjectData(obj)
             element = typeElement(getName(obj), obj.title, **data)
             self.provideSubElements(obj, element)
+            self.count += 1
             yield element
 
     def extractConcepts(self):
         for name, obj in self.concepts.items():
             if obj.conceptType != self.typeConcept:
+                self.count += 1
                 yield self.getConceptElement(name, obj)
 
     def extractResources(self):
         for name, obj in self.resources.items():
+            self.count += 1
             yield self.getResourceElement(name, obj)
 
     def extractChildren(self):
         for c in self.concepts.values():
             for r in self.getChildRelations(c):
+                self.count += 1
                 yield r
 
     def extractResourceRelations(self):
         for c in self.concepts.values():
             for r in self.getResourceRelations(c):
+                self.count += 1
                 yield r
 
     def extractNodes(self, parent=None, path=''):
@@ -144,10 +151,12 @@ class Extractor(Base):
                                 or elementTypes['node'])
             elem = elementClass(name, obj.title, path, obj.nodeType, **data)
             self.provideSubElements(obj, elem)
+            self.count += 1
             yield elem
             childPath = path and '/'.join((path, name)) or name
             for elem in self.extractNodes(obj, childPath):
                 #self.provideSubElements(obj, elem)
+                self.count += 1
                 yield elem
 
     def extractForParents(self, parents, predicates=None,
@@ -158,10 +167,12 @@ class Extractor(Base):
         conceptList = sorted(concepts, key=lambda x:
                                 (x.conceptType != self.typeConcept, getName(x)))
         for c in conceptList:
+            self.count += 1
             yield self.getConceptElement(getName(c), c)
         for c in conceptList:
             for r in c.getChildRelations(predicates):
                 if r.predicate != self.typePredicate and r.second in concepts:
+                    self.count += 1
                     yield self.getChildElement(r)
         if includeResources:
             resources = set()
@@ -169,10 +180,12 @@ class Extractor(Base):
                 for obj in c.getResources(predicates):
                     if obj not in resources:
                         resources.add(obj)
+                        self.count += 1
                         yield self.getResourceElement(getName(obj), obj)
             for c in conceptList:
                 for r in c.getResourceRelations(predicates):
                     if r.predicate != self.typePredicate and r.second in resources:
+                        self.count += 1
                         yield self.getResourceRelationElement(r)
 
     def collectConcepts(self, concept, predicates, includeSubconcepts, concepts):
@@ -216,6 +229,7 @@ class Extractor(Base):
     def getChildRelations(self, c, predicates=None):
         for r in c.getChildRelations(predicates):
             if r.predicate != self.typePredicate:
+                self.count += 1
                 yield self.getChildElement(r)
 
     def getChildElement(self, r):
@@ -229,6 +243,7 @@ class Extractor(Base):
     def getResourceRelations(self, c, predicates=None):
         for r in c.getResourceRelations(predicates):
             if r.predicate != self.typePredicate:
+                self.count += 1
                 yield self.getResourceRelationElement(r)
 
     def getResourceRelationElement(self, r):

organize/job/README.txt

@@ -40,6 +40,7 @@ method. As we haven't yet defined any job managers nothing happens.
 
   >>> executor = Executor(loopsRoot, TestRequest())
   >>> executor.processJobs()
+  'No job managers available.'
 
 We now register a job manager via an options setting on the loops root object.
 As the corresponding job manager is not yet defined an registered a
@@ -47,7 +48,7 @@ warning is issued.
 
   >>> loopsRoot.options = ['organize.job.managers:loops_notifier']
   >>> executor = Executor(loopsRoot, TestRequest())
-  >>> executor.processJobs()
+  >>> r = executor.processJobs()
   Job manager 'loops_notifier' not found.
 
 So let's now define a job manager class and register it as an adapter for

organize/job/browser.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -48,9 +48,18 @@ class Executor(object):
         return getLogger('loops.organize.job')
 
     def processJobs(self):
-        for name in self.options('organize.job.managers', []):
+        output = []
+        names = [n for n in self.request.get('job_managers', '').split(',') if n]
+        if not names:
+            names = self.options('organize.job.managers', [])
+        for name in names:
             manager = component.queryAdapter(self.context, IJobManager, name=name)
             if manager is None:
-                self.logger.warn("Job manager '%s' not found." % name)
+                msg = "Job manager '%s' not found." % name
+                self.logger.warn(msg)
+                output.append(msg)
             else:
-                manager.process()
+                output.append(manager.process())
+        if not output:
+            return 'No job managers available.'
+        return '\n'.join(m for m in output if m)

organize/tracking/README.txt

@@ -126,6 +126,7 @@ of job control.
   >>> rm = AccessRecordManager(loopsRoot)
   >>> rm.baseDir = testDir
   >>> rm.loadRecordsFromLog()
+  'AccessRecordManager: 2 records loaded.'
   >>> len(access)
   2
 

organize/tracking/access.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2009 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -105,7 +105,7 @@ class AccessRecordManager(BaseRecordManager, JobManager):
         self.baseDir = util.getVarDirectory()
 
     def process(self):
-        self.loadRecordsFromLog()
+        return self.loadRecordsFromLog()
 
     @Lazy
     def logfile(self):
@@ -116,9 +116,14 @@ class AccessRecordManager(BaseRecordManager, JobManager):
     def valid(self):
         return self.storage is not None and self.logfile
 
+    @Lazy
+    def log(self):
+        return logging.getLogger('AccessRecordManager')
+
     def loadRecordsFromLog(self):
         if not self.valid:
-            return
+            return 'AccessRecordManager: Feature not available.'
+        count = 0
         fn = self.logfile
         path = os.path.join(self.baseDir, fn)
         logger = loggers.get(fn)
@@ -128,27 +133,28 @@ class AccessRecordManager(BaseRecordManager, JobManager):
             return
         lf = open(path, 'r')
         for idx, line in enumerate(lf):
-            self.processLogRecord(idx, line)
+            if self.processLogRecord(idx, line):
+                count += 1
         lf.close()
         transaction.commit()
         logger.doRollover()
+        self.log.info('%i records loaded.' % count)
+        return 'AccessRecordManager: %i records loaded.' % count
 
     def processLogRecord(self, idx, line):
         if not line:
-            return
+            return False
         values = line.split(';')
         timeString = values.pop(0)
         version = values.pop(0)
         if version not in fields:
-            logging.getLogger('AccessRecordManager').warn(
+            self.log.warn('Undefined logging record version %r on record %i.'
-                        'Undefined logging record version %r on record %i.'
                                     % (version, idx))
-            return
+            return False
         if len(values) != len(fields[version]):
-            logging.getLogger('AccessRecordManager').warn(
+            self.log.warn('Length of record %i does not match version %r.'
-                        'Length of record %i does not match version %r.'
                                     % (idx, version))
-            return
+            return False
         data = {}
         for idx, field in enumerate(fields[version]):
             data[field] = values[idx]
@@ -159,9 +165,10 @@ class AccessRecordManager(BaseRecordManager, JobManager):
                                       timeStamp=timeStamp)
         for track in existing:
             if track.data == data:  # has been recorded already
-                return
+                return False
         self.storage.saveUserTrack(taskId, 0, personId, data,
                                    timeStamp=timeStamp)
+        return True
 
 
 class AccessRecordManagerView(AccessRecordManager):

tests/setup.py

@@ -35,6 +35,7 @@ from cybertools.composer.schema.field import FieldInstance, NumberFieldInstance
 from cybertools.composer.schema.field import DateFieldInstance, BooleanFieldInstance
 from cybertools.composer.schema.field import EmailFieldInstance, ListFieldInstance
 from cybertools.composer.schema.field import FileUploadFieldInstance
+from cybertools.composer.schema.grid.field import RecordsFieldInstance
 from cybertools.composer.schema.instance import Instance, Editor
 from cybertools.relation.tests import IntIdsStub
 from cybertools.relation.registry import RelationRegistry, IIndexableRelation
@@ -70,7 +71,7 @@ from loops.resource import Document, MediaAsset
 from loops.resource import IndexAttributes as ResourceIndexAttributes
 from loops.schema.factory import ResourceSchemaFactory, FileSchemaFactory, \
                                  NoteSchemaFactory
-from loops.schema.field import RelationSetFieldInstance
+from loops.schema.field import RelationFieldInstance, RelationSetFieldInstance
 from loops.security.common import grantAcquiredSecurity, revokeAcquiredSecurity
 from loops.security.policy import LoopsSecurityPolicy
 from loops.security.setter import BaseSecuritySetter
@@ -152,6 +153,8 @@ class TestSite(object):
         component.provideAdapter(BooleanFieldInstance, name='boolean')
         component.provideAdapter(ListFieldInstance, name='list')
         component.provideAdapter(FileUploadFieldInstance, name='fileupload')
+        component.provideAdapter(RecordsFieldInstance, name='records')
+        component.provideAdapter(RelationFieldInstance, name='relation')
         component.provideAdapter(RelationSetFieldInstance, name='relationset')
         component.provideAdapter(SchemaFactory)
         component.provideAdapter(ResourceSchemaFactory)