From f9ac64052905513e2223304fca41f332310275e2 Mon Sep 17 00:00:00 2001 From: helmutm Date: Mon, 14 Dec 2009 15:47:01 +0000 Subject: [PATCH] make job processing and export more informative git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@3653 fd906abe-77d9-0310-91a1-e0d9ade77398 --- external/base.py | 15 +++++++++++++++ organize/job/README.txt | 3 ++- organize/job/browser.py | 17 +++++++++++++---- organize/tracking/README.txt | 1 + organize/tracking/access.py | 35 +++++++++++++++++++++-------------- tests/setup.py | 5 ++++- 6 files changed, 56 insertions(+), 20 deletions(-) diff --git a/external/base.py b/external/base.py index a061b60..aebab2c 100644 --- a/external/base.py +++ b/external/base.py @@ -92,6 +92,8 @@ class Extractor(Base): implements(IExtractor) + count = 0 + def extract(self): return itertools.chain(self.extractTypes(), self.extractConcepts(), @@ -107,25 +109,30 @@ class Extractor(Base): data = self.getObjectData(obj) element = typeElement(getName(obj), obj.title, **data) self.provideSubElements(obj, element) + self.count += 1 yield element def extractConcepts(self): for name, obj in self.concepts.items(): if obj.conceptType != self.typeConcept: + self.count += 1 yield self.getConceptElement(name, obj) def extractResources(self): for name, obj in self.resources.items(): + self.count += 1 yield self.getResourceElement(name, obj) def extractChildren(self): for c in self.concepts.values(): for r in self.getChildRelations(c): + self.count += 1 yield r def extractResourceRelations(self): for c in self.concepts.values(): for r in self.getResourceRelations(c): + self.count += 1 yield r def extractNodes(self, parent=None, path=''): @@ -144,10 +151,12 @@ class Extractor(Base): or elementTypes['node']) elem = elementClass(name, obj.title, path, obj.nodeType, **data) self.provideSubElements(obj, elem) + self.count += 1 yield elem childPath = path and '/'.join((path, name)) or name for elem in self.extractNodes(obj, childPath): #self.provideSubElements(obj, elem) + self.count += 1 yield elem def extractForParents(self, parents, predicates=None, @@ -158,10 +167,12 @@ class Extractor(Base): conceptList = sorted(concepts, key=lambda x: (x.conceptType != self.typeConcept, getName(x))) for c in conceptList: + self.count += 1 yield self.getConceptElement(getName(c), c) for c in conceptList: for r in c.getChildRelations(predicates): if r.predicate != self.typePredicate and r.second in concepts: + self.count += 1 yield self.getChildElement(r) if includeResources: resources = set() @@ -169,10 +180,12 @@ class Extractor(Base): for obj in c.getResources(predicates): if obj not in resources: resources.add(obj) + self.count += 1 yield self.getResourceElement(getName(obj), obj) for c in conceptList: for r in c.getResourceRelations(predicates): if r.predicate != self.typePredicate and r.second in resources: + self.count += 1 yield self.getResourceRelationElement(r) def collectConcepts(self, concept, predicates, includeSubconcepts, concepts): @@ -216,6 +229,7 @@ class Extractor(Base): def getChildRelations(self, c, predicates=None): for r in c.getChildRelations(predicates): if r.predicate != self.typePredicate: + self.count += 1 yield self.getChildElement(r) def getChildElement(self, r): @@ -229,6 +243,7 @@ class Extractor(Base): def getResourceRelations(self, c, predicates=None): for r in c.getResourceRelations(predicates): if r.predicate != self.typePredicate: + self.count += 1 yield self.getResourceRelationElement(r) def getResourceRelationElement(self, r): diff --git a/organize/job/README.txt b/organize/job/README.txt index f6d99c7..cdc3122 100644 --- a/organize/job/README.txt +++ b/organize/job/README.txt @@ -40,6 +40,7 @@ method. As we haven't yet defined any job managers nothing happens. >>> executor = Executor(loopsRoot, TestRequest()) >>> executor.processJobs() + 'No job managers available.' We now register a job manager via an options setting on the loops root object. As the corresponding job manager is not yet defined an registered a @@ -47,7 +48,7 @@ warning is issued. >>> loopsRoot.options = ['organize.job.managers:loops_notifier'] >>> executor = Executor(loopsRoot, TestRequest()) - >>> executor.processJobs() + >>> r = executor.processJobs() Job manager 'loops_notifier' not found. So let's now define a job manager class and register it as an adapter for diff --git a/organize/job/browser.py b/organize/job/browser.py index d800d51..4f54043 100644 --- a/organize/job/browser.py +++ b/organize/job/browser.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# Copyright (c) 2009 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,9 +48,18 @@ class Executor(object): return getLogger('loops.organize.job') def processJobs(self): - for name in self.options('organize.job.managers', []): + output = [] + names = [n for n in self.request.get('job_managers', '').split(',') if n] + if not names: + names = self.options('organize.job.managers', []) + for name in names: manager = component.queryAdapter(self.context, IJobManager, name=name) if manager is None: - self.logger.warn("Job manager '%s' not found." % name) + msg = "Job manager '%s' not found." % name + self.logger.warn(msg) + output.append(msg) else: - manager.process() + output.append(manager.process()) + if not output: + return 'No job managers available.' + return '\n'.join(m for m in output if m) diff --git a/organize/tracking/README.txt b/organize/tracking/README.txt index 559f315..ddfecd0 100644 --- a/organize/tracking/README.txt +++ b/organize/tracking/README.txt @@ -126,6 +126,7 @@ of job control. >>> rm = AccessRecordManager(loopsRoot) >>> rm.baseDir = testDir >>> rm.loadRecordsFromLog() + 'AccessRecordManager: 2 records loaded.' >>> len(access) 2 diff --git a/organize/tracking/access.py b/organize/tracking/access.py index fe8e801..1be9efa 100644 --- a/organize/tracking/access.py +++ b/organize/tracking/access.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# Copyright (c) 2009 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -105,7 +105,7 @@ class AccessRecordManager(BaseRecordManager, JobManager): self.baseDir = util.getVarDirectory() def process(self): - self.loadRecordsFromLog() + return self.loadRecordsFromLog() @Lazy def logfile(self): @@ -116,9 +116,14 @@ class AccessRecordManager(BaseRecordManager, JobManager): def valid(self): return self.storage is not None and self.logfile + @Lazy + def log(self): + return logging.getLogger('AccessRecordManager') + def loadRecordsFromLog(self): if not self.valid: - return + return 'AccessRecordManager: Feature not available.' + count = 0 fn = self.logfile path = os.path.join(self.baseDir, fn) logger = loggers.get(fn) @@ -128,27 +133,28 @@ class AccessRecordManager(BaseRecordManager, JobManager): return lf = open(path, 'r') for idx, line in enumerate(lf): - self.processLogRecord(idx, line) + if self.processLogRecord(idx, line): + count += 1 lf.close() transaction.commit() logger.doRollover() + self.log.info('%i records loaded.' % count) + return 'AccessRecordManager: %i records loaded.' % count def processLogRecord(self, idx, line): if not line: - return + return False values = line.split(';') timeString = values.pop(0) version = values.pop(0) if version not in fields: - logging.getLogger('AccessRecordManager').warn( - 'Undefined logging record version %r on record %i.' - % (version, idx)) - return + self.log.warn('Undefined logging record version %r on record %i.' + % (version, idx)) + return False if len(values) != len(fields[version]): - logging.getLogger('AccessRecordManager').warn( - 'Length of record %i does not match version %r.' - % (idx, version)) - return + self.log.warn('Length of record %i does not match version %r.' + % (idx, version)) + return False data = {} for idx, field in enumerate(fields[version]): data[field] = values[idx] @@ -159,9 +165,10 @@ class AccessRecordManager(BaseRecordManager, JobManager): timeStamp=timeStamp) for track in existing: if track.data == data: # has been recorded already - return + return False self.storage.saveUserTrack(taskId, 0, personId, data, timeStamp=timeStamp) + return True class AccessRecordManagerView(AccessRecordManager): diff --git a/tests/setup.py b/tests/setup.py index f24be15..747ec7f 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -35,6 +35,7 @@ from cybertools.composer.schema.field import FieldInstance, NumberFieldInstance from cybertools.composer.schema.field import DateFieldInstance, BooleanFieldInstance from cybertools.composer.schema.field import EmailFieldInstance, ListFieldInstance from cybertools.composer.schema.field import FileUploadFieldInstance +from cybertools.composer.schema.grid.field import RecordsFieldInstance from cybertools.composer.schema.instance import Instance, Editor from cybertools.relation.tests import IntIdsStub from cybertools.relation.registry import RelationRegistry, IIndexableRelation @@ -70,7 +71,7 @@ from loops.resource import Document, MediaAsset from loops.resource import IndexAttributes as ResourceIndexAttributes from loops.schema.factory import ResourceSchemaFactory, FileSchemaFactory, \ NoteSchemaFactory -from loops.schema.field import RelationSetFieldInstance +from loops.schema.field import RelationFieldInstance, RelationSetFieldInstance from loops.security.common import grantAcquiredSecurity, revokeAcquiredSecurity from loops.security.policy import LoopsSecurityPolicy from loops.security.setter import BaseSecuritySetter @@ -152,6 +153,8 @@ class TestSite(object): component.provideAdapter(BooleanFieldInstance, name='boolean') component.provideAdapter(ListFieldInstance, name='list') component.provideAdapter(FileUploadFieldInstance, name='fileupload') + component.provideAdapter(RecordsFieldInstance, name='records') + component.provideAdapter(RelationFieldInstance, name='relation') component.provideAdapter(RelationSetFieldInstance, name='relationset') component.provideAdapter(SchemaFactory) component.provideAdapter(ResourceSchemaFactory)