cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make job processing and export more informative

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@3653 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2009-12-14 15:47:01 +00:00
parent 504092b268
commit f9ac640529
6 changed files with 56 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
external/base.py vendored
View file

@ -92,6 +92,8 @@ class Extractor(Base):
implements(IExtractor)
count = 0
def extract(self):
return itertools.chain(self.extractTypes(),
self.extractConcepts(),
@ -107,25 +109,30 @@ class Extractor(Base):
data = self.getObjectData(obj)
element = typeElement(getName(obj), obj.title, **data)
self.provideSubElements(obj, element)
self.count += 1
yield element
def extractConcepts(self):
for name, obj in self.concepts.items():
if obj.conceptType != self.typeConcept:
self.count += 1
yield self.getConceptElement(name, obj)
def extractResources(self):
for name, obj in self.resources.items():
self.count += 1
yield self.getResourceElement(name, obj)
def extractChildren(self):
for c in self.concepts.values():
for r in self.getChildRelations(c):
self.count += 1
yield r
def extractResourceRelations(self):
for c in self.concepts.values():
for r in self.getResourceRelations(c):
self.count += 1
yield r
def extractNodes(self, parent=None, path=''):
@ -144,10 +151,12 @@ class Extractor(Base):
or elementTypes['node'])
elem = elementClass(name, obj.title, path, obj.nodeType, **data)
self.provideSubElements(obj, elem)
self.count += 1
yield elem
childPath = path and '/'.join((path, name)) or name
for elem in self.extractNodes(obj, childPath):
#self.provideSubElements(obj, elem)
self.count += 1
yield elem
def extractForParents(self, parents, predicates=None,
@ -158,10 +167,12 @@ class Extractor(Base):
conceptList = sorted(concepts, key=lambda x:
(x.conceptType != self.typeConcept, getName(x)))
for c in conceptList:
self.count += 1
yield self.getConceptElement(getName(c), c)
for c in conceptList:
for r in c.getChildRelations(predicates):
if r.predicate != self.typePredicate and r.second in concepts:
self.count += 1
yield self.getChildElement(r)
if includeResources:
resources = set()
@ -169,10 +180,12 @@ class Extractor(Base):
for obj in c.getResources(predicates):
if obj not in resources:
resources.add(obj)
self.count += 1
yield self.getResourceElement(getName(obj), obj)
for c in conceptList:
for r in c.getResourceRelations(predicates):
if r.predicate != self.typePredicate and r.second in resources:
self.count += 1
yield self.getResourceRelationElement(r)
def collectConcepts(self, concept, predicates, includeSubconcepts, concepts):
@ -216,6 +229,7 @@ class Extractor(Base):
def getChildRelations(self, c, predicates=None):
for r in c.getChildRelations(predicates):
if r.predicate != self.typePredicate:
self.count += 1
yield self.getChildElement(r)
def getChildElement(self, r):
@ -229,6 +243,7 @@ class Extractor(Base):
def getResourceRelations(self, c, predicates=None):
for r in c.getResourceRelations(predicates):
if r.predicate != self.typePredicate:
self.count += 1
yield self.getResourceRelationElement(r)
def getResourceRelationElement(self, r):

3
organize/job/README.txt
View file

@ -40,6 +40,7 @@ method. As we haven't yet defined any job managers nothing happens.
>>> executor = Executor(loopsRoot, TestRequest())
>>> executor.processJobs()
'No job managers available.'
We now register a job manager via an options setting on the loops root object.
As the corresponding job manager is not yet defined an registered a
@ -47,7 +48,7 @@ warning is issued.
>>> loopsRoot.options = ['organize.job.managers:loops_notifier']
>>> executor = Executor(loopsRoot, TestRequest())
>>> executor.processJobs()
>>> r = executor.processJobs()
Job manager 'loops_notifier' not found.
So let's now define a job manager class and register it as an adapter for

17
organize/job/browser.py
View file

@ -1,5 +1,5 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
# Copyright (c) 2009 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -48,9 +48,18 @@ class Executor(object):
return getLogger('loops.organize.job')
def processJobs(self):
for name in self.options('organize.job.managers', []):
output = []
names = [n for n in self.request.get('job_managers', '').split(',') if n]
if not names:
names = self.options('organize.job.managers', [])
for name in names:
manager = component.queryAdapter(self.context, IJobManager, name=name)
if manager is None:
self.logger.warn("Job manager '%s' not found." % name)
msg = "Job manager '%s' not found." % name
self.logger.warn(msg)
output.append(msg)
else:
manager.process()
output.append(manager.process())
if not output:
return 'No job managers available.'
return '\n'.join(m for m in output if m)

1
organize/tracking/README.txt
View file

@ -126,6 +126,7 @@ of job control.
>>> rm = AccessRecordManager(loopsRoot)
>>> rm.baseDir = testDir
>>> rm.loadRecordsFromLog()
'AccessRecordManager: 2 records loaded.'
>>> len(access)
2

35
organize/tracking/access.py
View file

@ -1,5 +1,5 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
# Copyright (c) 2009 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@ -105,7 +105,7 @@ class AccessRecordManager(BaseRecordManager, JobManager):
self.baseDir = util.getVarDirectory()
def process(self):
self.loadRecordsFromLog()
return self.loadRecordsFromLog()
@Lazy
def logfile(self):
@ -116,9 +116,14 @@ class AccessRecordManager(BaseRecordManager, JobManager):
def valid(self):
return self.storage is not None and self.logfile
@Lazy
def log(self):
return logging.getLogger('AccessRecordManager')
def loadRecordsFromLog(self):
if not self.valid:
return
return 'AccessRecordManager: Feature not available.'
count = 0
fn = self.logfile
path = os.path.join(self.baseDir, fn)
logger = loggers.get(fn)
@ -128,27 +133,28 @@ class AccessRecordManager(BaseRecordManager, JobManager):
return
lf = open(path, 'r')
for idx, line in enumerate(lf):
self.processLogRecord(idx, line)
if self.processLogRecord(idx, line):
count += 1
lf.close()
transaction.commit()
logger.doRollover()
self.log.info('%i records loaded.' % count)
return 'AccessRecordManager: %i records loaded.' % count
def processLogRecord(self, idx, line):
if not line:
return
return False
values = line.split(';')
timeString = values.pop(0)
version = values.pop(0)
if version not in fields:
logging.getLogger('AccessRecordManager').warn(
'Undefined logging record version %r on record %i.'
% (version, idx))
return
self.log.warn('Undefined logging record version %r on record %i.'
% (version, idx))
return False
if len(values) != len(fields[version]):
logging.getLogger('AccessRecordManager').warn(
'Length of record %i does not match version %r.'
% (idx, version))
return
self.log.warn('Length of record %i does not match version %r.'
% (idx, version))
return False
data = {}
for idx, field in enumerate(fields[version]):
data[field] = values[idx]
@ -159,9 +165,10 @@ class AccessRecordManager(BaseRecordManager, JobManager):
timeStamp=timeStamp)
for track in existing:
if track.data == data: # has been recorded already
return
return False
self.storage.saveUserTrack(taskId, 0, personId, data,
timeStamp=timeStamp)
return True
class AccessRecordManagerView(AccessRecordManager):

5
tests/setup.py
View file

@ -35,6 +35,7 @@ from cybertools.composer.schema.field import FieldInstance, NumberFieldInstance
from cybertools.composer.schema.field import DateFieldInstance, BooleanFieldInstance
from cybertools.composer.schema.field import EmailFieldInstance, ListFieldInstance
from cybertools.composer.schema.field import FileUploadFieldInstance
from cybertools.composer.schema.grid.field import RecordsFieldInstance
from cybertools.composer.schema.instance import Instance, Editor
from cybertools.relation.tests import IntIdsStub
from cybertools.relation.registry import RelationRegistry, IIndexableRelation
@ -70,7 +71,7 @@ from loops.resource import Document, MediaAsset
from loops.resource import IndexAttributes as ResourceIndexAttributes
from loops.schema.factory import ResourceSchemaFactory, FileSchemaFactory, \
NoteSchemaFactory
from loops.schema.field import RelationSetFieldInstance
from loops.schema.field import RelationFieldInstance, RelationSetFieldInstance
from loops.security.common import grantAcquiredSecurity, revokeAcquiredSecurity
from loops.security.policy import LoopsSecurityPolicy
from loops.security.setter import BaseSecuritySetter
@ -152,6 +153,8 @@ class TestSite(object):
component.provideAdapter(BooleanFieldInstance, name='boolean')
component.provideAdapter(ListFieldInstance, name='list')
component.provideAdapter(FileUploadFieldInstance, name='fileupload')
component.provideAdapter(RecordsFieldInstance, name='records')
component.provideAdapter(RelationFieldInstance, name='relation')
component.provideAdapter(RelationSetFieldInstance, name='relationset')
component.provideAdapter(SchemaFactory)
component.provideAdapter(ResourceSchemaFactory)