cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

loops.integrator: ExternalCollectionAdapter working

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1689 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2007-04-14 16:52:39 +00:00
parent 03021dc798
commit a0b113195d
6 changed files with 97 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.txt
View file

@ -688,7 +688,7 @@ and possibly critcal cases:
>>> nc = NameChooser(resources)
>>> nc.chooseName(u'', Resource(u'abc: (cde)'))
u'abc_cde'
u'abc__cde'
>>> nc.chooseName(u'', Resource(u'\xdcml\xe4ut'))
u'uemlaeut'
>>> nc.chooseName(u'', Resource(u'A very very loooooong title'))

24
common.py
View file

@ -108,29 +108,43 @@ class NameChooser(BaseNameChooser):
def chooseName(self, name, obj):
if not name:
name = self.generateName(obj)
name = self.generateNameFromTitle(obj)
else:
name = self.normalizeName(name)
name = super(NameChooser, self).chooseName(name, obj)
return name
def generateName(self, obj):
def generateNameFromTitle(self, obj):
title = obj.title
result = []
if len(title) > 15:
words = title.split()
if len(words) > 1:
title = '_'.join((words[0], words[-1]))
for c in title:
return self.normalizeName(title)
def normalizeName(self, baseName):
result = []
for c in baseName:
try:
c = c.encode('ISO8859-15')
except UnicodeEncodeError:
# skip all characters not representable in ISO encoding
continue
if c in '._':
# separator and special characters to keep
result.append(c)
continue
if c in self.specialCharacters:
# transform umlauts and other special characters
result.append(self.specialCharacters[c].lower())
continue
if ord(c) > 127:
# map to ASCII characters
c = chr(ord(c) & 127)
if c in ('_., '):
if c in ':,/\\ ':
# replace separator characters with _
result.append('_')
# skip all other characters
elif not c.isalpha() and not c.isdigit():
continue
else:

29
integrator/README.txt
View file

@ -54,7 +54,6 @@ to the external system:
>>> aColl01.baseAddress = dataDir
>>> aColl01.address = 'topics'
Directory Collection Provider
-----------------------------
@ -66,24 +65,42 @@ object, the external collection itself.
>>> dcp = DirectoryCollectionProvider()
>>> sorted(dcp.collect(aColl01))
['programming/BeautifulProgram.pdf', 'programming/zope/zope3.txt']
[('programming/BeautifulProgram.pdf', datetime.datetime(2005, 4, 7, 12, 36, 56)),
('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))]
If we provide a selective pattern we get only part of the files:
If we provide a more selective pattern we get only part of the files:
>>> aColl01.pattern = r'.*\.txt'
>>> sorted(dcp.collect(aColl01))
['programming/zope/zope3.txt']
[('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))]
Let's now create the corresponding resource objects.
>>> aColl01.pattern = ''
>>> addresses = dcp.collect(aColl01)
>>> addresses = [e[0] for e in dcp.collect(aColl01)]
>>> res = list(dcp.createExtFileObjects(aColl01, addresses))
>>> len(sorted(r.__name__ for r in res))
2
>>> xf1 = res[0]
>>> xf1.__name__
u'programming/BeautifulProgram.pdf'
u'programming_beautifulprogram.pdf'
>>> xf1.title
u'BeautifulProgram'
>>> for r in res:
... del resources[r.__name__]
Working with the External Collection
------------------------------------
>>> component.provideUtility(DirectoryCollectionProvider())
>>> aColl01.update()
>>> res = coll01.getResources()
>>> len(res)
2
>>> sorted((r.__name__, r.title) for r in res)
[(u'programming_beautifulprogram.pdf', u'BeautifulProgram'),
(u'programming_zope_zope3.txt', u'zope3')]
Fin de partie

57
integrator/collection.py
View file

@ -23,13 +23,20 @@ file system.
$Id$
"""
import os, re
from datetime import datetime
import os, re, stat
from zope import component
from zope.lifecycleevent import ObjectModifiedEvent
from zope.event import notify
from zope.app.container.interfaces import INameChooser
from zope.component import adapts
from zope.interface import implements, Attribute
from zope.cachedescriptors.property import Lazy
from zope.schema.interfaces import IField
from zope.traversing.api import getName, getParent
from cybertools.text import mimetypes
from cybertools.typology.interfaces import IType
from loops.common import AdapterBase
from loops.interfaces import IResource, IConcept
@ -54,11 +61,23 @@ class ExternalCollectionAdapter(AdapterBase):
_adapterAttributes = ('context', '__parent__',)
_contextAttributes = list(IExternalCollection) + list(IConcept)
def create(self):
pass
def update(self):
pass
existing = self.context.getResources()
old = dict((obj.externalAddress, obj) for obj in existing)
new = []
provider = component.getUtility(IExternalCollectionProvider,
name=self.providerName or '')
for addr, mdate in provider.collect(self):
if addr in old:
if mdate > self.lastUpdated:
notify(ObjectModifiedEvent(old[addr]))
else:
new.append(addr)
if new:
newResources = provider.createExtFileObjects(self, new)
for r in newResources:
self.context.assignResource(r)
self.lastUpdated = datetime.today()
class DirectoryCollectionProvider(object):
@ -75,19 +94,24 @@ class DirectoryCollectionProvider(object):
del dirs[dirs.index('.svn')]
for f in files:
if pattern.match(f):
yield os.path.join(path[len(directory)+1:], f)
# may be it would be better to return a file's hash
# for checking for changes...
mtime = os.stat(os.path.join(path, f))[stat.ST_MTIME]
yield (os.path.join(path[len(directory)+1:], f),
datetime.fromtimestamp(mtime))
def createExtFileObjects(self, client, addresses, extFileType=None):
if extFileType is None:
extFileType = client.context.getLoopsRoot().getConceptManager()['extfile']
rm = client.context.getLoopsRoot().getResourceManager()
container = client.context.getLoopsRoot().getResourceManager()
directory = self.getDirectory(client)
for addr in addresses:
name = addr
name = self.generateName(container, addr)
title = self.generateTitle(addr)
obj = addAndConfigureObject(
rm, Resource, name,
title=addr.decode('UTF-8'),
type=extFileType,
container, Resource, name,
title=title,
resourceType=extFileType,
externalAddress=addr,
storage='fullpath',
storageParams=dict(subdirectory=directory))
@ -98,3 +122,14 @@ class DirectoryCollectionProvider(object):
address = client.address or ''
return os.path.join(baseAddress, address)
def generateName(self, container, name):
name = INameChooser(container).chooseName(name, None)
return name
def generateTitle(self, title):
title = os.path.split(title)[-1]
if '.' in title:
base, ext = title.rsplit('.', 1)
if ext.lower() in mimetypes.extensions.values():
title = base
return title.decode('UTF-8')

17
integrator/interfaces.py
View file

@ -53,17 +53,11 @@ class IExternalCollection(Interface):
description=_(u'A regular expression for selecting external objects '
'that should belong to this collection'),
required=False)
def create():
""" Select external objects that should belong to a collection
using all the informations in the attributes,
create a resource of type 'extfile' for each of them,
and associate them with this collection.
Fire appropriate events.
"""
lastUpdated = Attribute('Date and time of last update.')
def update():
""" Check for new, changed, or deleted external objects.
""" Select external objects that should belong to a collection
and check for new, changed, or deleted objects.
Create an 'extfile' resource for new ones, fire appropriate
events for new, changed, or deleted ones.
Resources for deleted objects are not removed but should
@ -77,8 +71,9 @@ class IExternalCollectionProvider(Interface):
def collect(clientCollection):
""" Select objects that should belong to a collection,
return an iterable of local address parts of the selected external
objects. The object specified by the 'clientCollection' argument
return an iterable of tuples of local address parts of the selected external
objects and their last modification date/time.
The object specified by the 'clientCollection' argument
is usually the caller of the method and should provide the
IExternalCollection interface.
"""

4
tests/setup.py
View file

@ -18,6 +18,7 @@ from cybertools.typology.interfaces import IType
from loops import Loops
from loops import util
from loops.common import NameChooser
from loops.interfaces import IIndexAttributes
from loops.concept import Concept
from loops.concept import IndexAttributes as ConceptIndexAttributes
@ -44,14 +45,13 @@ class TestSite(object):
component.provideAdapter(ConceptType)
component.provideAdapter(ResourceType)
component.provideAdapter(TypeConcept)
component.provideAdapter(NameChooser)
catalog = self.catalog = Catalog()
component.provideUtility(catalog, ICatalog)
catalog['loops_title'] = TextIndex('title', IIndexAttributes, True)
catalog['loops_text'] = TextIndex('text', IIndexAttributes, True)
catalog['loops_type'] = FieldIndex('tokenForSearch', IType, False)
component.provideAdapter(ConceptIndexAttributes)
component.provideAdapter(ResourceIndexAttributes)