diff --git a/README.txt b/README.txt index e72ee3f..d5d7597 100755 --- a/README.txt +++ b/README.txt @@ -688,7 +688,7 @@ and possibly critcal cases: >>> nc = NameChooser(resources) >>> nc.chooseName(u'', Resource(u'abc: (cde)')) - u'abc_cde' + u'abc__cde' >>> nc.chooseName(u'', Resource(u'\xdcml\xe4ut')) u'uemlaeut' >>> nc.chooseName(u'', Resource(u'A very very loooooong title')) diff --git a/common.py b/common.py index b955c0a..e6e4449 100644 --- a/common.py +++ b/common.py @@ -108,29 +108,43 @@ class NameChooser(BaseNameChooser): def chooseName(self, name, obj): if not name: - name = self.generateName(obj) + name = self.generateNameFromTitle(obj) + else: + name = self.normalizeName(name) name = super(NameChooser, self).chooseName(name, obj) return name - def generateName(self, obj): + def generateNameFromTitle(self, obj): title = obj.title - result = [] if len(title) > 15: words = title.split() if len(words) > 1: title = '_'.join((words[0], words[-1])) - for c in title: + return self.normalizeName(title) + + def normalizeName(self, baseName): + result = [] + for c in baseName: try: c = c.encode('ISO8859-15') except UnicodeEncodeError: + # skip all characters not representable in ISO encoding + continue + if c in '._': + # separator and special characters to keep + result.append(c) continue if c in self.specialCharacters: + # transform umlauts and other special characters result.append(self.specialCharacters[c].lower()) continue if ord(c) > 127: + # map to ASCII characters c = chr(ord(c) & 127) - if c in ('_., '): + if c in ':,/\\ ': + # replace separator characters with _ result.append('_') + # skip all other characters elif not c.isalpha() and not c.isdigit(): continue else: diff --git a/integrator/README.txt b/integrator/README.txt index 75a497e..2614360 100644 --- a/integrator/README.txt +++ b/integrator/README.txt @@ -54,7 +54,6 @@ to the external system: >>> aColl01.baseAddress = dataDir >>> aColl01.address = 'topics' - Directory Collection Provider ----------------------------- @@ -66,24 +65,42 @@ object, the external collection itself. >>> dcp = DirectoryCollectionProvider() >>> sorted(dcp.collect(aColl01)) - ['programming/BeautifulProgram.pdf', 'programming/zope/zope3.txt'] + [('programming/BeautifulProgram.pdf', datetime.datetime(2005, 4, 7, 12, 36, 56)), + ('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))] -If we provide a selective pattern we get only part of the files: +If we provide a more selective pattern we get only part of the files: >>> aColl01.pattern = r'.*\.txt' >>> sorted(dcp.collect(aColl01)) - ['programming/zope/zope3.txt'] + [('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))] Let's now create the corresponding resource objects. >>> aColl01.pattern = '' - >>> addresses = dcp.collect(aColl01) + >>> addresses = [e[0] for e in dcp.collect(aColl01)] >>> res = list(dcp.createExtFileObjects(aColl01, addresses)) >>> len(sorted(r.__name__ for r in res)) 2 >>> xf1 = res[0] >>> xf1.__name__ - u'programming/BeautifulProgram.pdf' + u'programming_beautifulprogram.pdf' + >>> xf1.title + u'BeautifulProgram' + + >>> for r in res: + ... del resources[r.__name__] + +Working with the External Collection +------------------------------------ + + >>> component.provideUtility(DirectoryCollectionProvider()) + >>> aColl01.update() + >>> res = coll01.getResources() + >>> len(res) + 2 + >>> sorted((r.__name__, r.title) for r in res) + [(u'programming_beautifulprogram.pdf', u'BeautifulProgram'), + (u'programming_zope_zope3.txt', u'zope3')] Fin de partie diff --git a/integrator/collection.py b/integrator/collection.py index 1be017c..870462f 100644 --- a/integrator/collection.py +++ b/integrator/collection.py @@ -23,13 +23,20 @@ file system. $Id$ """ -import os, re +from datetime import datetime +import os, re, stat + +from zope import component +from zope.lifecycleevent import ObjectModifiedEvent +from zope.event import notify +from zope.app.container.interfaces import INameChooser from zope.component import adapts from zope.interface import implements, Attribute from zope.cachedescriptors.property import Lazy from zope.schema.interfaces import IField from zope.traversing.api import getName, getParent +from cybertools.text import mimetypes from cybertools.typology.interfaces import IType from loops.common import AdapterBase from loops.interfaces import IResource, IConcept @@ -54,11 +61,23 @@ class ExternalCollectionAdapter(AdapterBase): _adapterAttributes = ('context', '__parent__',) _contextAttributes = list(IExternalCollection) + list(IConcept) - def create(self): - pass - def update(self): - pass + existing = self.context.getResources() + old = dict((obj.externalAddress, obj) for obj in existing) + new = [] + provider = component.getUtility(IExternalCollectionProvider, + name=self.providerName or '') + for addr, mdate in provider.collect(self): + if addr in old: + if mdate > self.lastUpdated: + notify(ObjectModifiedEvent(old[addr])) + else: + new.append(addr) + if new: + newResources = provider.createExtFileObjects(self, new) + for r in newResources: + self.context.assignResource(r) + self.lastUpdated = datetime.today() class DirectoryCollectionProvider(object): @@ -75,19 +94,24 @@ class DirectoryCollectionProvider(object): del dirs[dirs.index('.svn')] for f in files: if pattern.match(f): - yield os.path.join(path[len(directory)+1:], f) + # may be it would be better to return a file's hash + # for checking for changes... + mtime = os.stat(os.path.join(path, f))[stat.ST_MTIME] + yield (os.path.join(path[len(directory)+1:], f), + datetime.fromtimestamp(mtime)) def createExtFileObjects(self, client, addresses, extFileType=None): if extFileType is None: extFileType = client.context.getLoopsRoot().getConceptManager()['extfile'] - rm = client.context.getLoopsRoot().getResourceManager() + container = client.context.getLoopsRoot().getResourceManager() directory = self.getDirectory(client) for addr in addresses: - name = addr + name = self.generateName(container, addr) + title = self.generateTitle(addr) obj = addAndConfigureObject( - rm, Resource, name, - title=addr.decode('UTF-8'), - type=extFileType, + container, Resource, name, + title=title, + resourceType=extFileType, externalAddress=addr, storage='fullpath', storageParams=dict(subdirectory=directory)) @@ -98,3 +122,14 @@ class DirectoryCollectionProvider(object): address = client.address or '' return os.path.join(baseAddress, address) + def generateName(self, container, name): + name = INameChooser(container).chooseName(name, None) + return name + + def generateTitle(self, title): + title = os.path.split(title)[-1] + if '.' in title: + base, ext = title.rsplit('.', 1) + if ext.lower() in mimetypes.extensions.values(): + title = base + return title.decode('UTF-8') diff --git a/integrator/interfaces.py b/integrator/interfaces.py index a80e159..06c4dd3 100644 --- a/integrator/interfaces.py +++ b/integrator/interfaces.py @@ -53,17 +53,11 @@ class IExternalCollection(Interface): description=_(u'A regular expression for selecting external objects ' 'that should belong to this collection'), required=False) - - def create(): - """ Select external objects that should belong to a collection - using all the informations in the attributes, - create a resource of type 'extfile' for each of them, - and associate them with this collection. - Fire appropriate events. - """ + lastUpdated = Attribute('Date and time of last update.') def update(): - """ Check for new, changed, or deleted external objects. + """ Select external objects that should belong to a collection + and check for new, changed, or deleted objects. Create an 'extfile' resource for new ones, fire appropriate events for new, changed, or deleted ones. Resources for deleted objects are not removed but should @@ -77,8 +71,9 @@ class IExternalCollectionProvider(Interface): def collect(clientCollection): """ Select objects that should belong to a collection, - return an iterable of local address parts of the selected external - objects. The object specified by the 'clientCollection' argument + return an iterable of tuples of local address parts of the selected external + objects and their last modification date/time. + The object specified by the 'clientCollection' argument is usually the caller of the method and should provide the IExternalCollection interface. """ diff --git a/tests/setup.py b/tests/setup.py index 86035d8..7e79cf9 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -18,6 +18,7 @@ from cybertools.typology.interfaces import IType from loops import Loops from loops import util +from loops.common import NameChooser from loops.interfaces import IIndexAttributes from loops.concept import Concept from loops.concept import IndexAttributes as ConceptIndexAttributes @@ -44,14 +45,13 @@ class TestSite(object): component.provideAdapter(ConceptType) component.provideAdapter(ResourceType) component.provideAdapter(TypeConcept) + component.provideAdapter(NameChooser) catalog = self.catalog = Catalog() component.provideUtility(catalog, ICatalog) - catalog['loops_title'] = TextIndex('title', IIndexAttributes, True) catalog['loops_text'] = TextIndex('text', IIndexAttributes, True) catalog['loops_type'] = FieldIndex('tokenForSearch', IType, False) - component.provideAdapter(ConceptIndexAttributes) component.provideAdapter(ResourceIndexAttributes)