loops.integrator: ExternalCollectionAdapter working

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1689 fd906abe-77d9-0310-91a1-e0d9ade77398

README.txt

@@ -688,7 +688,7 @@ and possibly critcal cases:
 
   >>> nc = NameChooser(resources)
   >>> nc.chooseName(u'', Resource(u'abc: (cde)'))
-  u'abc_cde'
+  u'abc__cde'
   >>> nc.chooseName(u'', Resource(u'\xdcml\xe4ut'))
   u'uemlaeut'
   >>> nc.chooseName(u'', Resource(u'A very very loooooong title'))

common.py

@@ -108,29 +108,43 @@ class NameChooser(BaseNameChooser):
 
     def chooseName(self, name, obj):
         if not name:
-            name = self.generateName(obj)
+            name = self.generateNameFromTitle(obj)
+        else:
+            name = self.normalizeName(name)
         name = super(NameChooser, self).chooseName(name, obj)
         return name
 
-    def generateName(self, obj):
+    def generateNameFromTitle(self, obj):
         title = obj.title
-        result = []
         if len(title) > 15:
             words = title.split()
             if len(words) > 1:
                 title = '_'.join((words[0], words[-1]))
-        for c in title:
+        return self.normalizeName(title)
+
+    def normalizeName(self, baseName):
+        result = []
+        for c in baseName:
             try:
                 c = c.encode('ISO8859-15')
             except UnicodeEncodeError:
+                # skip all characters not representable in ISO encoding
+                continue
+            if c in '._':
+                # separator and special characters to keep
+                result.append(c)
                 continue
             if c in self.specialCharacters:
+                # transform umlauts and other special characters
                 result.append(self.specialCharacters[c].lower())
                 continue
             if ord(c) > 127:
+                # map to ASCII characters
                 c = chr(ord(c) & 127)
-            if c in ('_., '):
+            if c in ':,/\\ ':
+                # replace separator characters with _
                 result.append('_')
+                # skip all other characters
             elif not c.isalpha() and not c.isdigit():
                 continue
             else:

integrator/README.txt

@@ -54,7 +54,6 @@ to the external system:
   >>> aColl01.baseAddress = dataDir
   >>> aColl01.address = 'topics'
 
-
 Directory Collection Provider
 -----------------------------
 
@@ -66,24 +65,42 @@ object, the external collection itself.
   >>> dcp = DirectoryCollectionProvider()
 
   >>> sorted(dcp.collect(aColl01))
-  ['programming/BeautifulProgram.pdf', 'programming/zope/zope3.txt']
+  [('programming/BeautifulProgram.pdf', datetime.datetime(2005, 4, 7, 12, 36, 56)),
+   ('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))]
 
-If we provide a selective pattern we get only part of the files:
+If we provide a more selective pattern we get only part of the files:
 
   >>> aColl01.pattern = r'.*\.txt'
   >>> sorted(dcp.collect(aColl01))
-  ['programming/zope/zope3.txt']
+  [('programming/zope/zope3.txt', datetime.datetime(2007, 4, 12, 15, 16, 13))]
 
 Let's now create the corresponding resource objects.
 
   >>> aColl01.pattern = ''
-  >>> addresses = dcp.collect(aColl01)
+  >>> addresses = [e[0] for e in dcp.collect(aColl01)]
   >>> res = list(dcp.createExtFileObjects(aColl01, addresses))
   >>> len(sorted(r.__name__ for r in res))
   2
   >>> xf1 = res[0]
   >>> xf1.__name__
-  u'programming/BeautifulProgram.pdf'
+  u'programming_beautifulprogram.pdf'
+  >>> xf1.title
+  u'BeautifulProgram'
+
+  >>> for r in res:
+  ...     del resources[r.__name__]
+
+Working with the External Collection
+------------------------------------
+
+  >>> component.provideUtility(DirectoryCollectionProvider())
+  >>> aColl01.update()
+  >>> res = coll01.getResources()
+  >>> len(res)
+  2
+  >>> sorted((r.__name__, r.title) for r in res)
+  [(u'programming_beautifulprogram.pdf', u'BeautifulProgram'),
+   (u'programming_zope_zope3.txt', u'zope3')]
 
 
 Fin de partie

integrator/collection.py

@@ -23,13 +23,20 @@ file system.
 $Id$
 """
 
-import os, re
+from datetime import datetime
+import os, re, stat
+
+from zope import component
+from zope.lifecycleevent import ObjectModifiedEvent
+from zope.event import notify
+from zope.app.container.interfaces import INameChooser
 from zope.component import adapts
 from zope.interface import implements, Attribute
 from zope.cachedescriptors.property import Lazy
 from zope.schema.interfaces import IField
 from zope.traversing.api import getName, getParent
 
+from cybertools.text import mimetypes
 from cybertools.typology.interfaces import IType
 from loops.common import AdapterBase
 from loops.interfaces import IResource, IConcept
@@ -54,11 +61,23 @@ class ExternalCollectionAdapter(AdapterBase):
     _adapterAttributes = ('context', '__parent__',)
     _contextAttributes = list(IExternalCollection) + list(IConcept)
 
-    def create(self):
-        pass
-
     def update(self):
-        pass
+        existing = self.context.getResources()
+        old = dict((obj.externalAddress, obj) for obj in existing)
+        new = []
+        provider = component.getUtility(IExternalCollectionProvider,
+                                        name=self.providerName or '')
+        for addr, mdate in provider.collect(self):
+            if addr in old:
+                if mdate > self.lastUpdated:
+                    notify(ObjectModifiedEvent(old[addr]))
+            else:
+                new.append(addr)
+        if new:
+            newResources = provider.createExtFileObjects(self, new)
+            for r in newResources:
+                self.context.assignResource(r)
+        self.lastUpdated = datetime.today()
 
 
 class DirectoryCollectionProvider(object):
@@ -75,19 +94,24 @@ class DirectoryCollectionProvider(object):
                 del dirs[dirs.index('.svn')]
             for f in files:
                 if pattern.match(f):
-                    yield os.path.join(path[len(directory)+1:], f)
+                    # may be it would be better to return a file's hash
+                    # for checking for changes...
+                    mtime = os.stat(os.path.join(path, f))[stat.ST_MTIME]
+                    yield (os.path.join(path[len(directory)+1:], f),
+                           datetime.fromtimestamp(mtime))
 
     def createExtFileObjects(self, client, addresses, extFileType=None):
         if extFileType is None:
             extFileType = client.context.getLoopsRoot().getConceptManager()['extfile']
-        rm = client.context.getLoopsRoot().getResourceManager()
+        container = client.context.getLoopsRoot().getResourceManager()
         directory = self.getDirectory(client)
         for addr in addresses:
-            name = addr
+            name = self.generateName(container, addr)
+            title = self.generateTitle(addr)
             obj = addAndConfigureObject(
-                            rm, Resource, name,
+                            container, Resource, name,
-                            title=addr.decode('UTF-8'),
+                            title=title,
-                            type=extFileType,
+                            resourceType=extFileType,
                             externalAddress=addr,
                             storage='fullpath',
                             storageParams=dict(subdirectory=directory))
@@ -98,3 +122,14 @@ class DirectoryCollectionProvider(object):
         address = client.address or ''
         return os.path.join(baseAddress, address)
 
+    def generateName(self, container, name):
+        name = INameChooser(container).chooseName(name, None)
+        return name
+
+    def generateTitle(self, title):
+        title = os.path.split(title)[-1]
+        if '.' in title:
+            base, ext = title.rsplit('.', 1)
+            if ext.lower() in mimetypes.extensions.values():
+                title = base
+        return title.decode('UTF-8')

integrator/interfaces.py

@@ -53,17 +53,11 @@ class IExternalCollection(Interface):
             description=_(u'A regular expression for selecting external objects '
                            'that should belong to this collection'),
             required=False)
-
+    lastUpdated = Attribute('Date and time of last update.')
-    def create():
-        """ Select external objects that should belong to a collection
-            using all the informations in the attributes,
-            create a resource of type 'extfile' for each of them,
-            and associate them with this collection.
-            Fire appropriate events.
-        """
 
     def update():
-        """ Check for new, changed, or deleted external objects.
+        """ Select external objects that should belong to a collection
+            and check for new, changed, or deleted objects.
             Create an 'extfile' resource for new ones, fire appropriate
             events for new, changed, or deleted ones.
             Resources for deleted objects are not removed but should
@@ -77,8 +71,9 @@ class IExternalCollectionProvider(Interface):
 
     def collect(clientCollection):
         """ Select objects that should belong to a collection,
-            return an iterable of local address parts of the selected external
+            return an iterable of tuples of local address parts of the selected external
-            objects. The object specified by the 'clientCollection' argument
+            objects and their last modification date/time.
+            The object specified by the 'clientCollection' argument
             is usually the caller of the method and should provide the
             IExternalCollection interface.
         """

tests/setup.py

@@ -18,6 +18,7 @@ from cybertools.typology.interfaces import IType
 
 from loops import Loops
 from loops import util
+from loops.common import NameChooser
 from loops.interfaces import IIndexAttributes
 from loops.concept import Concept
 from loops.concept import IndexAttributes as ConceptIndexAttributes
@@ -44,14 +45,13 @@ class TestSite(object):
         component.provideAdapter(ConceptType)
         component.provideAdapter(ResourceType)
         component.provideAdapter(TypeConcept)
+        component.provideAdapter(NameChooser)
 
         catalog = self.catalog = Catalog()
         component.provideUtility(catalog, ICatalog)
-
         catalog['loops_title'] = TextIndex('title', IIndexAttributes, True)
         catalog['loops_text'] = TextIndex('text', IIndexAttributes, True)
         catalog['loops_type'] = FieldIndex('tokenForSearch', IType, False)
-
         component.provideAdapter(ConceptIndexAttributes)
         component.provideAdapter(ResourceIndexAttributes)