work in progress: loops.integrator with DirectoryCollectionProvider

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1686 fd906abe-77d9-0310-91a1-e0d9ade77398

configure.zcml

@@ -417,8 +417,11 @@
 
   <utility
       factory="cybertools.storage.filesystem.instanceVarSubdirectoryStorage"
-      name="varsubdir"
+      name="varsubdir" />
-      />
+
+  <utility
+      factory="cybertools.storage.filesystem.fullPathStorage"
+      name="fullpath" />
 
   <vocabulary
       factory="loops.concept.ConceptTypeSourceList"
@@ -447,6 +450,7 @@
 
   <include package=".knowledge" />
   <include package=".organize" />
+  <include package=".integrator" />
   <include package=".process" />
   <include package=".versioning" />
   <include package=".search" />

integrator/README.txt

@@ -29,6 +29,63 @@ configuration):
   >>> len(concepts) + len(resources)
   18
 
+
+External Collections
+====================
+
+The basis of our work will be ExternalCollection objects, i.e. concepts
+of the 'extcollection' type. We use an adapter for providing the attributes
+and methods of the external collect object.
+
+  >>> from loops.concept import Concept
+  >>> from loops.setup import addObject
+  >>> from loops.integrator.collection import ExternalCollectionAdapter
+  >>> tExternalCollection = concepts['extcollection']
+  >>> coll01 = addObject(concepts, Concept, 'coll01',
+  ...                    title=u'Collection One', type=tExternalCollection)
+  >>> aColl01 = ExternalCollectionAdapter(coll01)
+
+An external collection carries a set of attributes that control the access
+to the external system:
+
+  >>> aColl01.providerName, aColl01.baseAddress, aColl01.address, aColl01.pattern
+  (None, None, None, None)
+  >>> from loops.integrator.testsetup import dataDir
+  >>> aColl01.baseAddress = dataDir
+  >>> aColl01.address = 'topics'
+
+
+Directory Collection Provider
+-----------------------------
+
+The DirectoryCollectionProvider collects files from a directory in the
+file system. The parameters (directory paths) are provided by the calling
+object, the external collection itself.
+
+  >>> from loops.integrator.collection import DirectoryCollectionProvider
+  >>> dcp = DirectoryCollectionProvider()
+
+  >>> sorted(dcp.collect(aColl01))
+  ['programming/BeautifulProgram.pdf', 'programming/zope/zope3.txt']
+
+If we provide a selective pattern we get only part of the files:
+
+  >>> aColl01.pattern = r'.*\.txt'
+  >>> sorted(dcp.collect(aColl01))
+  ['programming/zope/zope3.txt']
+
+Let's now create the corresponding resource objects.
+
+  >>> aColl01.pattern = ''
+  >>> addresses = dcp.collect(aColl01)
+  >>> res = list(dcp.createExtFileObjects(aColl01, addresses))
+  >>> len(sorted(r.__name__ for r in res))
+  2
+  >>> xf1 = res[0]
+  >>> xf1.__name__
+  u'programming/BeautifulProgram.pdf'
+
+
 Fin de partie
 =============
 

integrator/collection.py

@@ -0,0 +1,99 @@
+#
+#  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Concept adapter(s) for external collections, e.g. a directory in the
+file system.
+
+$Id$
+"""
+
+import os, re
+from zope.component import adapts
+from zope.interface import implements, Attribute
+from zope.cachedescriptors.property import Lazy
+from zope.schema.interfaces import IField
+from zope.traversing.api import getName, getParent
+
+from cybertools.typology.interfaces import IType
+from loops.common import AdapterBase
+from loops.interfaces import IResource, IConcept
+from loops.integrator.interfaces import IExternalCollection
+from loops.integrator.interfaces import IExternalCollectionProvider
+from loops.resource import Resource
+from loops.setup import addObject
+from loops.type import TypeInterfaceSourceList
+
+
+TypeInterfaceSourceList.typeInterfaces += (IExternalCollection,)
+
+
+class ExternalCollectionAdapter(AdapterBase):
+    """ A concept adapter for accessing an external collection.
+        May delegate access to a named utility.
+    """
+
+    implements(IExternalCollection)
+    adapts(IConcept)
+
+    _adapterAttributes = ('context', '__parent__',)
+    _contextAttributes = list(IExternalCollection) + list(IConcept)
+
+    def create(self):
+        pass
+
+    def update(self):
+        pass
+
+
+class DirectoryCollectionProvider(object):
+    """ A utility that provides access to files in a directory.
+    """
+
+    implements(IExternalCollectionProvider)
+
+    def collect(self, client):
+        directory = self.getDirectory(client)
+        pattern = re.compile(client.pattern or '.*')
+        result = []
+        for path, dirs, files in os.walk(directory):
+            if files:
+                result.extend(os.path.join(path[len(directory)+1:], f)
+                                for f in files
+                                    if pattern.match(f))
+        return result
+
+    def createExtFileObjects(self, client, addresses, extFileType=None):
+        if extFileType is None:
+            extFileType = client.context.getLoopsRoot().getConceptManager()['extfile']
+        rm = client.context.getLoopsRoot().getResourceManager()
+        directory = self.getDirectory(client)
+        for addr in addresses:
+            name = addr
+            obj = addObject(rm, Resource, name,
+                            title=addr.decode('UTF-8'), type=extFileType,
+                            externalAddress=addr,
+                            storage='fullpath',
+                            storageParams=dict(subdirectory=directory))
+            yield obj
+
+    def getDirectory(self, client):
+        baseAddress = client.baseAddress or ''
+        address = client.address or ''
+        return os.path.join(baseAddress, address)
+

integrator/configure.zcml

@@ -6,4 +6,17 @@
    i18n_domain="zope"
    >
 
+  <zope:adapter factory="loops.integrator.collection.ExternalCollectionAdapter"
+                trusted="True" />
+
+  <zope:class class="loops.integrator.collection.ExternalCollectionAdapter">
+    <require permission="zope.View"
+             interface="loops.integrator.interfaces.IExternalCollection" />
+    <require permission="zope.ManageContent"
+             set_schema="loops.integrator.interfaces.IExternalCollection" />
+  </zope:class>
+
+  <zope:utility
+      factory="loops.integrator.collection.DirectoryCollectionProvider" />
+
 </configure>

integrator/interfaces.py

@@ -25,8 +25,91 @@ $Id$
 from zope.interface import Interface, Attribute
 from zope import interface, component, schema
 
+from loops.util import _
+
 
 class IExternalCollection(Interface):
-    """ A collection of resources.
+    """ A collection of resources, to be used for a concept adapter.
     """
 
+    providerName = schema.TextLine(
+            title=_(u'Provider name'),
+            description=_(u'The name of a utility that provides the '
+                           'external objects; default is a directory '
+                           'collection provider'),
+            required=False)
+    baseAddress = schema.TextLine(
+            title=_(u'Base address'),
+            description=_(u'A base path or URL for accessing this collection '
+                           'on the external system'),
+            required=False)
+    address = schema.TextLine(
+            title=_(u'Relative address'),
+            description=_(u'Optional second (local) part of the '
+                           'collection\'s address'),
+            required=False)
+    pattern = schema.TextLine(
+            title=_(u'Selection pattern'),
+            description=_(u'A regular expression for selecting external objects '
+                           'that should belong to this collection'),
+            required=False)
+
+    def create():
+        """ Select external objects that should belong to a collection
+            using all the informations in the attributes,
+            create a resource of type 'extfile' for each of them,
+            and associate them with this collection.
+            Fire appropriate events.
+        """
+
+    def update():
+        """ Check for new, changed, or deleted external objects.
+            Create an 'extfile' resource for new ones, fire appropriate
+            events for new, changed, or deleted ones.
+            Resources for deleted objects are not removed but should
+            be empty; they also should receive some state change.
+        """
+
+
+class IExternalCollectionProvider(Interface):
+    """ A utility that provides access to an external collection of objects.
+    """
+
+    def collect(clientCollection):
+        """ Select objects that should belong to a collection,
+            return an iterable of local address parts of the selected external
+            objects. The object specified by the 'clientCollection' argument
+            is usually the caller of the method and should provide the
+            IExternalCollection interface.
+        """
+
+    def createExtFileObjects(clientCollection, addresses, extFileType=None):
+        """ Create a resource of type 'extFileType' (default is the
+            type with the name 'extfile') for each of the addresses
+            provided. Return the list of objects created.
+        """
+
+
+class IAutoClassifier(Interface):
+    """ An adapter that more or less automagically assigns concepts to a
+        resource using some sort of selection criteria for the concepts
+        that should be considered.
+    """
+
+
+class IOntologyExporter(Interface):
+    """ An adapter for creating an XML file with all appropriate informations
+        from the context and its children, selecting children via a
+        pattern or a set of selection criteria.
+
+        This may then be used by an external tool for classifying
+        a set of external objects.
+    """
+
+
+class IClassificationImporter(Interface):
+    """ An Adapter for importing an XML file with classification
+        information for a collection of external objects."
+    """
+
+

integrator/testdata/topics/programming/zope/zope3.txt

@@ -0,0 +1,5 @@
+This file carries some information about Zope 3, the open-source
+web application server.
+
+It is used as an example document for integrating external resources
+into loops.

integrator/testsetup.py

@@ -4,15 +4,20 @@ Set up a loops site for testing.
 $Id$
 """
 
+import os
 from zope import component
 
 from loops import util
+from loops.interfaces import IExternalFile
 from loops.concept import Concept
 from loops.resource import Resource
 from loops.integrator.interfaces import IExternalCollection
 from loops.knowledge.setup import SetupManager as KnowledgeSetupManager
+from loops.setup import SetupManager, addObject
 from loops.tests.setup import TestSite as BaseTestSite
 
+dataDir = os.path.join(os.path.dirname(__file__), 'testdata')
+
 
 class TestSite(BaseTestSite):
 
@@ -24,9 +29,12 @@ class TestSite(BaseTestSite):
         concepts, resources, views = self.baseSetup()
 
         tType = concepts.getTypeConcept()
-
+        tExtFile = addObject(concepts, Concept, 'extfile',
-        tExtFile = concepts['extfile'] = Concept(u'External File')
+                                title=u'External File', type=tType,
-        tExtCollection = concepts['extcollection'] = Concept(u'External Collection')
+                                typeInterface=IExternalFile)
+        tExtCollection = addObject(concepts, Concept, 'extcollection',
+                                title=u'External Collection', type=tType,
+                                typeInterface=IExternalCollection)
 
         self.indexAll(concepts, resources)
         return concepts, resources, views

organize/interfaces.py

@@ -27,7 +27,6 @@ from zope import interface, component, schema
 from zope.app import zapi
 from zope.app.principalannotation import annotations
 from zope.app.security.interfaces import IAuthentication, PrincipalLookupError
-from zope.i18nmessageid import MessageFactory
 from zope.security.proxy import removeSecurityProxy
 
 from cybertools.organize.interfaces import IPerson as IBasePerson

resource.py

@@ -315,12 +315,20 @@ class ExternalFileAdapter(FileAdapter):
         self.context._storageParams = value
     storageParams = property(getStorageParams, setStorageParams)
 
-    @Lazy
+    def getExternalAddress(self):
-    def externalAddress(self):
+        return getattr(self.context, '_externalAddress', self.context.__name__)
+    def setExternalAddress(self, addr):
+        # TODO (?) - use intId as default?
+        self.context._externalAddress = addr
+    externalAddress = property(getExternalAddress, setExternalAddress)
+
+    #@Lazy
+    #def externalAddress(self):
         # or is this an editable attribute?
         # or some sort of subpath set during import?
         # anyway: an attribute of the context object.
-        return self.context.__name__
+        # TODO: use intId and store in special attribute for later reuse
+        #return self.context.__name__
 
     def setData(self, data):
         storageParams = self.storageParams