move metadata into Resource class; more on httpput transport

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1897 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/crawl/base.py

@@ -42,17 +42,18 @@ class CrawlingJob(Job):
         return self.collect()
 
 
-class Metadata(object):
+class Metadata(dict):
 
     implements(IMetadataSet)
 
     def __init__(self, data=dict()):
-        self.data = data
+        for k in data:
+            self[k] = data[k]
 
     def asXml(self):
         # TODO...
         return ''
 
     def set(self, key, value):
-        self.data['key'] = value
+        self['key'] = value
 

agent/crawl/filesystem.py

@@ -22,7 +22,8 @@ Filesystem crawler.
 $Id$
 """
 
-import os, re, stat
+import os
+from fnmatch import filter
 from datetime import datetime
 from twisted.internet.defer import Deferred
 from twisted.internet.task import coiterate
@@ -46,32 +47,31 @@ class CrawlingJob(BaseCrawlingJob):
         self.deferred.callback(self.collected)
 
     def crawlFilesystem(self):
-        criteria = self.params
+        directory = self.params.get('directory')
-        directory = criteria.get('directory')
+        pattern = self.params.get('pattern') or '*'
-        pattern = re.compile(criteria.get('pattern') or '.*')
+        lastRun = self.params.get('lastrun') or datetime(1980, 1, 1)
         for path, dirs, files in os.walk(directory):
             if '.svn' in dirs:
                 del dirs[dirs.index('.svn')]
-            for f in files:
+            for f in filter(files, pattern):
-                if pattern.match(f):
+                filename = os.path.join(path, f)
-                    filename = os.path.join(path, f)
+                mtime = datetime.fromtimestamp(os.path.getmtime(filename))
-                    mtime = datetime.fromtimestamp(
+                if mtime <= lastRun:  # file not changed
-                                os.stat(filename)[stat.ST_MTIME])
+                    continue
-                    # TODO: check modification time
+                meta = dict(
-                    meta = dict(
+                    path=filename,
-                        path=filename,
+                )
-                    )
+                self.collected.append(FileResource(filename, Metadata(meta)))
-                    self.collected.append((FileResource(filename),
+                yield None
-                                           Metadata(meta)))
-                    yield None
 
 
 class FileResource(object):
 
     implements(IResource)
 
-    def __init__(self, path):
+    def __init__(self, path, metadata=None):
         self.path = path
+        self.metadata = metadata
 
     @property
     def data(self):

agent/interfaces.py

@@ -112,9 +112,12 @@ class IResource(Interface):
     data = Attribute("A string, file, or similar representation of the "
                      "resource's content")
 
+    metadata = Attribute('Information describing this resource; '
+                         'should be an IMetadataSet object.')
+
 
 class IMetadataSet(Interface):
-    """ Metadata associated with a resource.
+    """ Metadata associated with a resource; sort of a mapping.
     """
 
     def asXML():
@@ -145,11 +148,9 @@ class ITransporter(Interface):
     userName = Attribute('User name for logging in to the server.')
     password = Attribute('Password for logging in to the server.')
 
-    def transfer(resource, metadata=None):
+    def transfer(resource):
-        """ Transfer the resource (typically just a file that may
+        """ Transfer the resource (an object providing IResource)
-            be read) to the server.
+            to the server.
-
-            The resource may be associated with a metadata set.
         """
 
 

agent/testing/crawl.py

@@ -39,7 +39,7 @@ class CrawlingJob(BaseCrawlingJob):
         return deferred
 
     def dataAvailable(self):
-        self.deferred.callback([(DummyResource(), None)])
+        self.deferred.callback([DummyResource()])
 
 
 class DummyResource(object):
@@ -47,3 +47,4 @@ class DummyResource(object):
     implements(IResource)
 
     data = 'Dummy resource data for testing purposes.'
+    metadata = None

agent/testing/transport.py

@@ -40,8 +40,8 @@ class TransportJob(BaseJob):
         if result is None:
             print 'No data available.'
         else:
-            for res, meta in result:
+            for resource in result:
-                d = self.transporter.transfer(res.data, meta)
+                d = self.transporter.transfer(resource)
         return Deferred()
 
 
@@ -49,14 +49,16 @@ class Transporter(BaseTransporter):
 
     jobFactory = TransportJob
 
-    def transfer(self, data, metadata=None):
+    def transfer(self, resource):
+        data = resource.data
         if type(data) is file:
             text = data.read()
             data.close()
         else:
             text = data
+        metadata = resource.metadata
         if metadata is not None:
-            print 'Metadata:', metadata.data
+            print 'Metadata:', metadata
         print 'Transferring:', text
         return Deferred()
 

agent/tests.py

@@ -62,6 +62,7 @@ def test_suite():
                 #standard_unittest.makeSuite(Test),
                 doctest.DocFileSuite('README.txt', optionflags=flags),
                 doctest.DocFileSuite('crawl/filesystem.txt', optionflags=flags),
+                doctest.DocFileSuite('transport/httpput.txt', optionflags=flags),
             ))
 
 if __name__ == '__main__':

agent/transport/base.py

@@ -48,9 +48,10 @@ class Transporter(object):
     def __init__(self, agent):
         self.agent = agent
 
-    def transfer(self, data, metadata=None):
+    def transfer(self, resource):
+        data = resource.data
         if type(data) is file:
-            data = text.read()
+            text = data.read()
             data.close()
         else:
             text = data

agent/transport/httpput.py

@@ -23,7 +23,7 @@ $Id$
 """
 
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred
+from twisted.web.client import getPage
 from zope.interface import implements
 
 from loops.agent.interfaces import ITransporter, ITransportJob
@@ -38,13 +38,13 @@ class TransportJob(Job):
         super(TransportJob, self).__init__()
         self.transporter = transporter
 
-    def execute(self, **kw):
+    def execute(self):
         result = kw.get('result')
         if result is None:
             print 'No data available.'
         else:
-            for r in result:
+            for resource, metadata in result:
-                d = self.transporter.transfer(r[0].data, r[1], str)
+                d = self.transporter.transfer(resource.data, metadata)
         return Deferred()
 
 
@@ -62,15 +62,18 @@ class Transporter(object):
 
     def __init__(self, agent):
         self.agent = agent
-        config = agent.config
+        conf = agent.config
+        # TODO: get settings from conf
 
-    def transfer(self, resource, metadata=None, resourceType=file):
+    def transfer(self, resource):
-        if resourceType is file:
+        data = resource.data
-            data = resource.read()
+        if type(data) is file:
+            text = resource.read()
             resource.close()
-        elif resourceType is str:
+        else:
-            data = resource
+            text = data
-        print 'Transferring:', data
+        metadata = resource.metadata
-        return Deferred()
+        url = self.serverURL + self.makePath(metadata)
-
+        d = getPage(url, method='PUT', postData=text)
+        return d
 

agent/transport/httpput.txt

@@ -0,0 +1,15 @@
+======================================================
+loops.agent.transport.httpput - The HTTP PUT Transport
+======================================================
+
+  ($Id$)
+
+  >>> from time import time
+
+  >>> from loops.agent.core import Agent
+  >>> from loops.agent.transport.httpput import Transporter, TransportJob
+
+  >>> agent = Agent()
+  >>> transporter = Transporter(agent)
+  >>> job = TransportJob(transporter)
+