extend base.Transporter class to full-blown HTTP transporter, thus eliminating the httpput module

git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1901 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/crawl/filesystem.py

@@ -73,6 +73,8 @@ class FileResource(object):
         self.path = path
         self.metadata = metadata
 
+    application = 'filesystem'
+
     @property
     def data(self):
         return open(self.path, 'r')

agent/crawl/filesystem.txt

@@ -27,7 +27,7 @@ transferred.
 
   >>> from loops.agent.testing import transport
   >>> transporter = transport.Transporter(agent)
-  >>> transportJob = transporter.jobFactory(transporter)
+  >>> transportJob = transporter.createJob()
   >>> crawlJob.successors.append(transportJob)
 
 We are now ready to schedule the job and let the reactor execute it.

agent/interfaces.py

@@ -30,7 +30,9 @@ class IAgent(Interface):
         and transfers these to its server.
     """
 
-    scheduler = Attribute('IScheduler instance')
+    scheduler = Attribute('IScheduler instance.')
+    transporter = Attribute('The transporter to be used for transferring '
+                            'objects.')
 
 
 class IScheduler(Interface):
@@ -111,13 +113,17 @@ class IResource(Interface):
 
     data = Attribute("A string, file, or similar representation of the "
                      "resource's content")
-
+    path = Attribute('A filesystem path or some other information '
+                     'uniquely identifying the resource on the client '
+                     'machine for the current user.')
+    application = Attribute('The name of the application that provided '
+                            'the resource.')
     metadata = Attribute('Information describing this resource; '
                          'should be an IMetadataSet object.')
 
 
 class IMetadataSet(Interface):
-    """ Metadata associated with a resource; sort of a mapping.
+    """ Metadata associated with a resource; a mapping.
     """
 
     def asXML():
@@ -148,9 +154,13 @@ class ITransporter(Interface):
     userName = Attribute('User name for logging in to the server.')
     password = Attribute('Password for logging in to the server.')
 
+    def createJob():
+        """ Return a transport job for this transporter.
+        """
+
     def transfer(resource):
         """ Transfer the resource (an object providing IResource)
-            to the server.
+            to the server and return a Deferred.
         """
 
 
@@ -185,6 +195,8 @@ class IConfigurator(Interface):
         """
 
 
+# future extensions
+
 class IPackageManager(Interface):
     """ Allows to install, update, or remove software packages (plugins,
         typically as Python eggs) from a server.

agent/testing/crawl.py

@@ -26,7 +26,7 @@ from twisted.internet import reactor
 from twisted.internet.defer import Deferred
 from zope.interface import implements
 
-from loops.agent.interfaces import ICrawlingJob, IResource
+from loops.agent.interfaces import IResource
 from loops.agent.crawl.base import CrawlingJob as BaseCrawlingJob
 
 
@@ -47,4 +47,6 @@ class DummyResource(object):
     implements(IResource)
 
     data = 'Dummy resource data for testing purposes.'
+    path = '/dummy/data'
+    application = 'dummy'
     metadata = None

agent/testing/transport.py

@@ -27,28 +27,11 @@ from twisted.internet.defer import Deferred
 from zope.interface import implements
 
 from loops.agent.interfaces import ITransportJob, ITransporter
-from loops.agent.schedule import Job
-from loops.agent.transport.base import TransportJob as BaseJob
 from loops.agent.transport.base import Transporter as BaseTransporter
 
 
-class TransportJob(BaseJob):
-
-    def execute(self):
-        kw = self.params
-        result = kw.get('result')
-        if result is None:
-            print 'No data available.'
-        else:
-            for resource in result:
-                d = self.transporter.transfer(resource)
-        return Deferred()
-
-
 class Transporter(BaseTransporter):
 
-    jobFactory = TransportJob
-
     def transfer(self, resource):
         data = resource.data
         if type(data) is file:

agent/transport/base.py

@@ -23,7 +23,7 @@ $Id$
 """
 
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, DeferredList, fail
 from zope.interface import implements
 
 from loops.agent.interfaces import ITransporter, ITransportJob
@@ -38,6 +38,22 @@ class TransportJob(Job):
         super(TransportJob, self).__init__()
         self.transporter = transporter
 
+    def execute(self):
+        result = self.params.get('result')
+        if result is None:
+            return fail('No data available.')
+        transfers = []
+        for resource in result:
+            d = self.transporter.transfer(resource)
+            transfers.append(d)
+            d.addCallback(self.logTransfer)
+        return DeferredList(transfers)
+
+    def logTransfer(self, result):
+        # TODO: logging
+        # self.transporter.agent.logger.log(...)
+        pass
+
 
 class Transporter(object):
 
@@ -45,21 +61,37 @@ class Transporter(object):
 
     jobFactory = TransportJob
 
+    serverURL = None
+    method = None
+    machineName = None
+    userName = None
+    password = None
+
     def __init__(self, agent):
         self.agent = agent
 
+    def createJob(self):
+        return self.jobFactory(self)
+
     def transfer(self, resource):
-        """ Should be overridden by subclass - this is just an example
-            how it may look like.
-        """
         data = resource.data
         if type(data) is file:
-            text = data.read()
+            text = resource.read()
-            data.close()
+            resource.close()
         else:
             text = data
+        path = resource.path
+        app = resource.application
+        deferreds = []
         metadata = resource.metadata
-        d = Deferred()  # or call something that returns a deferred
+        if metadata is not None:
-        return d
+            url = self.makePath('meta', app, path)
+            deferreds.append(
+                    getPage(url, method=self.method, postData=metadata.asXML()))
+        url = self.makePath('data', app, path)
+        deferreds.append(getPage(url, method=self.method, postData=text))
+        return DeferredList(deferreds)
 
+    def makePath(self, infoType, app, path):
+        return '/'.join((self.serverURL, infoType, app, path))
 

agent/transport/httpput.py

@@ -1,79 +0,0 @@
-#
-#  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
-#
-#  This program is free software; you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation; either version 2 of the License, or
-#  (at your option) any later version.
-#
-#  This program is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with this program; if not, write to the Free Software
-#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-#
-
-"""
-Transferring of data/files to the server.
-
-$Id$
-"""
-
-from twisted.internet import reactor
-from twisted.web.client import getPage
-from zope.interface import implements
-
-from loops.agent.interfaces import ITransporter, ITransportJob
-from loops.agent.schedule import Job
-
-
-class TransportJob(Job):
-
-    implements(ITransportJob)
-
-    def __init__(self, transporter):
-        super(TransportJob, self).__init__()
-        self.transporter = transporter
-
-    def execute(self):
-        result = kw.get('result')
-        if result is None:
-            print 'No data available.'
-        else:
-            for resource, metadata in result:
-                d = self.transporter.transfer(resource.data, metadata)
-        return Deferred()
-
-
-class Transporter(object):
-
-    implements(ITransporter)
-
-    jobFactory = TransportJob
-
-    serverURL = None
-    method = None
-    machineName = None
-    userName = None
-    password = None
-
-    def __init__(self, agent):
-        self.agent = agent
-        conf = agent.config
-        # TODO: get settings from conf
-
-    def transfer(self, resource):
-        data = resource.data
-        if type(data) is file:
-            text = resource.read()
-            resource.close()
-        else:
-            text = data
-        metadata = resource.metadata
-        url = self.serverURL + self.makePath(metadata)
-        d = getPage(url, method='PUT', postData=text)
-        return d
-

agent/transport/httpput.txt

@@ -7,9 +7,9 @@ loops.agent.transport.httpput - The HTTP PUT Transport
   >>> from time import time
 
   >>> from loops.agent.core import Agent
-  >>> from loops.agent.transport.httpput import Transporter, TransportJob
+  >>> from loops.agent.transport.base import Transporter
 
   >>> agent = Agent()
   >>> transporter = Transporter(agent)
-  >>> job = TransportJob(transporter)
+  >>> job = transporter.createJob()