diff --git a/agent/crawl/filesystem.py b/agent/crawl/filesystem.py index bfc804d..b85c475 100644 --- a/agent/crawl/filesystem.py +++ b/agent/crawl/filesystem.py @@ -73,6 +73,8 @@ class FileResource(object): self.path = path self.metadata = metadata + application = 'filesystem' + @property def data(self): return open(self.path, 'r') diff --git a/agent/crawl/filesystem.txt b/agent/crawl/filesystem.txt index 26a63b9..079dafa 100644 --- a/agent/crawl/filesystem.txt +++ b/agent/crawl/filesystem.txt @@ -27,7 +27,7 @@ transferred. >>> from loops.agent.testing import transport >>> transporter = transport.Transporter(agent) - >>> transportJob = transporter.jobFactory(transporter) + >>> transportJob = transporter.createJob() >>> crawlJob.successors.append(transportJob) We are now ready to schedule the job and let the reactor execute it. diff --git a/agent/interfaces.py b/agent/interfaces.py index bbabd5b..5b240b9 100644 --- a/agent/interfaces.py +++ b/agent/interfaces.py @@ -30,7 +30,9 @@ class IAgent(Interface): and transfers these to its server. """ - scheduler = Attribute('IScheduler instance') + scheduler = Attribute('IScheduler instance.') + transporter = Attribute('The transporter to be used for transferring ' + 'objects.') class IScheduler(Interface): @@ -111,13 +113,17 @@ class IResource(Interface): data = Attribute("A string, file, or similar representation of the " "resource's content") - + path = Attribute('A filesystem path or some other information ' + 'uniquely identifying the resource on the client ' + 'machine for the current user.') + application = Attribute('The name of the application that provided ' + 'the resource.') metadata = Attribute('Information describing this resource; ' 'should be an IMetadataSet object.') class IMetadataSet(Interface): - """ Metadata associated with a resource; sort of a mapping. + """ Metadata associated with a resource; a mapping. """ def asXML(): @@ -148,9 +154,13 @@ class ITransporter(Interface): userName = Attribute('User name for logging in to the server.') password = Attribute('Password for logging in to the server.') + def createJob(): + """ Return a transport job for this transporter. + """ + def transfer(resource): """ Transfer the resource (an object providing IResource) - to the server. + to the server and return a Deferred. """ @@ -185,6 +195,8 @@ class IConfigurator(Interface): """ +# future extensions + class IPackageManager(Interface): """ Allows to install, update, or remove software packages (plugins, typically as Python eggs) from a server. diff --git a/agent/testing/crawl.py b/agent/testing/crawl.py index 0452eef..66952d5 100644 --- a/agent/testing/crawl.py +++ b/agent/testing/crawl.py @@ -26,7 +26,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred from zope.interface import implements -from loops.agent.interfaces import ICrawlingJob, IResource +from loops.agent.interfaces import IResource from loops.agent.crawl.base import CrawlingJob as BaseCrawlingJob @@ -47,4 +47,6 @@ class DummyResource(object): implements(IResource) data = 'Dummy resource data for testing purposes.' + path = '/dummy/data' + application = 'dummy' metadata = None diff --git a/agent/testing/transport.py b/agent/testing/transport.py index 47274cd..508b61f 100644 --- a/agent/testing/transport.py +++ b/agent/testing/transport.py @@ -27,28 +27,11 @@ from twisted.internet.defer import Deferred from zope.interface import implements from loops.agent.interfaces import ITransportJob, ITransporter -from loops.agent.schedule import Job -from loops.agent.transport.base import TransportJob as BaseJob from loops.agent.transport.base import Transporter as BaseTransporter -class TransportJob(BaseJob): - - def execute(self): - kw = self.params - result = kw.get('result') - if result is None: - print 'No data available.' - else: - for resource in result: - d = self.transporter.transfer(resource) - return Deferred() - - class Transporter(BaseTransporter): - jobFactory = TransportJob - def transfer(self, resource): data = resource.data if type(data) is file: diff --git a/agent/transport/base.py b/agent/transport/base.py index 56f89fe..0099c81 100644 --- a/agent/transport/base.py +++ b/agent/transport/base.py @@ -23,7 +23,7 @@ $Id$ """ from twisted.internet import reactor -from twisted.internet.defer import Deferred +from twisted.internet.defer import Deferred, DeferredList, fail from zope.interface import implements from loops.agent.interfaces import ITransporter, ITransportJob @@ -38,6 +38,22 @@ class TransportJob(Job): super(TransportJob, self).__init__() self.transporter = transporter + def execute(self): + result = self.params.get('result') + if result is None: + return fail('No data available.') + transfers = [] + for resource in result: + d = self.transporter.transfer(resource) + transfers.append(d) + d.addCallback(self.logTransfer) + return DeferredList(transfers) + + def logTransfer(self, result): + # TODO: logging + # self.transporter.agent.logger.log(...) + pass + class Transporter(object): @@ -45,21 +61,37 @@ class Transporter(object): jobFactory = TransportJob + serverURL = None + method = None + machineName = None + userName = None + password = None + def __init__(self, agent): self.agent = agent + def createJob(self): + return self.jobFactory(self) + def transfer(self, resource): - """ Should be overridden by subclass - this is just an example - how it may look like. - """ data = resource.data if type(data) is file: - text = data.read() - data.close() + text = resource.read() + resource.close() else: text = data + path = resource.path + app = resource.application + deferreds = [] metadata = resource.metadata - d = Deferred() # or call something that returns a deferred - return d + if metadata is not None: + url = self.makePath('meta', app, path) + deferreds.append( + getPage(url, method=self.method, postData=metadata.asXML())) + url = self.makePath('data', app, path) + deferreds.append(getPage(url, method=self.method, postData=text)) + return DeferredList(deferreds) + def makePath(self, infoType, app, path): + return '/'.join((self.serverURL, infoType, app, path)) diff --git a/agent/transport/httpput.py b/agent/transport/httpput.py deleted file mode 100644 index fe3cf71..0000000 --- a/agent/transport/httpput.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) 2007 Helmut Merz helmutm@cy55.de -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# - -""" -Transferring of data/files to the server. - -$Id$ -""" - -from twisted.internet import reactor -from twisted.web.client import getPage -from zope.interface import implements - -from loops.agent.interfaces import ITransporter, ITransportJob -from loops.agent.schedule import Job - - -class TransportJob(Job): - - implements(ITransportJob) - - def __init__(self, transporter): - super(TransportJob, self).__init__() - self.transporter = transporter - - def execute(self): - result = kw.get('result') - if result is None: - print 'No data available.' - else: - for resource, metadata in result: - d = self.transporter.transfer(resource.data, metadata) - return Deferred() - - -class Transporter(object): - - implements(ITransporter) - - jobFactory = TransportJob - - serverURL = None - method = None - machineName = None - userName = None - password = None - - def __init__(self, agent): - self.agent = agent - conf = agent.config - # TODO: get settings from conf - - def transfer(self, resource): - data = resource.data - if type(data) is file: - text = resource.read() - resource.close() - else: - text = data - metadata = resource.metadata - url = self.serverURL + self.makePath(metadata) - d = getPage(url, method='PUT', postData=text) - return d - diff --git a/agent/transport/httpput.txt b/agent/transport/httpput.txt index 343220a..da808a4 100644 --- a/agent/transport/httpput.txt +++ b/agent/transport/httpput.txt @@ -7,9 +7,9 @@ loops.agent.transport.httpput - The HTTP PUT Transport >>> from time import time >>> from loops.agent.core import Agent - >>> from loops.agent.transport.httpput import Transporter, TransportJob + >>> from loops.agent.transport.base import Transporter >>> agent = Agent() >>> transporter = Transporter(agent) - >>> job = TransportJob(transporter) + >>> job = transporter.createJob()