From af16d445c66f0868fb2a76224cbfa08d6db9e49a Mon Sep 17 00:00:00 2001 From: scrat Date: Sun, 22 Jun 2008 15:58:38 +0000 Subject: [PATCH] further work on the transporter integration.added functionality for testing a simulated rpcserver similar to what has been done for outlook crawler. until now unfortunately using the testcase for the transporter does not work, because the params handed over in the doctest do not arrive at the Transporter object. This is why the doctest is not yet registered in cybertools.agent/test.py git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2718 fd906abe-77d9-0310-91a1-e0d9ade77398 --- agent/system/rpcapi.py | 33 +++++++++++++++ agent/testing/main_transport.py | 74 +++++++++++++++++++++++++++++++++ agent/testing/rpcserver.py | 47 +++++++-------------- agent/testing/transporter.cfg | 10 +++++ agent/transport/remote.py | 30 ++++++------- agent/transport/transporter.cfg | 8 ---- 6 files changed, 147 insertions(+), 55 deletions(-) create mode 100644 agent/system/rpcapi.py create mode 100644 agent/testing/main_transport.py create mode 100644 agent/testing/transporter.cfg delete mode 100644 agent/transport/transporter.cfg diff --git a/agent/system/rpcapi.py b/agent/system/rpcapi.py new file mode 100644 index 0000000..2a89df6 --- /dev/null +++ b/agent/system/rpcapi.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Configuration controlled import of twisted xmlrpc functionality + +$Id: rpcapi.py +""" + +def setup(config): + global xmlrpc + if config.transportserver.xmlrpc == 'testing': + from cybertools.agent.testing.rpcserver import RPCServer + else: + try: + from twisted.web import xmlrpc + except ImportError: + from cybertools.agent.testing.rpcserver import RPCServer \ No newline at end of file diff --git a/agent/testing/main_transport.py b/agent/testing/main_transport.py new file mode 100644 index 0000000..cd4e7a9 --- /dev/null +++ b/agent/testing/main_transport.py @@ -0,0 +1,74 @@ +#! /usr/bin/env python2.4 +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Agent application. + +$Id$ +""" + +import os +from twisted.internet import reactor + +from cybertools.agent.base.agent import Master + + +application = None # contains application object if started via twistd + + +def getConfig(): + agentHome = os.path.abspath(os.path.dirname(__file__)) + configName = 'transporter.cfg' + configFile = open(os.path.join(agentHome, configName)) + config = configFile.read() + configFile.close() + return config + + +def setup(configInfo=None): + if configInfo is None: + configInfo = getConfig() + master = Master(configInfo) + setupEnvironment(master.config) + master.setup() + print 'Starting agent application...' + print 'Using controllers %s.' % ', '.join(master.config.controller.names) + return master + + +def setupEnvironment(config): + from cybertools.agent.base import agent, control, job, log, schedule + from cybertools.agent.core import agent, control, schedule + from cybertools.agent.control import cmdline + from cybertools.agent.system import rpcapi + rpcapi.setup(config) + from cybertools.agent.transport import remote + + +def startReactor(): + reactor.run() + print 'Agent application has been stopped.' + + +if __name__ == '__main__': + master = setup() + controller = master.controllers[0] + controller.createAgent('transport.remote', 'sample03') + controller.enterJob('sample', 'sample03', params=dict(serverURL='', machineName='', method='', userName='', password='')) + startReactor() diff --git a/agent/testing/rpcserver.py b/agent/testing/rpcserver.py index 995567d..077e2ef 100644 --- a/agent/testing/rpcserver.py +++ b/agent/testing/rpcserver.py @@ -17,19 +17,12 @@ # """ -Providing access for remote agent instances by listening for requests -from remote transport agents. +Fake rpcserver for testing purposes $Id$ """ -from twisted.web import xmlrpc, server, resource -from twisted.internet import defer, reactor -from cybertools.agent.base.agent import Agent - -application = None - -class RPCServer(xmlrpc.XMLRPC): +class RPCServer(object): serverURL = '' method = '' @@ -37,7 +30,6 @@ class RPCServer(xmlrpc.XMLRPC): userName = '' password = '' controller = '' - close = reactor.stop def __init__(self, serverURL = '', method = '', machineName = '', userName = '', password = '', controlObj= None): @@ -47,33 +39,24 @@ class RPCServer(xmlrpc.XMLRPC): self.userName = userName self.password = password self.controller = controlObj - xmlrpc.XMLRPC.__init__(self) + + def callRemote(self, methodName, *params): + """ + intended to simulate the callRemote command of a real xmlrpcserver + that takes a method name and calls the method, returning the results + as xml formatted strings + """ + method = getattr(self, methodName) + return method(*params) - def xmlrpc_transfer(self, resource): - if self.controller is not None: - # pass resource object to controller - # this is done BEFORE the metadata is handed over - # call notify method of controller - pass - print resource - return "Resource received: ", resource - - def xmlrpc_getMetadata(self, metadata): + def getMetadata(self, metadata): if self.controller is not None: # pass metadata to controller # this is done AFTER the resource (like e.g. file or mail) # is handed over pass - print metadata - metadata = "Echo: ", metadata - return metadata + return "Metadata received!" def xmlrpc_shutdownRPCServer(): - self.close() - - -if __name__ == '__main__': - from twisted.internet import reactor - site = RPCServer() - reactor.listenTCP(8082, server.Site(site)) - reactor.run() \ No newline at end of file + return "xmlrRPC server shutdown completed!" + \ No newline at end of file diff --git a/agent/testing/transporter.cfg b/agent/testing/transporter.cfg new file mode 100644 index 0000000..4092e23 --- /dev/null +++ b/agent/testing/transporter.cfg @@ -0,0 +1,10 @@ +# +# sample.cfg - agent configuration for demonstration and testing purposes +# +# $Id$ +# + +controller(names=['core.sample']) +scheduler(name='core') +logger(name='default', standard=30) +transportserver.xmlrpc='testing' diff --git a/agent/transport/remote.py b/agent/transport/remote.py index 35c5c7a..3dc9e26 100644 --- a/agent/transport/remote.py +++ b/agent/transport/remote.py @@ -25,8 +25,9 @@ $Id$ """ from zope.interface import implements -from twisted.web import xmlrpc +from cybertools.agent.system import rpcapi +from cybertools.agent.base.agent import Master from cybertools.agent.core.agent import QueueableAgent from cybertools.agent.interfaces import ITransporter from cybertools.agent.crawl.base import Metadata @@ -36,7 +37,6 @@ from cybertools.agent.components import agents from cybertools.util.config import Configurator - class Transporter(QueueableAgent): implements(ITransporter) @@ -49,20 +49,19 @@ class Transporter(QueueableAgent): password = '' resource = None - def __init__(self, master, configuration): + def __init__(self, master, params): super(Transporter, self).__init__(master) - if isinstance(configuration, Configurator): - self.config = configuration - else: # configuration is path to config file - self.config = Configurator() - self.config.load(configuration) - - self.serverURL = self.config.xmlrpcClient.serverURL - self.server = xmlrpc.Proxy(self.serverURL) - self.method = self.config.xmlrpcClient.method - self.machineName = self.config.xmlrpcClient.machineName - self.userName = self.config.xmlrpcClient.userName - self.password = self.config.xmlrpcClient.password +## if isinstance(configuration, Configurator): +## self.config = configuration +## else: # configuration is path to config file +## self.config = Configurator() +## self.config.load(configuration) + self.serverURL = params[serverURL] + self.server = rpcapi.xmlrpc.Proxy(self.serverURL) + self.method = params[method] + self.machineName = params[machineName] + self.userName = params[userName] + self.password = params[password] def transfer(self, resource): """ Transfer the resource (an object providing IResource) @@ -86,6 +85,7 @@ class Transporter(QueueableAgent): This callback method is called when resource and metadata have been transferred successfully. """ + print successMessage pass # def process(self): diff --git a/agent/transport/transporter.cfg b/agent/transport/transporter.cfg deleted file mode 100644 index 37f40a9..0000000 --- a/agent/transport/transporter.cfg +++ /dev/null @@ -1,8 +0,0 @@ -# -# sample.cfg - agent configuration for demonstration and testing purposes -# -# $Id$ -# - -xmlrpcClient(serverURL='http://localhost:8082', method='PUT', machineName='sr', - userName='loopsAdmin', password='loops')