further work on the transporter integration.added functionality for testing a simulated rpcserver similar to what has been done for outlook crawler.

until now unfortunately using the testcase for the transporter does not work, because the params handed over in the doctest do not arrive at the Transporter object. This is why the doctest is not yet registered in cybertools.agent/test.py

git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2718 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/system/rpcapi.py

@@ -0,0 +1,33 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Configuration controlled import of twisted xmlrpc functionality
+
+$Id: rpcapi.py
+"""
+
+def setup(config):
+    global xmlrpc
+    if config.transportserver.xmlrpc == 'testing':
+        from cybertools.agent.testing.rpcserver import RPCServer
+    else:
+        try:
+            from twisted.web import xmlrpc
+        except ImportError:
+            from cybertools.agent.testing.rpcserver import RPCServer

agent/testing/main_transport.py

@@ -0,0 +1,74 @@
+#! /usr/bin/env python2.4
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Agent application.
+
+$Id$
+"""
+
+import os
+from twisted.internet import reactor
+
+from cybertools.agent.base.agent import Master
+
+
+application = None  # contains application object if started via twistd
+
+
+def getConfig():
+    agentHome = os.path.abspath(os.path.dirname(__file__))
+    configName = 'transporter.cfg'
+    configFile = open(os.path.join(agentHome, configName))
+    config = configFile.read()
+    configFile.close()
+    return config
+
+
+def setup(configInfo=None):
+    if configInfo is None:
+        configInfo = getConfig()
+    master = Master(configInfo)
+    setupEnvironment(master.config)
+    master.setup()
+    print 'Starting agent application...'
+    print 'Using controllers %s.' % ', '.join(master.config.controller.names)
+    return master
+
+
+def setupEnvironment(config):
+    from cybertools.agent.base import agent, control, job, log, schedule
+    from cybertools.agent.core import agent, control, schedule
+    from cybertools.agent.control import cmdline
+    from cybertools.agent.system import rpcapi
+    rpcapi.setup(config)
+    from cybertools.agent.transport import remote
+
+
+def startReactor():
+    reactor.run()
+    print 'Agent application has been stopped.'
+
+
+if __name__ == '__main__':
+    master = setup()
+    controller = master.controllers[0]
+    controller.createAgent('transport.remote', 'sample03')
+    controller.enterJob('sample', 'sample03', params=dict(serverURL='', machineName='', method='', userName='', password=''))
+    startReactor()

agent/testing/rpcserver.py

@@ -17,19 +17,12 @@
 #
 
 """
-Providing access for remote agent instances by listening for requests
-from remote transport agents.
+Fake rpcserver for testing purposes
 
 $Id$
 """
 
-from twisted.web import xmlrpc, server, resource
-from twisted.internet import defer, reactor
-from cybertools.agent.base.agent import Agent
-
-application = None
-
-class RPCServer(xmlrpc.XMLRPC):
+class RPCServer(object):
 
     serverURL = ''
     method = ''
@@ -37,7 +30,6 @@ class RPCServer(xmlrpc.XMLRPC):
     userName = ''
     password = ''
     controller = ''
-    close = reactor.stop
 
     def __init__(self, serverURL = '', method = '', machineName = '',
                  userName = '', password = '', controlObj= None):
@@ -47,33 +39,24 @@ class RPCServer(xmlrpc.XMLRPC):
         self.userName = userName
         self.password = password
         self.controller = controlObj
-        xmlrpc.XMLRPC.__init__(self)
+        
+    def callRemote(self, methodName, *params):
+        """
+        intended to simulate the callRemote command of a real xmlrpcserver
+        that takes a method name and calls the method, returning the results
+        as xml formatted strings
+        """
+        method = getattr(self, methodName)
+        return method(*params)
 
-    def xmlrpc_transfer(self, resource):
-        if self.controller is not None:
-            # pass resource object to controller
-            # this is done BEFORE the metadata is handed over
-            # call notify method of controller
-            pass
-        print resource
-        return "Resource received: ", resource
-
-    def xmlrpc_getMetadata(self, metadata):
+    def getMetadata(self, metadata):
         if self.controller is not None:
             # pass metadata to controller
             # this is done AFTER the resource (like e.g. file or mail)
             # is handed over
             pass
-        print metadata
-        metadata = "Echo: ", metadata
-        return metadata
+        return "Metadata received!"
 
     def xmlrpc_shutdownRPCServer():
-        self.close()
-
-
-if __name__ == '__main__':
-    from twisted.internet import reactor
-    site = RPCServer()
-    reactor.listenTCP(8082, server.Site(site))
-    reactor.run()
+        return "xmlrRPC server shutdown completed!"
+    

agent/testing/transporter.cfg

@@ -0,0 +1,10 @@
+#
+#   sample.cfg - agent configuration for demonstration and testing purposes
+#
+#   $Id$
+#
+
+controller(names=['core.sample'])
+scheduler(name='core')
+logger(name='default', standard=30)
+transportserver.xmlrpc='testing'

agent/transport/remote.py

@@ -25,8 +25,9 @@ $Id$
 """
 
 from zope.interface import implements
-from twisted.web import xmlrpc
+from cybertools.agent.system import rpcapi
 
+from cybertools.agent.base.agent import Master
 from cybertools.agent.core.agent import QueueableAgent
 from cybertools.agent.interfaces import ITransporter
 from cybertools.agent.crawl.base import Metadata
@@ -36,7 +37,6 @@ from cybertools.agent.components import agents
 from cybertools.util.config import Configurator
 
 
-
 class Transporter(QueueableAgent):
 
     implements(ITransporter)
@@ -49,20 +49,19 @@ class Transporter(QueueableAgent):
     password = ''
     resource = None
 
-    def __init__(self, master, configuration):
+    def __init__(self, master, params):
         super(Transporter, self).__init__(master)
-        if isinstance(configuration, Configurator):
-            self.config = configuration
-        else:   # configuration is path to config file
-            self.config = Configurator()
-            self.config.load(configuration)
-            
-        self.serverURL = self.config.xmlrpcClient.serverURL
-        self.server = xmlrpc.Proxy(self.serverURL)
-        self.method = self.config.xmlrpcClient.method
-        self.machineName = self.config.xmlrpcClient.machineName
-        self.userName = self.config.xmlrpcClient.userName
-        self.password = self.config.xmlrpcClient.password
+##        if isinstance(configuration, Configurator):
+##            self.config = configuration
+##        else:   # configuration is path to config file
+##            self.config = Configurator()
+##            self.config.load(configuration)
+        self.serverURL = params[serverURL]
+        self.server = rpcapi.xmlrpc.Proxy(self.serverURL)
+        self.method = params[method]
+        self.machineName = params[machineName]
+        self.userName = params[userName]
+        self.password = params[password]
 
     def transfer(self, resource):
         """ Transfer the resource (an object providing IResource)
@@ -86,6 +85,7 @@ class Transporter(QueueableAgent):
         This callback method is called when resource and metadata
         have been transferred successfully.
         """
+        print successMessage
         pass
 
 #    def process(self):

agent/transport/transporter.cfg

@@ -1,8 +0,0 @@
-#
-#   sample.cfg - agent configuration for demonstration and testing purposes
-#
-#   $Id$
-#
-
-xmlrpcClient(serverURL='http://localhost:8082', method='PUT', machineName='sr',
-              userName='loopsAdmin', password='loops')