From e0503fea5b670b5a9b5f9b053f593c22df19086f Mon Sep 17 00:00:00 2001 From: helmutm Date: Fri, 17 Oct 2008 20:46:28 +0000 Subject: [PATCH] add 'talk' package providing a common communication framework git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2923 fd906abe-77d9-0310-91a1-e0d9ade77398 --- agent/main.py | 14 +++-- agent/system/http.py | 31 +++++++++++ agent/system/{sftpapi.py => sftp.py} | 2 +- agent/system/{rpcapi.py => xmlrpc.py} | 4 +- agent/talk/README.txt | 23 ++++++++ agent/talk/__init__.py | 4 ++ agent/talk/http.py | 55 +++++++++++++++++++ .../ajaxclient.py => testing/http.py} | 28 ++++++---- agent/tests.py | 1 + agent/transport/file/sftp.py | 1 + agent/transport/remote.py | 8 +-- agent/transport/transporter.txt | 8 +-- 12 files changed, 150 insertions(+), 29 deletions(-) create mode 100644 agent/system/http.py rename agent/system/{sftpapi.py => sftp.py} (94%) rename agent/system/{rpcapi.py => xmlrpc.py} (89%) create mode 100644 agent/talk/README.txt create mode 100644 agent/talk/__init__.py create mode 100644 agent/talk/http.py rename agent/{control/ajaxclient.py => testing/http.py} (58%) diff --git a/agent/main.py b/agent/main.py index ec167c9..144ea5b 100755 --- a/agent/main.py +++ b/agent/main.py @@ -53,17 +53,19 @@ def setup(configInfo=None): def setupEnvironment(config): + # self registration of components: from cybertools.agent.base import agent, control, job, log, schedule from cybertools.agent.core import agent, control, schedule - from cybertools.agent.control import cmdline, remote, ajaxclient + from cybertools.agent.control import cmdline, remote + from cybertools.agent.crawl import base, filesystem, outlook from cybertools.agent.transport import remote, loops + # API registration: from cybertools.agent.system.windows import api api.setup(config) - from cybertools.agent.system import rpcapi - rpcapi.setup(config) - from cybertools.agent.system import sftpapi - sftpapi.setup(config) - from cybertools.agent.crawl import base, filesystem, outlook + from cybertools.agent.system import http, xmlrpc, sftp + http.setup(config) + xmlrpc.setup(config) + sftp.setup(config) def startReactor(): diff --git a/agent/system/http.py b/agent/system/http.py new file mode 100644 index 0000000..bfae41a --- /dev/null +++ b/agent/system/http.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Configuration-controlled import of HTTP communication functionality. + +$Id: rpcapi.py +""" + +def setup(config): + global listener, getPage + if config.talk.http == 'testing': + from cybertools.agent.testing.http import listener, getPage + else: + from twisted.internet import reactor as listener + from twisted.web.client import getPage diff --git a/agent/system/sftpapi.py b/agent/system/sftp.py similarity index 94% rename from agent/system/sftpapi.py rename to agent/system/sftp.py index d393655..3d8259d 100644 --- a/agent/system/sftpapi.py +++ b/agent/system/sftp.py @@ -17,7 +17,7 @@ # """ -Configuration controlled import of sftp functionality +Configuration-controlled import of sftp functionality. $Id: rpcapi.py """ diff --git a/agent/system/rpcapi.py b/agent/system/xmlrpc.py similarity index 89% rename from agent/system/rpcapi.py rename to agent/system/xmlrpc.py index b321e35..2fa7193 100644 --- a/agent/system/rpcapi.py +++ b/agent/system/xmlrpc.py @@ -25,6 +25,6 @@ $Id: rpcapi.py def setup(config): global xmlrpc if config.transport.remote.server == 'testing': - from cybertools.agent.testing.rpcserver import RPCServer, xmlrpc + from cybertools.agent.testing.rpcserver import xmlrpc else: - from twisted.web import xmlrpc \ No newline at end of file + from twisted.web import xmlrpc diff --git a/agent/talk/README.txt b/agent/talk/README.txt new file mode 100644 index 0000000..20039c3 --- /dev/null +++ b/agent/talk/README.txt @@ -0,0 +1,23 @@ +================================================ +Agents for Job Execution and Communication Tasks +================================================ + + ($Id$) + + >>> config = ''' + ... controller(names=['core.sample']) + ... scheduler(name='core') + ... logger(name='default', standard=30) + ... talk.http = 'testing' + ... ''' + >>> from cybertools.agent.main import setup + >>> master = setup(config) + Starting agent application... + Using controllers core.sample. + + +Communication Handling +====================== + + >>> from cybertools.agent.talk.http import Handler + diff --git a/agent/talk/__init__.py b/agent/talk/__init__.py new file mode 100644 index 0000000..4bc90fb --- /dev/null +++ b/agent/talk/__init__.py @@ -0,0 +1,4 @@ +""" +$Id$ +""" + diff --git a/agent/talk/http.py b/agent/talk/http.py new file mode 100644 index 0000000..ecb0471 --- /dev/null +++ b/agent/talk/http.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Handling asynchronous and possibly asymmetric communication tasks. + +$Id$ +""" + +from twisted.internet import defer +from twisted.web.resource import Resource +from twisted.web.server import Site +from zope.interface import implements + +from cybertools.agent.base.agent import Master +from cybertools.agent.system.http import listener + + +class RootResource(Resource): + + def getChild(self, path, request): + return CommandHandler(path) + + +class CommandHandler(Resource): + + def __init__(self, path): + self.command = path + + def render(self, request): + return '{"message": "OK"}' + + +class Handler(object): + + def listen(self, port): + return listener.listenTCP(port, Site(RootResource())) + + def send(self, clientId, data): + return defer.Deferred() diff --git a/agent/control/ajaxclient.py b/agent/testing/http.py similarity index 58% rename from agent/control/ajaxclient.py rename to agent/testing/http.py index 1f4b1dd..db93678 100644 --- a/agent/control/ajaxclient.py +++ b/agent/testing/http.py @@ -17,24 +17,28 @@ # """ -Controller that receives and responds to requests from a browser (AJAX) client; -in parallel it sends informations to the client by responding to -polling requests from the client. +Fake testing objects/functions for HTTP communication. $Id$ """ -from zope.interface import implements - -from cybertools.agent.base.agent import Master -from cybertools.agent.core.control import SampleController -from cybertools.agent.components import controllers +from twisted.internet.defer import Deferred, succeed -class ClientController(SampleController): +class Listener(object): - def setup(self): - super(ClientController, self).setup() + site = port = None + + def listenTCP(self, port, site): + self.port = port + self.site = site + self.resource = site.resource + deferred = self.deferred = Deferred() + return deferred -controllers.register(ClientController, Master, name='ajaxclient') +listener = Listener() + + +def getPage(url, contextFactory=None, method='GET', postdata=None, **kwargs): + return succeed('{"message": "OK"}') diff --git a/agent/tests.py b/agent/tests.py index f9f1408..f73ebe6 100755 --- a/agent/tests.py +++ b/agent/tests.py @@ -42,6 +42,7 @@ def test_suite(): testSuite = unittest.TestSuite(( unittest.makeSuite(Test), DocFileSuite('README.txt', optionflags=flags), + DocFileSuite('talk/README.txt', optionflags=flags), DocFileSuite('crawl/README.txt', optionflags=flags), DocFileSuite('crawl/filesystem.txt', optionflags=flags), DocFileSuite('crawl/outlook.txt', optionflags=flags), diff --git a/agent/transport/file/sftp.py b/agent/transport/file/sftp.py index 7fd8f91..d9169de 100644 --- a/agent/transport/file/sftp.py +++ b/agent/transport/file/sftp.py @@ -28,6 +28,7 @@ from twisted.internet import defer, protocol, reactor CHUNKSIZE = 4096 + class FileTransfer(protocol.ClientFactory): """ Transfers files to a remote SCP/SFTP server. """ diff --git a/agent/transport/remote.py b/agent/transport/remote.py index 56f4617..840d8b7 100644 --- a/agent/transport/remote.py +++ b/agent/transport/remote.py @@ -28,8 +28,8 @@ from twisted.internet import defer from zope.interface import implements import os -from cybertools.agent.system import rpcapi -from cybertools.agent.system import sftpapi +from cybertools.agent.system import xmlrpc +from cybertools.agent.system import sftp from cybertools.agent.base.agent import Master from cybertools.agent.core.agent import QueueableAgent from cybertools.agent.interfaces import ITransporter @@ -51,11 +51,11 @@ class Transporter(QueueableAgent): super(Transporter, self).__init__(master) config = master.config serverURL = config.transport.remote.url - self.server = rpcapi.xmlrpc.Proxy(serverURL) + self.server = xmlrpc.xmlrpc.Proxy(serverURL) userName = config.transport.remote.ftp.user password = config.transport.remote.ftp.password host = config.transport.remote.ftp.url - self.ftpServer = sftpapi.FileTransfer(host, self.port, userName, password) + self.ftpServer = sftp.FileTransfer(host, self.port, userName, password) def process(self): return self.transfer(self.params['resource']) diff --git a/agent/transport/transporter.txt b/agent/transport/transporter.txt index f16ac86..bce0ed6 100644 --- a/agent/transport/transporter.txt +++ b/agent/transport/transporter.txt @@ -19,13 +19,13 @@ Agents for Job Execution and Communication Tasks Transporter -============== +=========== The agent uses Twisted's cooperative multitasking model. -The Transporter is used to contact an xmlrpc Server and transmit the metadata to the other -loops system. The Transporter is derived from Queueable agent, because also here applies that only one -item at a time is transmitted. +The Transporter is used to contact an xmlrpc Server and transmit the metadata +to the other loops system. The Transporter is derived from Queueable agent +to ensure that only one item at a time is transmitted. Returns a deferred that must be supplied with a callback method (and in most cases also an errback method).