From f3002188d5096f20a9a3f62f8ffec07be7ef1917 Mon Sep 17 00:00:00 2001 From: helmutm Date: Fri, 10 Aug 2007 15:46:14 +0000 Subject: [PATCH] added simple server and client scripts for testing the HTTP transport; transport now basically working git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1905 fd906abe-77d9-0310-91a1-e0d9ade77398 --- agent/core.py | 4 ++- agent/crawl/base.py | 2 +- agent/schedule.py | 7 +++++ agent/testing/client.py | 56 ++++++++++++++++++++++++++++++++++++ agent/testing/server.py | 64 +++++++++++++++++++++++++++++++++++++++++ agent/transport/base.py | 26 ++++++++++------- 6 files changed, 147 insertions(+), 12 deletions(-) create mode 100644 agent/testing/client.py create mode 100644 agent/testing/server.py diff --git a/agent/core.py b/agent/core.py index c43309a..c464923 100644 --- a/agent/core.py +++ b/agent/core.py @@ -27,7 +27,7 @@ from zope.interface import implements from loops.agent.interfaces import IAgent from loops.agent.config import Configurator from loops.agent.crawl import filesystem -from loops.agent.schedule import Scheduler +from loops.agent.schedule import Scheduler, Stopper from loops.agent.transport import httpput @@ -51,6 +51,8 @@ class Agent(object): config = self.config = Configurator('ui', 'crawl', 'transport') config.load(conf) self.scheduler = Scheduler(self) + self.stopper = Stopper() + self.stopper.scheduler = self.scheduler def scheduleJobsFromConfig(self): config = self.config diff --git a/agent/crawl/base.py b/agent/crawl/base.py index 59417cb..04575fe 100644 --- a/agent/crawl/base.py +++ b/agent/crawl/base.py @@ -50,7 +50,7 @@ class Metadata(dict): for k in data: self[k] = data[k] - def asXml(self): + def asXML(self): # TODO... return '' diff --git a/agent/schedule.py b/agent/schedule.py index b45c33f..268e7c1 100644 --- a/agent/schedule.py +++ b/agent/schedule.py @@ -96,3 +96,10 @@ class Job(object): newJob.params = self.params newJob.successors = [s.copy() for s in self.successors] + +class Stopper(Job): + + def execute(self): + reactor.stop() + return succeed('Agent stopped.') + diff --git a/agent/testing/client.py b/agent/testing/client.py new file mode 100644 index 0000000..43e66b2 --- /dev/null +++ b/agent/testing/client.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +A dummy client application for testing purposes. + +Run this from above the loops directory with:: + + python loops/agent/testing/client.py + +$Id$ +""" + +import os +from time import time +from twisted.internet import reactor + +from loops.agent.core import Agent +from loops.agent.crawl.filesystem import CrawlingJob +from loops.agent.transport.base import Transporter +from loops.agent.tests import baseDir + +agent = Agent() +scheduler = agent.scheduler + +dirname = os.path.join(baseDir, 'testing', 'data') +crawlJob = CrawlingJob(directory=dirname) + +transporter = Transporter(agent) +transporter.serverURL = 'http://localhost:8123/loops' +transportJob = transporter.createJob() +crawlJob.successors.append(transportJob) +transportJob.successors.append(agent.stopper) + +scheduler.schedule(crawlJob) + +print 'Starting reactor.' + +reactor.run() + +print 'Reactor stopped.' diff --git a/agent/testing/server.py b/agent/testing/server.py new file mode 100644 index 0000000..9c8a84d --- /dev/null +++ b/agent/testing/server.py @@ -0,0 +1,64 @@ +# +# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +A dummy webserver for testing the loops.agent HTTP transport. + +Run this with:: + + twistd -noy loops/agent/testing/server.py + +$Id$ +""" + +from twisted.application import service, internet +from twisted.web import http + + +class RequestHandler(http.Request): + + def process(self): + print '***', self.uri + print '***', self.content.read() + if self.method in ('GET', 'POST'): + self.write('

Hello World

') + self.write('

dir(self): %s

' % dir(self)) + self.write('

self.path: %s

' % self.path) + self.write('

self.uri: %s

' % self.uri) + self.write('

self.args: %s

' % self.args) + self.finish() + + +class HttpServer(http.HTTPChannel): + + requestFactory = RequestHandler + + +class HttpFactory(http.HTTPFactory): + + protocol = HttpServer + + +class HttpService(internet.TCPServer): + + def __init__(self): + internet.TCPServer.__init__(self, 8123, HttpFactory()) + + +application = service.Application('Simple Webserver') +HttpService().setServiceParent(application) diff --git a/agent/transport/base.py b/agent/transport/base.py index c86e654..f9fb2c0 100644 --- a/agent/transport/base.py +++ b/agent/transport/base.py @@ -24,6 +24,7 @@ $Id$ from twisted.internet import reactor from twisted.internet.defer import Deferred, DeferredList, fail +from twisted.web.client import getPage from zope.interface import implements from loops.agent.interfaces import ITransporter, ITransportJob @@ -61,11 +62,11 @@ class Transporter(object): jobFactory = TransportJob - serverURL = None - method = None - machineName = None - userName = None - password = None + serverURL = 'http://localhost:8080' + method = 'PUT' + machineName = 'unknown' + userName = 'nobody' + password = 'secret' def __init__(self, agent): self.agent = agent @@ -76,10 +77,12 @@ class Transporter(object): def transfer(self, resource): data = resource.data if type(data) is file: - text = resource.read() - resource.close() + text = data.read() + data.close() else: text = data + # TODO: encode text + # TODO: set headers, esp Content-Type path = resource.path app = resource.application deferreds = [] @@ -87,13 +90,16 @@ class Transporter(object): if metadata is not None: url = self.makePath('meta', app, path, 'xml') deferreds.append( - getPage(url, method=self.method, postData=metadata.asXML())) + getPage(url, method=self.method, postdata=metadata.asXML())) url = self.makePath('data', app, path) - deferreds.append(getPage(url, method=self.method, postData=text)) + deferreds.append(getPage(url, method=self.method, postdata=text)) return DeferredList(deferreds) def makePath(self, infoType, app, path, extension=None): - fullPath = '/'.join((self.serverURL, infoType, app, path)) + if path.startswith('/'): + path = path[1:] + fullPath = '/'.join((self.serverURL, infoType, + self.machineName, self.userName, app, path)) if extension: fullPath += '.' + extension return fullPath