From 621efb60812e9897d69ba2891b24ee8b786d8e11 Mon Sep 17 00:00:00 2001 From: helmutm Date: Fri, 22 Jun 2007 15:15:25 +0000 Subject: [PATCH] set up dummy crawler and transport for testing; implement crawl-transfer sequence via job scheduler git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@1800 fd906abe-77d9-0310-91a1-e0d9ade77398 --- agent/README.txt | 61 ++++++++++++++++++++++------------ agent/interfaces.py | 17 ++++++++-- agent/schedule.py | 3 +- agent/testing/__init__.py | 4 +++ agent/testing/crawl.py | 60 +++++++++++++++++++++++++++++++++ agent/testing/transport.py | 68 ++++++++++++++++++++++++++++++++++++++ agent/tests.py | 19 +++++++++-- 7 files changed, 205 insertions(+), 27 deletions(-) create mode 100644 agent/testing/__init__.py create mode 100644 agent/testing/crawl.py create mode 100644 agent/testing/transport.py diff --git a/agent/README.txt b/agent/README.txt index 679272e..fd2ffe7 100644 --- a/agent/README.txt +++ b/agent/README.txt @@ -10,6 +10,12 @@ collecting informations and transferring them to the loops server. This package does not depend on zope or the other loops packages but represents a standalone application. +But we need a reactor for working with Twisted; in order not to block +testing when running the reactor we use reactor.iterate() calls +wrapped in a ``tester`` object. + + >>> from loops.agent.tests import tester + Basic Implementation, Agent Core ================================ @@ -24,23 +30,6 @@ most cases also an errback method). >>> agent = Agent() -Browser-based User Interface -============================ - -The user interface is provided via a browser-based application -based on Twisted and Nevow. - - -Configuration Management -======================== - -Functionality - -- Storage of configuration parameters -- Interface to the browser-based user interface that allows the - editing of configuration parameters - - Scheduling ========== @@ -60,11 +49,9 @@ Configuration (per job) ... print 'executing' ... return d - >>> scheduler.schedule(TestJob(), int(time())+1) + >>> scheduler.schedule(TestJob(), int(time())) - >>> from twisted.internet import reactor - >>> ignore = reactor.callLater(2, reactor.stop) - >>> reactor.run() + >>> tester.iterate() executing @@ -85,6 +72,21 @@ Configuration (per crawl job) - predefined metadata +The Dummy Crawler +----------------- + + >>> from testing.crawl import CrawlingJob + >>> from testing.transport import Transporter, TransportJob + + >>> crawl = CrawlingJob() + >>> transporter = Transporter() + >>> transport = TransportJob(transporter) + >>> crawl.successors.append(transport) + >>> scheduler.schedule(crawl, int(time())) + + >>> tester.iterate() + Transferring: Dummy resource data for testing purposes. + Local File System ----------------- @@ -181,3 +183,20 @@ Configuration (per install/update job) - command: install, update, remove - package names + +Configuration Management +======================== + +Functionality + +- Storage of configuration parameters +- Interface to the browser-based user interface that allows the + editing of configuration parameters + + +Browser-based User Interface +============================ + +The user interface is provided via a browser-based application +based on Twisted and Nevow. + diff --git a/agent/interfaces.py b/agent/interfaces.py index 13b2e16..eb2a1be 100644 --- a/agent/interfaces.py +++ b/agent/interfaces.py @@ -43,7 +43,7 @@ class IScheduler(Interface): def schedule(job, startTime): """ Register the job given for execution at the intended start - date/time. + date/time and return the job. """ def getJobsToExecute(startTime=None): @@ -95,8 +95,12 @@ class ICrawlingJob(IScheduledJob): 'for all resources found.') def collect(**criteria): - """ Return a collection of resource/metadata pairs that should be transferred - to the server using the selection criteria given. + """ Return a deferred that upon callback will provide a + collection of resource/metadata pairs that should be transferred + to the server. + + Use the selection criteria given to filter the resources that + should be collected. """ @@ -140,6 +144,13 @@ class ITransporter(Interface): """ +class ITransportJob(IScheduledJob): + """ A job managing the the transfer of a resource to the server. + """ + + transporter = Attribute('The transporter object to user for transer.') + + class IConfigurator(Interface): """ Manages (stores and receives) configuration information. """ diff --git a/agent/schedule.py b/agent/schedule.py index 1ff53c7..cfd9393 100644 --- a/agent/schedule.py +++ b/agent/schedule.py @@ -72,7 +72,8 @@ class Job(object): def finishRun(self, result): for job in self.successors: - job.run(job, **job.params) + job.params['result'] = result + job.run(**job.params) # TODO: remove from queue # TODO: logging # TODO: reschedule if told by configuration diff --git a/agent/testing/__init__.py b/agent/testing/__init__.py new file mode 100644 index 0000000..4bc90fb --- /dev/null +++ b/agent/testing/__init__.py @@ -0,0 +1,4 @@ +""" +$Id$ +""" + diff --git a/agent/testing/crawl.py b/agent/testing/crawl.py new file mode 100644 index 0000000..a71bcc3 --- /dev/null +++ b/agent/testing/crawl.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +A dummy crawler for testing purposes. + +$Id$ +""" + +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from zope.interface import implements + +from loops.agent.interfaces import ICrawlingJob, IMetadataSet +from loops.agent.schedule import Job + + +class CrawlingJob(Job): + + implements(ICrawlingJob) + + def __init__(self): + self.predefinedMetadata = {} + super(CrawlingJob, self).__init__() + + def execute(self, **kw): + return self.collect(**kw) + + def collect(self, **criteria): + deferred = self.deferred = Deferred() + reactor.callLater(0, self.dataAvailable) + return deferred + + def dataAvailable(self): + self.deferred.callback([(DummyResource(), Metadata())]) + + +class Metadata(object): + + implements(IMetadataSet) + + +class DummyResource(object): + + data = 'Dummy resource data for testing purposes.' diff --git a/agent/testing/transport.py b/agent/testing/transport.py new file mode 100644 index 0000000..e72c07c --- /dev/null +++ b/agent/testing/transport.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +A dummy transport for testing purposes. + +$Id$ +""" + +from twisted.internet import reactor +from twisted.internet.defer import Deferred +from zope.interface import implements + +from loops.agent.interfaces import ITransportJob, ITransporter +from loops.agent.schedule import Job + + +class Transporter(object): + + implements(ITransporter) + + serverURL = None + method = None + machineName = None + userName = None + password = None + + def transfer(self, resource, metadata=None, resourceType=file): + if resourceType is file: + data = resource.read() + resource.close() + elif resourceType is str: + data = resource + print 'Transferring:', data + return Deferred() + + +class TransportJob(Job): + + implements(ITransportJob) + + def __init__(self, transporter): + super(TransportJob, self).__init__() + self.transporter = transporter + + def execute(self, **kw): + result = kw.get('result') + if result is None: + print 'No data available.' + else: + for r in result: + d = self.transporter.transfer(r[0].data, r[1], str) + return Deferred() diff --git a/agent/tests.py b/agent/tests.py index fa43e7b..01f8ff7 100755 --- a/agent/tests.py +++ b/agent/tests.py @@ -1,3 +1,8 @@ +# +# Run with ``trial2.4 tests.py`` to execute the twisted unit tests. +# Run with ``python tests.py`` to execute the doctests. +# + # $Id$ import unittest as standard_unittest @@ -11,6 +16,16 @@ from loops.agent.core import Agent from loops.agent.schedule import Job +class Tester(object): + + def iterate(self, n=10, delays={}): + for i in range(n): + delay = delays.get(i, 0) + reactor.iterate(delay) + +tester = Tester() + + class TestJob(Job): def execute(self, deferred, **kw): @@ -40,9 +55,9 @@ class Test(unittest.TestCase): def test_suite(): flags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS return standard_unittest.TestSuite(( - standard_unittest.makeSuite(Test), + #standard_unittest.makeSuite(Test), doctest.DocFileSuite('README.txt', optionflags=flags), )) if __name__ == '__main__': - unittest.main(defaultTest='test_suite') + standard_unittest.main(defaultTest='test_suite')