work in progress: crawling: let the doctest use the full procedure via master and controller

git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2489 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/__init__.py

@@ -6,3 +6,4 @@ $Id$
 
 from cybertools.agent.base import agent, control, job, log, schedule
 from cybertools.agent.core import agent, control, schedule
+from cybertools.agent.crawl import base

agent/base/agent.py

@@ -75,6 +75,9 @@ class Master(Agent):
     def setupAgents(self, agentSpecs):
         for spec in agentSpecs:
             agent = agents(self, spec.type)
+            if agent is None:
+                print spec.type
+                return
             agent.name = spec.name
             self.children[spec.name] = agent
 

agent/base/control.py

@@ -56,6 +56,10 @@ class SampleController(Controller):
     def _getAgents(self):
         return [AgentSpecification(name, type) for name, type in self.agents]
 
+    def createAgent(self, agentType, name):
+        spec = AgentSpecification(name, agentType)
+        self.agent.setupAgents([spec])
+
     def enterJob(self, jobType, agent):
         self.jobNumber += 1
         spec = JobSpecification(jobType, '%05i' % self.jobNumber, agent=agent)

agent/core/README.txt

@@ -1,31 +0,0 @@
-================================================
-Agents for Job Execution and Communication Tasks
-================================================
-
-Agents collect informations and transfer them e.g. to a loops server.
-
-  ($Id$)
-
-This package does not depend on zope or the other loops packages
-but represents a standalone application.
-
-But we need a reactor for working with Twisted; in order not to block
-testing when running the reactor we use reactor.iterate() calls
-wrapped in a ``tester`` object.
-
-  >>> from cybertools.agent.tests import tester
-
-
-Master Agent
-============
-
-The agent uses Twisted's cooperative multitasking model.
-
-This means that all calls to services (like crawler, transporter, ...)
-return a deferred that must be supplied with a callback method (and in
-most cases also an errback method).
-
-  >>> #from cybertools.agent.core.agent import Master
-  >>> #master = Master()
-
-

agent/crawl/README.txt

@@ -2,35 +2,45 @@
 Agents for Job Execution and Communication Tasks
 ================================================
 
-Agents collect informations and transfer them e.g. to a loops server.
+  ($Id$)
 
-  ($Id: README.txt 2413 2008-02-23 14:07:15Z helmutm $)
+  >>> from cybertools.agent.base.agent import Master
 
-This package does not depend on zope or the other loops packages
+  >>> config = '''
-but represents a standalone application.
+  ... controller(name='core.sample')
-
+  ... scheduler(name='core')
-But we need a reactor for working with Twisted; in order not to block
+  ... logger(name='default', standard=30)
-testing when running the reactor we use reactor.iterate() calls
+  ... '''
-wrapped in a ``tester`` object.
+  >>> master = Master(config)
-
+  >>> master.setup()
-  >>> from cybertools.agent.tests import tester
 
 
 Crawler
-============
+=======
 
 The agent uses Twisted's cooperative multitasking model.
 
-Crawler is the base class for all derived Crawlers like the filesystem crawler
+Crawler is the base class for all derived crawlers like the filesystem crawler
 and the mailcrawler. The SampleCrawler returns a deferred that already had a
 callback being called, so it will return at once.
+
 Returns a deferred that must be supplied with a callback method (and in
 most cases also an errback method).
 
-  >>> from cybertools.agent.crawl.base import SampleCrawler
+We create the sample crawler via the master's controller. The sample
-  >>> from twisted.internet import defer
+controller provides a simple method for this purpose.
-  >>> crawler = SampleCrawler()
+
-  >>> deferred = crawler.collect()
+  >>> controller = master.controllers[0]
+  >>> controller.createAgent('crawl.sample', 'crawler01')
+
+In the next step we request the start of a job, again via the controller.
+
+  >>> controller.enterJob('sample', 'crawler01')
+
+The job is not executed immediately - we have to hand over control to
+the twisted reactor first.
+
+  >>> from cybertools.agent.tests import tester
+  >>> tester.iterate()
   SampleCrawler is collecting.
 
-

agent/crawl/base.py

@@ -24,18 +24,19 @@ $Id: base.py
 
 from zope.interface import implements
 
-from cybertools.agent.base.agent import Agent
+from cybertools.agent.base.agent import Master
+from cybertools.agent.core.agent import QueueableAgent
 from cybertools.agent.interfaces import ICrawler
 from cybertools.agent.components import agents
 from twisted.internet.defer import succeed
 
 
-class Crawler(object):
+class Crawler(QueueableAgent):
 
     implements(ICrawler)
 
-    def __init__(self):
+    def process(self):
-        pass
+        return self.collect()
 
     def collect(self, filter=None):
         d = defer.succeed([])
@@ -49,6 +50,5 @@ class SampleCrawler(Crawler):
         d = succeed([])
         return d
 
-
+agents.register(SampleCrawler, Master, name='crawl.sample')
-agents.register(SampleCrawler, Agent, name='crawl.sample')
 

agent/crawl/mail.py

@@ -25,8 +25,7 @@ $Id: base.py
 from zope.interface import implements
 
 from cybertools.agent.agent import Agent
-from cybertools.agent.interfaces import ICrawler
+from cybertools.agent.crawl.base import Crawler
-from cybertools.agent.crawl.base import Crawler as BaseCrawler
 from cybertools.agent.components import agents
 from twisted.internet.defer import succeed
 

agent/tests.py

@@ -42,9 +42,7 @@ def test_suite():
     testSuite = unittest.TestSuite((
             unittest.makeSuite(Test),
             DocFileSuite('README.txt', optionflags=flags),
-            DocFileSuite('core/README.txt', optionflags=flags),
             DocFileSuite('crawl/README.txt', optionflags=flags),
-            #DocFileSuite('transport/httpput.txt', optionflags=flags),
     ))
     return testSuite