work in progress: cybertools.agent - more on base and sample implementations

git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2414 fd906abe-77d9-0310-91a1-e0d9ade77398

agent/README.txt

@@ -16,28 +16,31 @@ Sub-Packages
 ============
 
 Top-level
-    Generic interfaces, ``tests`` module, this ``README.txt`` file.
+    Generic interfaces, ``commponents``: adapter registry,
+    ``tests`` module, this ``README.txt`` file.
 
 base
-    Base and sample classes
+    Base and sample classes.
 
 core
     Agent and scheduling implementations.
 
 control
-    Communication with an external agent control and job database application
+    Communication with an external agent control and job database application.
 
 crawl
     Scanning/crawling some system, e.g. the database of an email application,
-    the local file system, or an external document source
+    the local file system, or an external document source.
 
 transport
     Transfer of information objects to agents on another machine or
-    to an information management application (e.g. loops)
+    to an information management application (e.g. loops).
 
 util
     Various utility modules, e.g. a backport of the
-    ``twisted.internet.task.coiterate()`` function from Twisted 2.5.
+    ``twisted.internet.task.coiterate()`` function from Twisted 2.5 so that
+    we can use the Twisted version coming with Zope 3.3.1 for
+    cybertools.agent.
 
 All sub-packages except ``base`` depend on Twisted.
 
@@ -77,7 +80,7 @@ framework Twisted there is some basic stuff (mainly interfaces and
 base classes with basic, sample, or dummy implementations) that is
 independent of Twisted.
 
-The code resides in in the ``base`` sub-package.
+The code for this resides in in the ``base`` sub-package.
 
 Master Agent and Configuration
 ------------------------------
@@ -103,24 +106,53 @@ the path to the configuration file.
 
   >>> master.config
   controller.name = 'sample'
-  logger.name = 'sample'
+  logger.name = 'default'
+  logger.standard = 20
   scheduler.name = 'sample'
 
-Controller
-----------
+Controllers
+-----------
 
-Creation of agents and scheduling of jobs is controlled by the controller
-object. This is typically associated with a sort of control storage that
+Creation of agents and scheduling of jobs is controlled by controller
+objects. These are typically associated with a sort of control storage that
 provides agent and job specifications and receives the results of job
 execution.
 
-We open the controller and read in the specifications via the master agent's
-``setup`` method.
+  >>> master.controllers
+  [<cybertools.agent.base.control.SampleController object ...>]
+
+We make the contollers provide the specifications via the master agent's
+``setup()`` method.
 
   >>> master.setup()
 
 Other Agents
 ------------
 
+The above ``setup()`` call has triggered the creation of one child agent -
+that is all the sample controller provides.
+
+  >>> master.children
+  {'sample01': <cybertools.agent.base.agent.SampleAgent object ...>}
+
+Let's check a few attributes of the newly created agent.
+
+  >>> agent01 = master.children['sample01']
+  >>> agent01.master is master
+  True
+  >>> agent01.config is master.config
+  True
+
 Job Scheduling and Execution
 ----------------------------
+
+  >>> master.scheduler
+  <cybertools.agent.base.schedule.Scheduler object ...>
+
+Logging
+-------
+
+  >>> master.logger
+  <cybertools.agent.base.log.Logger object ...>
+  >>> agent01.logger is master.logger
+  True

agent/__init__.py

@@ -2,3 +2,6 @@
 $Id$
 """
 
+# register default adapters
+
+from cybertools.agent.base import agent, control, job, log, schedule

agent/base/agent.py

@@ -25,6 +25,8 @@ $Id$
 from zope.interface import implements
 
 from cybertools.agent.interfaces import IAgent
+from cybertools.agent.components import agents
+from cybertools.agent.components import controllers, loggers, schedulers
 from cybertools.util.config import Configurator
 
 
@@ -33,24 +35,43 @@ class Agent(object):
     implements(IAgent)
 
     master = None
+    config = None
     logger = None
 
+    def __init__(self, master):
+        self.master = master
+        self.config = master.config
+        self.logger = master.logger
+
     def execute(self, job, params=None):
         pass
 
 
 class Master(Agent):
 
-    config = None
-    controller = None
     scheduler = None
 
     def __init__(self, configuration=None):
-        self.config = Configurator()
+        config = self.config = Configurator()
+        self.master = self
+        self.controllers = []
+        self.children = {}
         if configuration is not None:
-            self.config.load(configuration)
+            config.load(configuration)
+            self.logger = loggers(self, name=config.logger.name)
+            self.controllers.append(controllers(self, name=config.controller.name))
+            self.scheduler = schedulers(self, name=config.scheduler.name)
 
     def setup(self):
+        for cont in self.controllers:
+            cont.setupAgent()
+
+    def setupAgents(self, agentSpecs):
+        for spec in agentSpecs:
+            agent = agents(self, spec.type)
+            self.children[spec.name] = agent
+
+    def setupJobs(self, jobSpecs):
         pass
 
 
@@ -58,3 +79,5 @@ class SampleAgent(Agent):
 
     pass
 
+agents.register(SampleAgent, Master, name='sample')
+

agent/base/control.py

@@ -24,6 +24,8 @@ $Id$
 
 from zope.interface import implements
 
+from cybertools.agent.base.agent import Master
+from cybertools.agent.components import controllers
 from cybertools.agent.interfaces import IController
 
 
@@ -31,3 +33,42 @@ class Controller(object):
 
     implements(IController)
 
+    def __init__(self, agent):
+        self.agent = agent
+
+    def setupAgent(self):
+        self.agent.setupAgents(self._getAgents())
+        self.agent.setupJobs(self._getCurrentJobs())
+
+    def _getAgents(self):
+        return []
+
+    def _getCurrentJobs(self):
+        return []
+
+
+class SampleController(Controller):
+
+    def _getAgents(self):
+        return [AgentSpecification('sample01', 'sample')]
+
+controllers.register(SampleController, Master, name='sample')
+
+
+class AgentSpecification(object):
+
+    def __init__(self, name, type, **kw):
+        self.name = name
+        self.type = type
+        for k, v in kw.items():
+            setattr(self, k, v)
+
+
+class JobSpecification(object):
+
+    def __init__(self, name, type, **kw):
+        self.name = name
+        self.type = type
+        for k, v in kw.items():
+            setattr(self, k, v)
+

agent/base/job.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by

agent/base/log.py

@@ -1,5 +1,5 @@
 #
-#  Copyright (c) 2007 Helmut Merz helmutm@cy55.de
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
 #
 #  This program is free software; you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
@@ -27,7 +27,9 @@ import sys
 import time
 from zope.interface import implements
 
-from loops.agent.interfaces import ILogger, ILogRecord
+from cybertools.agent.base.agent import Agent
+from cybertools.agent.components import loggers
+from cybertools.agent.interfaces import ILogger, ILogRecord
 
 
 class LogRecord(object):
@@ -48,20 +50,20 @@ class LogRecord(object):
         return ' '.join(msg)
 
 
-class Logger(list):
+class Logger(object):
 
     implements(ILogger)
 
     recordFactory = LogRecord
 
-
     def __init__(self, agent):
         self.agent = agent
+        self.records = []
+        self.externalLoggers = []
         self.setup()
 
     def setup(self):
-        self.externalLoggers = []
-        conf = self.agent.config.logging
+        conf = self.agent.config.logger
         if conf.standard:
             logger = logging.getLogger()
             logger.level = conf.standard
@@ -70,7 +72,9 @@ class Logger(list):
 
     def log(self, data):
         record = self.recordFactory(self, data)
-        self.append(record)
+        self.records.append(record)
         for logger in self.externalLoggers:
             logger.info(str(record))
 
+
+loggers.register(Logger, Agent, name='default')

agent/base/sample.cfg

@@ -6,4 +6,4 @@
 
 controller(name='sample')
 scheduler(name='sample')
-logger(name='sample')
+logger(name='default', standard=20)

agent/base/schedule.py

@@ -22,13 +22,20 @@ Basic (sample) job scheduler.
 $Id$
 """
 
-from time import time
+
 from zope.interface import implements
 
-from loops.agent.interfaces import IScheduler
+from cybertools.agent.base.agent import Master
+from cybertools.agent.components import schedulers
+from cybertools.agent.interfaces import IScheduler
 
 
 class Scheduler(object):
 
     implements(IScheduler)
 
+    def __init__(self, agent):
+        self.agent = agent
+
+
+schedulers.register(Scheduler, Master, name='sample')

agent/components.py

@@ -0,0 +1,32 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Component registries.
+
+$Id$
+"""
+
+from cybertools.util.adapter import AdapterFactory
+
+
+agents = AdapterFactory()
+controllers = AdapterFactory()
+jobs = AdapterFactory()
+loggers = AdapterFactory()
+schedulers = AdapterFactory()

agent/interfaces.py

@@ -32,8 +32,11 @@ class IAgent(Interface):
     """
 
     master = Attribute('IMaster instance.')
+    config = Attribute('Configuration settings.')
     logger = Attribute('Logger instance to be used for recording '
                     'job execution and execution results.')
+    children = Attribute('A collection of agents that are managed by this '
+                    'master.')
 
     def execute(job, params=None):
         """ Execute a job.
@@ -44,15 +47,30 @@ class IMaster(IAgent):
     """ The top-level controller agent.
     """
 
-    config = Attribute('Central configuration.')
-    controller = Attribute('IController instance.')
+    config = Attribute('Central configuration settings.')
+    controllers = Attribute('Collection of IController instances.')
     scheduler = Attribute('IScheduler instance.')
 
     def setup():
-        """ Load agent specifications from the controller and set up
-            the corresponding agents. Then load the specifications of
-            active jobs from the controller and schedule the corresponding
-            jobs.
+        """ Set up the master agent by triggering all assigned controllers.
+            Each controller will then call the master agent's callback
+            methods ``setupAgents()`` and ``setupJobs()``.
+        """
+
+    def setupAgents(agentSpecs):
+        """ Callback for loading agent specifications from the controller
+            and setting up the corresponding agents.
+
+            Will be called upon agent setup and later when the controller
+            wants to provide new agent information.
+        """
+
+    def setupJobs(jobSpecs):
+        """ Callback for loading the specifications of active jobs from
+            the controller and scheduling the corresponding jobs.
+
+            Will be called upon agent setup and later when the controller
+            wants to provide new job information.
         """
 
 
@@ -98,6 +116,11 @@ class IController(Interface):
         information.
     """
 
+    def setupAgent():
+        """ Set up the controllers's agent by calling the agent's
+            callback methods.
+        """
+
 
 class IScheduler(Interface):
     """ Manages jobs and cares that they are started at the appropriate

agent/transport/client.py

@@ -0,0 +1,27 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Transferring information to or requesting information from a remote
+cybertools.agent instance with a corresponding server agent.
+
+$Id$
+"""
+
+from zope.interface import implements
+

agent/transport/file/__init__.py

@@ -0,0 +1,4 @@
+"""
+$Id$
+"""
+

agent/transport/file/dav.py

@@ -0,0 +1,27 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Transferring files to a remote site via WebDAV.
+
+$Id$
+"""
+
+from zope.interface import implements
+
+

agent/transport/local.py

@@ -0,0 +1,28 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Transferring information to an application on the same machine, typically
+a loops site on a local Zope instance.
+
+$Id$
+"""
+
+from zope.interface import implements
+
+

agent/transport/server.py

@@ -0,0 +1,28 @@
+#
+#  Copyright (c) 2008 Helmut Merz helmutm@cy55.de
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with this program; if not, write to the Free Software
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+
+"""
+Providing access for remote cybertools.agent instances by listening
+for requests from client agents.
+
+$Id$
+"""
+
+from zope.interface import implements
+
+

util/adapter.py

@@ -53,4 +53,6 @@ class AdapterFactory(object):
             check also for its base classes.
         """
         adapter = self.queryAdapter(obj, name)
-        return adapter is not None and adapter(obj) or None
+        if adapter is None:
+            return None
+        return adapter(obj)