diff --git a/agent/README.txt b/agent/README.txt index b10943b..ac1f4bd 100644 --- a/agent/README.txt +++ b/agent/README.txt @@ -16,28 +16,31 @@ Sub-Packages ============ Top-level - Generic interfaces, ``tests`` module, this ``README.txt`` file. + Generic interfaces, ``commponents``: adapter registry, + ``tests`` module, this ``README.txt`` file. base - Base and sample classes + Base and sample classes. core Agent and scheduling implementations. control - Communication with an external agent control and job database application + Communication with an external agent control and job database application. crawl Scanning/crawling some system, e.g. the database of an email application, - the local file system, or an external document source + the local file system, or an external document source. transport Transfer of information objects to agents on another machine or - to an information management application (e.g. loops) + to an information management application (e.g. loops). util Various utility modules, e.g. a backport of the - ``twisted.internet.task.coiterate()`` function from Twisted 2.5. + ``twisted.internet.task.coiterate()`` function from Twisted 2.5 so that + we can use the Twisted version coming with Zope 3.3.1 for + cybertools.agent. All sub-packages except ``base`` depend on Twisted. @@ -77,7 +80,7 @@ framework Twisted there is some basic stuff (mainly interfaces and base classes with basic, sample, or dummy implementations) that is independent of Twisted. -The code resides in in the ``base`` sub-package. +The code for this resides in in the ``base`` sub-package. Master Agent and Configuration ------------------------------ @@ -103,24 +106,53 @@ the path to the configuration file. >>> master.config controller.name = 'sample' - logger.name = 'sample' + logger.name = 'default' + logger.standard = 20 scheduler.name = 'sample' -Controller ----------- +Controllers +----------- -Creation of agents and scheduling of jobs is controlled by the controller -object. This is typically associated with a sort of control storage that +Creation of agents and scheduling of jobs is controlled by controller +objects. These are typically associated with a sort of control storage that provides agent and job specifications and receives the results of job execution. -We open the controller and read in the specifications via the master agent's -``setup`` method. + >>> master.controllers + [] + +We make the contollers provide the specifications via the master agent's +``setup()`` method. >>> master.setup() Other Agents ------------ +The above ``setup()`` call has triggered the creation of one child agent - +that is all the sample controller provides. + + >>> master.children + {'sample01': } + +Let's check a few attributes of the newly created agent. + + >>> agent01 = master.children['sample01'] + >>> agent01.master is master + True + >>> agent01.config is master.config + True + Job Scheduling and Execution ---------------------------- + + >>> master.scheduler + + +Logging +------- + + >>> master.logger + + >>> agent01.logger is master.logger + True diff --git a/agent/__init__.py b/agent/__init__.py index 4bc90fb..c102675 100644 --- a/agent/__init__.py +++ b/agent/__init__.py @@ -2,3 +2,6 @@ $Id$ """ +# register default adapters + +from cybertools.agent.base import agent, control, job, log, schedule diff --git a/agent/base/agent.py b/agent/base/agent.py index 880b770..43ef6d3 100644 --- a/agent/base/agent.py +++ b/agent/base/agent.py @@ -25,6 +25,8 @@ $Id$ from zope.interface import implements from cybertools.agent.interfaces import IAgent +from cybertools.agent.components import agents +from cybertools.agent.components import controllers, loggers, schedulers from cybertools.util.config import Configurator @@ -33,24 +35,43 @@ class Agent(object): implements(IAgent) master = None + config = None logger = None + def __init__(self, master): + self.master = master + self.config = master.config + self.logger = master.logger + def execute(self, job, params=None): pass class Master(Agent): - config = None - controller = None scheduler = None def __init__(self, configuration=None): - self.config = Configurator() + config = self.config = Configurator() + self.master = self + self.controllers = [] + self.children = {} if configuration is not None: - self.config.load(configuration) + config.load(configuration) + self.logger = loggers(self, name=config.logger.name) + self.controllers.append(controllers(self, name=config.controller.name)) + self.scheduler = schedulers(self, name=config.scheduler.name) def setup(self): + for cont in self.controllers: + cont.setupAgent() + + def setupAgents(self, agentSpecs): + for spec in agentSpecs: + agent = agents(self, spec.type) + self.children[spec.name] = agent + + def setupJobs(self, jobSpecs): pass @@ -58,3 +79,5 @@ class SampleAgent(Agent): pass +agents.register(SampleAgent, Master, name='sample') + diff --git a/agent/base/control.py b/agent/base/control.py index 582d5ce..8b6c73a 100644 --- a/agent/base/control.py +++ b/agent/base/control.py @@ -24,6 +24,8 @@ $Id$ from zope.interface import implements +from cybertools.agent.base.agent import Master +from cybertools.agent.components import controllers from cybertools.agent.interfaces import IController @@ -31,3 +33,42 @@ class Controller(object): implements(IController) + def __init__(self, agent): + self.agent = agent + + def setupAgent(self): + self.agent.setupAgents(self._getAgents()) + self.agent.setupJobs(self._getCurrentJobs()) + + def _getAgents(self): + return [] + + def _getCurrentJobs(self): + return [] + + +class SampleController(Controller): + + def _getAgents(self): + return [AgentSpecification('sample01', 'sample')] + +controllers.register(SampleController, Master, name='sample') + + +class AgentSpecification(object): + + def __init__(self, name, type, **kw): + self.name = name + self.type = type + for k, v in kw.items(): + setattr(self, k, v) + + +class JobSpecification(object): + + def __init__(self, name, type, **kw): + self.name = name + self.type = type + for k, v in kw.items(): + setattr(self, k, v) + diff --git a/agent/base/job.py b/agent/base/job.py index 0fbe7de..e8e02d7 100644 --- a/agent/base/job.py +++ b/agent/base/job.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/agent/base/log.py b/agent/base/log.py index 6732edf..26a7197 100644 --- a/agent/base/log.py +++ b/agent/base/log.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2007 Helmut Merz helmutm@cy55.de +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,9 @@ import sys import time from zope.interface import implements -from loops.agent.interfaces import ILogger, ILogRecord +from cybertools.agent.base.agent import Agent +from cybertools.agent.components import loggers +from cybertools.agent.interfaces import ILogger, ILogRecord class LogRecord(object): @@ -48,20 +50,20 @@ class LogRecord(object): return ' '.join(msg) -class Logger(list): +class Logger(object): implements(ILogger) recordFactory = LogRecord - def __init__(self, agent): self.agent = agent + self.records = [] + self.externalLoggers = [] self.setup() def setup(self): - self.externalLoggers = [] - conf = self.agent.config.logging + conf = self.agent.config.logger if conf.standard: logger = logging.getLogger() logger.level = conf.standard @@ -70,7 +72,9 @@ class Logger(list): def log(self, data): record = self.recordFactory(self, data) - self.append(record) + self.records.append(record) for logger in self.externalLoggers: logger.info(str(record)) + +loggers.register(Logger, Agent, name='default') diff --git a/agent/base/sample.cfg b/agent/base/sample.cfg index aca8734..8338d97 100644 --- a/agent/base/sample.cfg +++ b/agent/base/sample.cfg @@ -6,4 +6,4 @@ controller(name='sample') scheduler(name='sample') -logger(name='sample') +logger(name='default', standard=20) diff --git a/agent/base/schedule.py b/agent/base/schedule.py index 6a181eb..65e4ad0 100644 --- a/agent/base/schedule.py +++ b/agent/base/schedule.py @@ -22,13 +22,20 @@ Basic (sample) job scheduler. $Id$ """ -from time import time + from zope.interface import implements -from loops.agent.interfaces import IScheduler +from cybertools.agent.base.agent import Master +from cybertools.agent.components import schedulers +from cybertools.agent.interfaces import IScheduler class Scheduler(object): implements(IScheduler) + def __init__(self, agent): + self.agent = agent + + +schedulers.register(Scheduler, Master, name='sample') diff --git a/agent/components.py b/agent/components.py new file mode 100644 index 0000000..c33a7c0 --- /dev/null +++ b/agent/components.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Component registries. + +$Id$ +""" + +from cybertools.util.adapter import AdapterFactory + + +agents = AdapterFactory() +controllers = AdapterFactory() +jobs = AdapterFactory() +loggers = AdapterFactory() +schedulers = AdapterFactory() diff --git a/agent/interfaces.py b/agent/interfaces.py index 9081920..6567d91 100644 --- a/agent/interfaces.py +++ b/agent/interfaces.py @@ -32,8 +32,11 @@ class IAgent(Interface): """ master = Attribute('IMaster instance.') + config = Attribute('Configuration settings.') logger = Attribute('Logger instance to be used for recording ' 'job execution and execution results.') + children = Attribute('A collection of agents that are managed by this ' + 'master.') def execute(job, params=None): """ Execute a job. @@ -44,15 +47,30 @@ class IMaster(IAgent): """ The top-level controller agent. """ - config = Attribute('Central configuration.') - controller = Attribute('IController instance.') + config = Attribute('Central configuration settings.') + controllers = Attribute('Collection of IController instances.') scheduler = Attribute('IScheduler instance.') def setup(): - """ Load agent specifications from the controller and set up - the corresponding agents. Then load the specifications of - active jobs from the controller and schedule the corresponding - jobs. + """ Set up the master agent by triggering all assigned controllers. + Each controller will then call the master agent's callback + methods ``setupAgents()`` and ``setupJobs()``. + """ + + def setupAgents(agentSpecs): + """ Callback for loading agent specifications from the controller + and setting up the corresponding agents. + + Will be called upon agent setup and later when the controller + wants to provide new agent information. + """ + + def setupJobs(jobSpecs): + """ Callback for loading the specifications of active jobs from + the controller and scheduling the corresponding jobs. + + Will be called upon agent setup and later when the controller + wants to provide new job information. """ @@ -98,6 +116,11 @@ class IController(Interface): information. """ + def setupAgent(): + """ Set up the controllers's agent by calling the agent's + callback methods. + """ + class IScheduler(Interface): """ Manages jobs and cares that they are started at the appropriate diff --git a/agent/transport/client.py b/agent/transport/client.py new file mode 100644 index 0000000..7e80728 --- /dev/null +++ b/agent/transport/client.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Transferring information to or requesting information from a remote +cybertools.agent instance with a corresponding server agent. + +$Id$ +""" + +from zope.interface import implements + diff --git a/agent/transport/file/__init__.py b/agent/transport/file/__init__.py new file mode 100644 index 0000000..4bc90fb --- /dev/null +++ b/agent/transport/file/__init__.py @@ -0,0 +1,4 @@ +""" +$Id$ +""" + diff --git a/agent/transport/file/dav.py b/agent/transport/file/dav.py new file mode 100644 index 0000000..2dad9b2 --- /dev/null +++ b/agent/transport/file/dav.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Transferring files to a remote site via WebDAV. + +$Id$ +""" + +from zope.interface import implements + + diff --git a/agent/transport/local.py b/agent/transport/local.py new file mode 100644 index 0000000..9807cb2 --- /dev/null +++ b/agent/transport/local.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Transferring information to an application on the same machine, typically +a loops site on a local Zope instance. + +$Id$ +""" + +from zope.interface import implements + + diff --git a/agent/transport/server.py b/agent/transport/server.py new file mode 100644 index 0000000..f1543ba --- /dev/null +++ b/agent/transport/server.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2008 Helmut Merz helmutm@cy55.de +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +""" +Providing access for remote cybertools.agent instances by listening +for requests from client agents. + +$Id$ +""" + +from zope.interface import implements + + diff --git a/util/adapter.py b/util/adapter.py index 61c533f..70c196d 100644 --- a/util/adapter.py +++ b/util/adapter.py @@ -53,4 +53,6 @@ class AdapterFactory(object): check also for its base classes. """ adapter = self.queryAdapter(obj, name) - return adapter is not None and adapter(obj) or None + if adapter is None: + return None + return adapter(obj)