cco/cybertools
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rearrange system startup so that components are not registered during initial import but via a controlled setup

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2500 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2008-04-07 06:36:48 +00:00
parent 5aee688a12
commit e4108ed7d1
14 changed files with 313 additions and 268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
agent/README.txt
View file

@ -1,4 +1,4 @@
================================================ ================================================
Agents for Job Execution and Communication Tasks Agents for Job Execution and Communication Tasks
================================================ ================================================
@ -101,8 +101,10 @@ configuration are provided by the controller.
So we are now ready to create a master agent and configure it by supplying So we are now ready to create a master agent and configure it by supplying
the path to the configuration file. the path to the configuration file.
>>> from cybertools.agent.base.agent import Master >>> from cybertools.agent.main import setup
>>> master = Master(configFile) >>> master = setup(configFile)
Starting agent application...
Using controllers base.sample.
>>> configFile.close() >>> configFile.close()
>>> master.config >>> master.config
@ -201,8 +203,9 @@ the twisted reactor.
... scheduler(name='core') ... scheduler(name='core')
... logger(name='default', standard=30) ... logger(name='default', standard=30)
... ''' ... '''
>>> master = Master(config) >>> master = setup(config)
>>> master.setup() Starting agent application...
Using controllers core.sample.
>>> master.scheduler >>> master.scheduler
<cybertools.agent.core.schedule.Scheduler object ...> <cybertools.agent.core.schedule.Scheduler object ...>

7
agent/__init__.py
View file

@ -1,10 +1,3 @@
""" """
$Id$ $Id$
""" """
# register default adapters
from cybertools.agent.base import agent, control, job, log, schedule
from cybertools.agent.core import agent, control, schedule
from cybertools.agent.control import cmdline
from cybertools.agent.crawl import base

12
agent/base/agent.py
View file

@ -60,22 +60,24 @@ class Master(Agent):
name = 'master' name = 'master'
scheduler = None scheduler = None
logger = None
def __init__(self, configuration): def __init__(self, configuration):
if isinstance(configuration, Configurator): if isinstance(configuration, Configurator):
config = configuration self.config = configuration
else: # configuration is path to config file else: # configuration is path to config file
config = self.config = Configurator() self.config = Configurator()
config.load(configuration) self.config.load(configuration)
self.master = self self.master = self
self.controllers = [] self.controllers = []
self.children = {} self.children = {}
def setup(self):
config = self.config
self.logger = loggers(self, name=config.logger.name) self.logger = loggers(self, name=config.logger.name)
for n in config.controller.names: for n in config.controller.names:
self.controllers.append(controllers(self, n)) self.controllers.append(controllers(self, n))
self.scheduler = schedulers(self, name=config.scheduler.name) self.scheduler = schedulers(self, name=config.scheduler.name)
def setup(self):
for cont in self.controllers: for cont in self.controllers:
cont.setup() cont.setup()

2
agent/common.py
View file

@ -17,7 +17,7 @@
# #
""" """
Common stuff. Miscellaneous common stuff.
$Id$ $Id$
""" """

8
agent/crawl/README.txt
View file

@ -4,15 +4,15 @@ Agents for Job Execution and Communication Tasks
($Id$) ($Id$)
>>> from cybertools.agent.base.agent import Master
>>> config = ''' >>> config = '''
... controller(names=['core.sample']) ... controller(names=['core.sample'])
... scheduler(name='core') ... scheduler(name='core')
... logger(name='default', standard=30) ... logger(name='default', standard=30)
... ''' ... '''
>>> master = Master(config) >>> from cybertools.agent.main import setup
>>> master.setup() >>> master = setup(config)
Starting agent application...
Using controllers core.sample.
Crawler Crawler

344
agent/crawl/outlook.py
View file

@ -1,172 +1,172 @@
# #
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de # Copyright (c) 2008 Helmut Merz helmutm@cy55.de
# #
# This program is free software; you can redistribute it and/or modify # This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or # the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# This program is distributed in the hope that it will be useful, # This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
# #
""" """
Outlook Crawler Class. Outlook Crawler Class.
$Id: mail.py 2493 2008-04-03 11:17:37Z helmutm $ $Id$
""" """
import re import re
from zope.interface import implements from zope.interface import implements
from twisted.internet import defer from twisted.internet import defer
import win32com.client import win32com.client
import ctypes import ctypes
import win32api, win32process, win32con import win32api, win32process, win32con
from cybertools.agent.base.agent import Agent, Master from cybertools.agent.base.agent import Agent, Master
from cybertools.agent.crawl.mail import MailCrawler from cybertools.agent.crawl.mail import MailCrawler
from cybertools.agent.crawl.mail import MailResource from cybertools.agent.crawl.mail import MailResource
from cybertools.agent.components import agents from cybertools.agent.components import agents
# some constants # some constants
COMMASPACE = ', ' COMMASPACE = ', '
class OutlookCrawler(MailCrawler): class OutlookCrawler(MailCrawler):
keys = "" keys = ""
inbox = "" inbox = ""
subfolders = "" subfolders = ""
pattern = "" pattern = ""
def collect(self, filter=None): def collect(self, filter=None):
self.d = defer.Deferred() self.d = defer.Deferred()
self.oOutlookApp = None self.oOutlookApp = None
if self.findOutlook(): if self.findOutlook():
self.fetchCriteria() self.fetchCriteria()
self.d.addCallback(self.crawlFolders) self.d.addCallback(self.crawlFolders)
else: else:
pass pass
#self.d.addErrback([]) #self.d.addErrback([])
return d return d
def fetchCriteria(self): def fetchCriteria(self):
criteria = self.params criteria = self.params
self.keys = criteria.get('keys') self.keys = criteria.get('keys')
self.inbox = criteria.get('inbox') #boolean self.inbox = criteria.get('inbox') #boolean
self.subfolders = criteria.get('subfolders') #boolean self.subfolders = criteria.get('subfolders') #boolean
self.pattern = criteria.get('pattern') self.pattern = criteria.get('pattern')
if self.pattern != '': if self.pattern != '':
self.pattern = re.compile(criteria.get('pattern') or '.*') self.pattern = re.compile(criteria.get('pattern') or '.*')
def crawlFolders(self): def crawlFolders(self):
onMAPI = self.oOutlookApp.GetNamespace("MAPI") onMAPI = self.oOutlookApp.GetNamespace("MAPI")
ofInbox = \ ofInbox = \
onMAPI.GetDefaultFolder(win32com.client.constants.olFolderInbox) onMAPI.GetDefaultFolder(win32com.client.constants.olFolderInbox)
# fetch mails from inbox # fetch mails from inbox
if self.inbox: if self.inbox:
self.loadMailsFromFolder(ofInbox) self.loadMailsFromFolder(ofInbox)
# fetch mails of inbox subfolders # fetch mails of inbox subfolders
if self.subfolders and self.pattern is None: if self.subfolders and self.pattern is None:
lInboxSubfolders = getattr(ofInbox, 'Folders') lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()): for of in range(lInboxSubfolders.__len__()):
# get a MAPI-subfolder object and load its emails # get a MAPI-subfolder object and load its emails
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1)) self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
elif self.subfolders and self.pattern: elif self.subfolders and self.pattern:
lInboxSubfolders = getattr(ofInbox, 'Folders') lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()): for of in range(lInboxSubfolders.__len__()):
# get specified MAPI-subfolder object and load its emails # get specified MAPI-subfolder object and load its emails
if self.pattern.match(getattr(lInboxSubfolders.Item(of + 1), 'Name')): if self.pattern.match(getattr(lInboxSubfolders.Item(of + 1), 'Name')):
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1)) self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
return self.result return self.result
def loadMailsFromFolder(self, folder): def loadMailsFromFolder(self, folder):
# get items of the folder # get items of the folder
folderItems = getattr(folder, 'Items') folderItems = getattr(folder, 'Items')
for item in range(len(folderItems)): for item in range(len(folderItems)):
mail = folderItems.Item(item+1) mail = folderItems.Item(item+1)
if mail.Class == win32com.client.constants.olMail: if mail.Class == win32com.client.constants.olMail:
if self.keys is None: if self.keys is None:
self.keys = [] self.keys = []
for key in mail._prop_map_get_: for key in mail._prop_map_get_:
if isinstance(getattr(mail, key), (int, str, unicode)): if isinstance(getattr(mail, key), (int, str, unicode)):
self.keys.append(key) self.keys.append(key)
record = {} record = {}
for key in self.keys: for key in self.keys:
record[key] = getattr(mail, key) record[key] = getattr(mail, key)
# Create the mime email object # Create the mime email object
msg = self.createEmailMime(record) msg = self.createEmailMime(record)
# Create a resource and append it to the result list # Create a resource and append it to the result list
self.createResource(msg, folder, "Microsoft Office Outlook") self.createResource(msg, folder, "Microsoft Office Outlook")
def login(self): def login(self):
pass pass
def handleOutlookDialog(self): def handleOutlookDialog(self):
""" """
This function handles the outlook dialog, which appears if someone This function handles the outlook dialog, which appears if someone
tries to access to MS Outlook. tries to access to MS Outlook.
""" """
hwnd = None hwnd = None
while True: while True:
hwnd = ctypes.windll.user32.FindWindowExA(None, hwnd, None, None) hwnd = ctypes.windll.user32.FindWindowExA(None, hwnd, None, None)
if hwnd == None: if hwnd == None:
break break
else: else:
val = u"\0" * 1024 val = u"\0" * 1024
ctypes.windll.user32.GetWindowTextW(hwnd, val, len(val)) ctypes.windll.user32.GetWindowTextW(hwnd, val, len(val))
val = val.replace(u"\000", u"") val = val.replace(u"\000", u"")
if val and repr(val) == "u'Microsoft Office Outlook'": if val and repr(val) == "u'Microsoft Office Outlook'":
print repr(val) print repr(val)
# get the Main Control # get the Main Control
form = findTopWindow(wantedText='Microsoft Office Outlook') form = findTopWindow(wantedText='Microsoft Office Outlook')
controls = findControls(form) controls = findControls(form)
# get the check box # get the check box
checkBox = findControl(form, wantedText='Zugriff') checkBox = findControl(form, wantedText='Zugriff')
setCheckBox(checkBox, 1) setCheckBox(checkBox, 1)
# get the combo box # get the combo box
comboBox = findControl(form, wantedClass='ComboBox') comboBox = findControl(form, wantedClass='ComboBox')
items = getComboboxItems(comboBox) items = getComboboxItems(comboBox)
selectComboboxItem(comboBox, items[3])#'10 Minuten' selectComboboxItem(comboBox, items[3])#'10 Minuten'
# finally get the button and click it # finally get the button and click it
button = findControl(form, wantedText = 'Erteilen') button = findControl(form, wantedText = 'Erteilen')
clickButton(button) clickButton(button)
break break
def findOutlook(self): def findOutlook(self):
outlookFound = 0 outlookFound = 0
try: try:
self.oOutlookApp = \ self.oOutlookApp = \
win32com.client.gencache.EnsureDispatch("Outlook.Application") win32com.client.gencache.EnsureDispatch("Outlook.Application")
outlookFound = 1 outlookFound = 1
except: except:
pass pass
return outlookFound return outlookFound
def createEmailMime(self, emails): def createEmailMime(self, emails):
# Create the container (outer) email message. # Create the container (outer) email message.
msg = MIMEMultipart() msg = MIMEMultipart()
msg['Subject'] = emails['Subject'].encode('utf-8') msg['Subject'] = emails['Subject'].encode('utf-8')
if emails.has_key('SenderEmailAddress'): if emails.has_key('SenderEmailAddress'):
sender = str(emails['SenderEmailAddress'].encode('utf-8')) sender = str(emails['SenderEmailAddress'].encode('utf-8'))
else: else:
sender = str(emails['SenderName'].encode('utf-8')) sender = str(emails['SenderName'].encode('utf-8'))
msg['From'] = sender msg['From'] = sender
recipients = [] recipients = []
if emails.has_key('Recipients'): if emails.has_key('Recipients'):
for rec in range(emails['Recipients'].__len__()): for rec in range(emails['Recipients'].__len__()):
recipients.append(getattr(emails['Recipients'].Item(rec+1), 'Address')) recipients.append(getattr(emails['Recipients'].Item(rec+1), 'Address'))
msg['To'] = COMMASPACE.join(recipients) msg['To'] = COMMASPACE.join(recipients)
else: else:
recipients.append(emails['To']) recipients.append(emails['To'])
msg['To'] = COMMASPACE.join(recipients) msg['To'] = COMMASPACE.join(recipients)
msg.preamble = emails['Body'].encode('utf-8') msg.preamble = emails['Body'].encode('utf-8')
return msg return msg
agents.register(OutlookCrawler, Master, name='crawl.outlook') agents.register(OutlookCrawler, Master, name='crawl.outlook')

102
agent/crawl/Outlook.txt → agent/crawl/outlook.txt
View file

@ -1,51 +1,51 @@
================================================ ================================================
Agents for Job Execution and Communication Tasks Agents for Job Execution and Communication Tasks
================================================ ================================================
>>> from cybertools.agent.base.agent import Master >>> from cybertools.agent.base.agent import Master
>>> config = ''' >>> config = '''
... controller(names=['core.sample']) ... controller(names=['core.sample'])
... scheduler(name='core') ... scheduler(name='core')
... logger(name='default', standard=30) ... logger(name='default', standard=30)
... ''' ... '''
>>> master = Master(config) >>> master = Master(config)
>>> master.setup() >>> master.setup()
OutlookCrawler OutlookCrawler
============== ==============
The agent uses Twisted's cooperative multitasking model. The agent uses Twisted's cooperative multitasking model.
OutlookCrawler is derived from MailCrawler. The OutlookCrawler returns a deferred OutlookCrawler is derived from MailCrawler. The OutlookCrawler returns a deferred
which itself holds a list of MailResource Objects. which itself holds a list of MailResource Objects.
Returns a deferred that must be supplied with a callback method (and in Returns a deferred that must be supplied with a callback method (and in
most cases also an errback method). most cases also an errback method).
The TestCase here is using subsidiary methods which replace calls to the "real Outlook The TestCase here is using subsidiary methods which replace calls to the "real Outlook
dlls". dlls".
>>> controller = master.controllers[0] >>> controller = master.controllers[0]
>>> controller.createAgent('crawl.outlook', 'sample02') >>> controller.createAgent('crawl.outlook', 'sample02')
In the next step we request the start of a job, again via the controller. In the next step we request the start of a job, again via the controller.
>>> controller.enterJob('outlook', 'sample02') >>> controller.enterJob('outlook', 'sample02')
The job is not executed immediately - we have to hand over control to The job is not executed immediately - we have to hand over control to
the twisted reactor first. the twisted reactor first.
>>> from cybertools.agent.crawl.outlook import OutlookCrawler >>> from cybertools.agent.crawl.outlook import OutlookCrawler
>>> from cybertools.agent.crawl.outlookstub import OutlookCrawlerStub >>> from cybertools.agent.crawl.outlookstub import OutlookCrawlerStub
>>> OutlookCrawler.findOutlook = OutlookCrawlerStub.findOutlook >>> OutlookCrawler.findOutlook = OutlookCrawlerStub.findOutlook
>>> OutlookCrawler = OutlookCrawlerStub >>> OutlookCrawler = OutlookCrawlerStub
>>> from cybertools.agent.tests import tester >>> from cybertools.agent.tests import tester
>>> tester.iterate() >>> tester.iterate()
Returning reference to Outlook Application Returning reference to Outlook Application
Retrieving Parameters Retrieving Parameters
Crawling Folders Crawling Folders
loading mails from folder loading mails from folder
Job 00001 completed; result: []; Job 00001 completed; result: [];

20
agent/crawl/outlookstub.py
View file

@ -1,20 +0,0 @@
from cybertools.agent.crawl.mail import MailCrawler
class OutlookCrawlerStub(MailCrawler):
def __init__(self):
pass
def findOutlook(self):
print "Returning reference to Outlook Application"
def fetchCriteria(self):
print "Retrieving Parameters"
def crawlFolders(self):
print "Crawling Folders"
self.loadMailsFromFolder()
def loadMailsFromFolder(self):
print "loading mails from folder"
self.createResource("SampleMail", "OutlookStubFolder", "OutlookStub")

15
agent/main.py
View file

@ -41,11 +41,22 @@ def getConfig():
return config return config
def setup(): def setup(configInfo=None):
master = Master(getConfig()) if configInfo is None:
configInfo = getConfig()
master = Master(configInfo)
setupEnvironment(master.config)
master.setup() master.setup()
print 'Starting agent application...' print 'Starting agent application...'
print 'Using controllers %s.' % ', '.join(master.config.controller.names) print 'Using controllers %s.' % ', '.join(master.config.controller.names)
return master
def setupEnvironment(config):
from cybertools.agent.base import agent, control, job, log, schedule
from cybertools.agent.core import agent, control, schedule
from cybertools.agent.control import cmdline
from cybertools.agent.crawl import base #, outlook
def startReactor(): def startReactor():

4
agent/system/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""
$Id$
"""

4
agent/system/windows/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""
$Id$
"""

24
agent/system/windows/api.py Normal file
View file

@ -0,0 +1,24 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Conficuration-controlled import of Windows API functions.
$Id$
"""

24
agent/testing/winapi.py Normal file
View file

@ -0,0 +1,24 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Fake Windows API functions for testing purposes.
$Id$
"""

2
agent/tests.py
View file

@ -43,7 +43,7 @@ def test_suite():
unittest.makeSuite(Test), unittest.makeSuite(Test),
DocFileSuite('README.txt', optionflags=flags), DocFileSuite('README.txt', optionflags=flags),
DocFileSuite('crawl/README.txt', optionflags=flags), DocFileSuite('crawl/README.txt', optionflags=flags),
DocFileSuite('crawl/Outlook.txt', optionflags=flags), #DocFileSuite('crawl/outlook.txt', optionflags=flags),
)) ))
return testSuite return testSuite