cco/cybertools
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rearrange system startup so that components are not registered during initial import but via a controlled setup

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/cybertools/trunk@2500 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2008-04-07 06:36:48 +00:00
parent 5aee688a12
commit e4108ed7d1
14 changed files with 313 additions and 268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
agent/README.txt
View file

@ -1,4 +1,4 @@
================================================
================================================
Agents for Job Execution and Communication Tasks
================================================
@ -101,8 +101,10 @@ configuration are provided by the controller.
So we are now ready to create a master agent and configure it by supplying
the path to the configuration file.
>>> from cybertools.agent.base.agent import Master
>>> master = Master(configFile)
>>> from cybertools.agent.main import setup
>>> master = setup(configFile)
Starting agent application...
Using controllers base.sample.
>>> configFile.close()
>>> master.config
@ -201,8 +203,9 @@ the twisted reactor.
... scheduler(name='core')
... logger(name='default', standard=30)
... '''
>>> master = Master(config)
>>> master.setup()
>>> master = setup(config)
Starting agent application...
Using controllers core.sample.
>>> master.scheduler
<cybertools.agent.core.schedule.Scheduler object ...>

7
agent/__init__.py
View file

@ -1,10 +1,3 @@
"""
$Id$
"""
# register default adapters
from cybertools.agent.base import agent, control, job, log, schedule
from cybertools.agent.core import agent, control, schedule
from cybertools.agent.control import cmdline
from cybertools.agent.crawl import base

12
agent/base/agent.py
View file

@ -60,22 +60,24 @@ class Master(Agent):
name = 'master'
scheduler = None
logger = None
def __init__(self, configuration):
if isinstance(configuration, Configurator):
config = configuration
self.config = configuration
else: # configuration is path to config file
config = self.config = Configurator()
config.load(configuration)
self.config = Configurator()
self.config.load(configuration)
self.master = self
self.controllers = []
self.children = {}
def setup(self):
config = self.config
self.logger = loggers(self, name=config.logger.name)
for n in config.controller.names:
self.controllers.append(controllers(self, n))
self.scheduler = schedulers(self, name=config.scheduler.name)
def setup(self):
for cont in self.controllers:
cont.setup()

2
agent/common.py
View file

@ -17,7 +17,7 @@
#
"""
Common stuff.
Miscellaneous common stuff.
$Id$
"""

8
agent/crawl/README.txt
View file

@ -4,15 +4,15 @@ Agents for Job Execution and Communication Tasks
($Id$)
>>> from cybertools.agent.base.agent import Master
>>> config = '''
... controller(names=['core.sample'])
... scheduler(name='core')
... logger(name='default', standard=30)
... '''
>>> master = Master(config)
>>> master.setup()
>>> from cybertools.agent.main import setup
>>> master = setup(config)
Starting agent application...
Using controllers core.sample.
Crawler

344
agent/crawl/outlook.py
View file

@ -1,172 +1,172 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Outlook Crawler Class.
$Id: mail.py 2493 2008-04-03 11:17:37Z helmutm $
"""
import re
from zope.interface import implements
from twisted.internet import defer
import win32com.client
import ctypes
import win32api, win32process, win32con
from cybertools.agent.base.agent import Agent, Master
from cybertools.agent.crawl.mail import MailCrawler
from cybertools.agent.crawl.mail import MailResource
from cybertools.agent.components import agents
# some constants
COMMASPACE = ', '
class OutlookCrawler(MailCrawler):
keys = ""
inbox = ""
subfolders = ""
pattern = ""
def collect(self, filter=None):
self.d = defer.Deferred()
self.oOutlookApp = None
if self.findOutlook():
self.fetchCriteria()
self.d.addCallback(self.crawlFolders)
else:
pass
#self.d.addErrback([])
return d
def fetchCriteria(self):
criteria = self.params
self.keys = criteria.get('keys')
self.inbox = criteria.get('inbox') #boolean
self.subfolders = criteria.get('subfolders') #boolean
self.pattern = criteria.get('pattern')
if self.pattern != '':
self.pattern = re.compile(criteria.get('pattern') or '.*')
def crawlFolders(self):
onMAPI = self.oOutlookApp.GetNamespace("MAPI")
ofInbox = \
onMAPI.GetDefaultFolder(win32com.client.constants.olFolderInbox)
# fetch mails from inbox
if self.inbox:
self.loadMailsFromFolder(ofInbox)
# fetch mails of inbox subfolders
if self.subfolders and self.pattern is None:
lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()):
# get a MAPI-subfolder object and load its emails
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
elif self.subfolders and self.pattern:
lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()):
# get specified MAPI-subfolder object and load its emails
if self.pattern.match(getattr(lInboxSubfolders.Item(of + 1), 'Name')):
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
return self.result
def loadMailsFromFolder(self, folder):
# get items of the folder
folderItems = getattr(folder, 'Items')
for item in range(len(folderItems)):
mail = folderItems.Item(item+1)
if mail.Class == win32com.client.constants.olMail:
if self.keys is None:
self.keys = []
for key in mail._prop_map_get_:
if isinstance(getattr(mail, key), (int, str, unicode)):
self.keys.append(key)
record = {}
for key in self.keys:
record[key] = getattr(mail, key)
# Create the mime email object
msg = self.createEmailMime(record)
# Create a resource and append it to the result list
self.createResource(msg, folder, "Microsoft Office Outlook")
def login(self):
pass
def handleOutlookDialog(self):
"""
This function handles the outlook dialog, which appears if someone
tries to access to MS Outlook.
"""
hwnd = None
while True:
hwnd = ctypes.windll.user32.FindWindowExA(None, hwnd, None, None)
if hwnd == None:
break
else:
val = u"\0" * 1024
ctypes.windll.user32.GetWindowTextW(hwnd, val, len(val))
val = val.replace(u"\000", u"")
if val and repr(val) == "u'Microsoft Office Outlook'":
print repr(val)
# get the Main Control
form = findTopWindow(wantedText='Microsoft Office Outlook')
controls = findControls(form)
# get the check box
checkBox = findControl(form, wantedText='Zugriff')
setCheckBox(checkBox, 1)
# get the combo box
comboBox = findControl(form, wantedClass='ComboBox')
items = getComboboxItems(comboBox)
selectComboboxItem(comboBox, items[3])#'10 Minuten'
# finally get the button and click it
button = findControl(form, wantedText = 'Erteilen')
clickButton(button)
break
def findOutlook(self):
outlookFound = 0
try:
self.oOutlookApp = \
win32com.client.gencache.EnsureDispatch("Outlook.Application")
outlookFound = 1
except:
pass
return outlookFound
def createEmailMime(self, emails):
# Create the container (outer) email message.
msg = MIMEMultipart()
msg['Subject'] = emails['Subject'].encode('utf-8')
if emails.has_key('SenderEmailAddress'):
sender = str(emails['SenderEmailAddress'].encode('utf-8'))
else:
sender = str(emails['SenderName'].encode('utf-8'))
msg['From'] = sender
recipients = []
if emails.has_key('Recipients'):
for rec in range(emails['Recipients'].__len__()):
recipients.append(getattr(emails['Recipients'].Item(rec+1), 'Address'))
msg['To'] = COMMASPACE.join(recipients)
else:
recipients.append(emails['To'])
msg['To'] = COMMASPACE.join(recipients)
msg.preamble = emails['Body'].encode('utf-8')
return msg
agents.register(OutlookCrawler, Master, name='crawl.outlook')
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Outlook Crawler Class.
$Id$
"""
import re
from zope.interface import implements
from twisted.internet import defer
import win32com.client
import ctypes
import win32api, win32process, win32con
from cybertools.agent.base.agent import Agent, Master
from cybertools.agent.crawl.mail import MailCrawler
from cybertools.agent.crawl.mail import MailResource
from cybertools.agent.components import agents
# some constants
COMMASPACE = ', '
class OutlookCrawler(MailCrawler):
keys = ""
inbox = ""
subfolders = ""
pattern = ""
def collect(self, filter=None):
self.d = defer.Deferred()
self.oOutlookApp = None
if self.findOutlook():
self.fetchCriteria()
self.d.addCallback(self.crawlFolders)
else:
pass
#self.d.addErrback([])
return d
def fetchCriteria(self):
criteria = self.params
self.keys = criteria.get('keys')
self.inbox = criteria.get('inbox') #boolean
self.subfolders = criteria.get('subfolders') #boolean
self.pattern = criteria.get('pattern')
if self.pattern != '':
self.pattern = re.compile(criteria.get('pattern') or '.*')
def crawlFolders(self):
onMAPI = self.oOutlookApp.GetNamespace("MAPI")
ofInbox = \
onMAPI.GetDefaultFolder(win32com.client.constants.olFolderInbox)
# fetch mails from inbox
if self.inbox:
self.loadMailsFromFolder(ofInbox)
# fetch mails of inbox subfolders
if self.subfolders and self.pattern is None:
lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()):
# get a MAPI-subfolder object and load its emails
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
elif self.subfolders and self.pattern:
lInboxSubfolders = getattr(ofInbox, 'Folders')
for of in range(lInboxSubfolders.__len__()):
# get specified MAPI-subfolder object and load its emails
if self.pattern.match(getattr(lInboxSubfolders.Item(of + 1), 'Name')):
self.loadMailsFromFolder(lInboxSubfolders.Item(of + 1))
return self.result
def loadMailsFromFolder(self, folder):
# get items of the folder
folderItems = getattr(folder, 'Items')
for item in range(len(folderItems)):
mail = folderItems.Item(item+1)
if mail.Class == win32com.client.constants.olMail:
if self.keys is None:
self.keys = []
for key in mail._prop_map_get_:
if isinstance(getattr(mail, key), (int, str, unicode)):
self.keys.append(key)
record = {}
for key in self.keys:
record[key] = getattr(mail, key)
# Create the mime email object
msg = self.createEmailMime(record)
# Create a resource and append it to the result list
self.createResource(msg, folder, "Microsoft Office Outlook")
def login(self):
pass
def handleOutlookDialog(self):
"""
This function handles the outlook dialog, which appears if someone
tries to access to MS Outlook.
"""
hwnd = None
while True:
hwnd = ctypes.windll.user32.FindWindowExA(None, hwnd, None, None)
if hwnd == None:
break
else:
val = u"\0" * 1024
ctypes.windll.user32.GetWindowTextW(hwnd, val, len(val))
val = val.replace(u"\000", u"")
if val and repr(val) == "u'Microsoft Office Outlook'":
print repr(val)
# get the Main Control
form = findTopWindow(wantedText='Microsoft Office Outlook')
controls = findControls(form)
# get the check box
checkBox = findControl(form, wantedText='Zugriff')
setCheckBox(checkBox, 1)
# get the combo box
comboBox = findControl(form, wantedClass='ComboBox')
items = getComboboxItems(comboBox)
selectComboboxItem(comboBox, items[3])#'10 Minuten'
# finally get the button and click it
button = findControl(form, wantedText = 'Erteilen')
clickButton(button)
break
def findOutlook(self):
outlookFound = 0
try:
self.oOutlookApp = \
win32com.client.gencache.EnsureDispatch("Outlook.Application")
outlookFound = 1
except:
pass
return outlookFound
def createEmailMime(self, emails):
# Create the container (outer) email message.
msg = MIMEMultipart()
msg['Subject'] = emails['Subject'].encode('utf-8')
if emails.has_key('SenderEmailAddress'):
sender = str(emails['SenderEmailAddress'].encode('utf-8'))
else:
sender = str(emails['SenderName'].encode('utf-8'))
msg['From'] = sender
recipients = []
if emails.has_key('Recipients'):
for rec in range(emails['Recipients'].__len__()):
recipients.append(getattr(emails['Recipients'].Item(rec+1), 'Address'))
msg['To'] = COMMASPACE.join(recipients)
else:
recipients.append(emails['To'])
msg['To'] = COMMASPACE.join(recipients)
msg.preamble = emails['Body'].encode('utf-8')
return msg
agents.register(OutlookCrawler, Master, name='crawl.outlook')

102
agent/crawl/Outlook.txt → agent/crawl/outlook.txt
View file

@ -1,51 +1,51 @@
================================================
Agents for Job Execution and Communication Tasks
================================================
>>> from cybertools.agent.base.agent import Master
>>> config = '''
... controller(names=['core.sample'])
... scheduler(name='core')
... logger(name='default', standard=30)
... '''
>>> master = Master(config)
>>> master.setup()
OutlookCrawler
==============
The agent uses Twisted's cooperative multitasking model.
OutlookCrawler is derived from MailCrawler. The OutlookCrawler returns a deferred
which itself holds a list of MailResource Objects.
Returns a deferred that must be supplied with a callback method (and in
most cases also an errback method).
The TestCase here is using subsidiary methods which replace calls to the "real Outlook
dlls".
>>> controller = master.controllers[0]
>>> controller.createAgent('crawl.outlook', 'sample02')
In the next step we request the start of a job, again via the controller.
>>> controller.enterJob('outlook', 'sample02')
The job is not executed immediately - we have to hand over control to
the twisted reactor first.
>>> from cybertools.agent.crawl.outlook import OutlookCrawler
>>> from cybertools.agent.crawl.outlookstub import OutlookCrawlerStub
>>> OutlookCrawler.findOutlook = OutlookCrawlerStub.findOutlook
>>> OutlookCrawler = OutlookCrawlerStub
>>> from cybertools.agent.tests import tester
>>> tester.iterate()
Returning reference to Outlook Application
Retrieving Parameters
Crawling Folders
loading mails from folder
Job 00001 completed; result: [];
================================================
Agents for Job Execution and Communication Tasks
================================================
>>> from cybertools.agent.base.agent import Master
>>> config = '''
... controller(names=['core.sample'])
... scheduler(name='core')
... logger(name='default', standard=30)
... '''
>>> master = Master(config)
>>> master.setup()
OutlookCrawler
==============
The agent uses Twisted's cooperative multitasking model.
OutlookCrawler is derived from MailCrawler. The OutlookCrawler returns a deferred
which itself holds a list of MailResource Objects.
Returns a deferred that must be supplied with a callback method (and in
most cases also an errback method).
The TestCase here is using subsidiary methods which replace calls to the "real Outlook
dlls".
>>> controller = master.controllers[0]
>>> controller.createAgent('crawl.outlook', 'sample02')
In the next step we request the start of a job, again via the controller.
>>> controller.enterJob('outlook', 'sample02')
The job is not executed immediately - we have to hand over control to
the twisted reactor first.
>>> from cybertools.agent.crawl.outlook import OutlookCrawler
>>> from cybertools.agent.crawl.outlookstub import OutlookCrawlerStub
>>> OutlookCrawler.findOutlook = OutlookCrawlerStub.findOutlook
>>> OutlookCrawler = OutlookCrawlerStub
>>> from cybertools.agent.tests import tester
>>> tester.iterate()
Returning reference to Outlook Application
Retrieving Parameters
Crawling Folders
loading mails from folder
Job 00001 completed; result: [];

20
agent/crawl/outlookstub.py
View file

@ -1,20 +0,0 @@
from cybertools.agent.crawl.mail import MailCrawler
class OutlookCrawlerStub(MailCrawler):
def __init__(self):
pass
def findOutlook(self):
print "Returning reference to Outlook Application"
def fetchCriteria(self):
print "Retrieving Parameters"
def crawlFolders(self):
print "Crawling Folders"
self.loadMailsFromFolder()
def loadMailsFromFolder(self):
print "loading mails from folder"
self.createResource("SampleMail", "OutlookStubFolder", "OutlookStub")

15
agent/main.py
View file

@ -41,11 +41,22 @@ def getConfig():
return config
def setup():
master = Master(getConfig())
def setup(configInfo=None):
if configInfo is None:
configInfo = getConfig()
master = Master(configInfo)
setupEnvironment(master.config)
master.setup()
print 'Starting agent application...'
print 'Using controllers %s.' % ', '.join(master.config.controller.names)
return master
def setupEnvironment(config):
from cybertools.agent.base import agent, control, job, log, schedule
from cybertools.agent.core import agent, control, schedule
from cybertools.agent.control import cmdline
from cybertools.agent.crawl import base #, outlook
def startReactor():

4
agent/system/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""
$Id$
"""

4
agent/system/windows/__init__.py Normal file
View file

@ -0,0 +1,4 @@
"""
$Id$
"""

24
agent/system/windows/api.py Normal file
View file

@ -0,0 +1,24 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Conficuration-controlled import of Windows API functions.
$Id$
"""

24
agent/testing/winapi.py Normal file
View file

@ -0,0 +1,24 @@
#
# Copyright (c) 2008 Helmut Merz helmutm@cy55.de
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
"""
Fake Windows API functions for testing purposes.
$Id$
"""

2
agent/tests.py
View file

@ -43,7 +43,7 @@ def test_suite():
unittest.makeSuite(Test),
DocFileSuite('README.txt', optionflags=flags),
DocFileSuite('crawl/README.txt', optionflags=flags),
DocFileSuite('crawl/Outlook.txt', optionflags=flags),
#DocFileSuite('crawl/outlook.txt', optionflags=flags),
))
return testSuite