diff --git a/agent/core.py b/agent/core.py index 95a7c75..296bc14 100644 --- a/agent/core.py +++ b/agent/core.py @@ -23,6 +23,7 @@ $Id$ """ from time import time +import tempfile from zope.interface import implements from loops.agent.interfaces import IAgent from loops.agent.config import Configurator @@ -55,6 +56,7 @@ class Agent(object): self.stopper = Stopper() self.stopper.scheduler = self.scheduler self.logger = Logger(self) + self.tempdir = tempfile.mkdtemp(prefix='loops_') def scheduleJobsFromConfig(self, stop=False): config = self.config diff --git a/agent/crawl/outlook.py b/agent/crawl/outlook.py index 14e9181..dc4039a 100644 --- a/agent/crawl/outlook.py +++ b/agent/crawl/outlook.py @@ -9,146 +9,107 @@ Tobias Schmid 26.07.2007 """ import win32com.client +import re from email.mime.multipart import MIMEMultipart +from twisted.internet.defer import Deferred +from twisted.internet.task import coiterate +from zope.interface import implements + +from loops.agent.interfaces import IResource +from loops.agent.crawl.base import CrawlingJob as BaseCrawlingJob +from loops.agent.crawl.base import Metadata + + # DEBUG FLAGS -DEBUG = 0 +DEBUG = 1 DEBUG_WRITELINE = 1 # some constants COMMASPACE = ', ' -class MSOutlook: - def __init__(self): - self.outlookFound = 0 +class CrawlingJob(BaseCrawlingJob): + + keys = "" + inbox = "" + subfolders = "" + pattern = "" + + def collect(self): + self.collected = [] + coiterate(self.crawlOutlook()).addCallback(self.finished) + # TODO: addErrback() + self.deferred = Deferred() + return self.deferred + + def finished(self, result): + self.deferred.callback(self.collected) + + def crawlOutlook(self): + outlookFound = 0 try: - self.oOutlookApp = \ + oOutlookApp = \ win32com.client.gencache.EnsureDispatch("Outlook.Application") - self.outlookFound = 1 + outlookFound = 1 except: print "MSOutlook: unable to load Outlook" - self.records = [] - self.mailList = [] - - - def loadInbox(self, keys=None, subfolders=False): - if not self.outlookFound: + records = [] + + if not outlookFound: return + # fetch the params + criteria = self.params + self.keys = criteria.get('keys') + self.inbox = criteria.get('inbox') #boolean + self.subfolders = criteria.get('subfolders') #boolean + self.pattern = criteria.get('pattern') + if self.pattern != '': + self.pattern = re.compile(criteria.get('pattern') or '.*') + else: + self.pattern = None + + if DEBUG_WRITELINE: print 'MSOutlook.loadInbox() ===> starting' - # this should use more try/except blocks or nested blocks - onMAPI = self.oOutlookApp.GetNamespace("MAPI") + # catch Inbox folder + onMAPI = oOutlookApp.GetNamespace("MAPI") ofInbox = \ onMAPI.GetDefaultFolder(win32com.client.constants.olFolderInbox) # fetch the mails of the inbox folder if DEBUG_WRITELINE: print 'MSOutlook.loadInbox() ===> fetch mails of inbox folder' - - for om in range(len(ofInbox.Items)): - mail = ofInbox.Items.Item(om + 1) - if mail.Class == win32com.client.constants.olMail: - if keys is None: - # if we were't give a set of keys to use - # then build up a list of keys that we will be - # able to process - # I didn't include fields of type time, though - # those could probably be interpreted - keys = [] - for key in mail._prop_map_get_: - if isinstance(getattr(mail, key), (int, str, unicode)): - keys.append(key) - if DEBUG: - keys.sort() - print 'Fields\n======================================' - for key in keys: - print key - record = {} - for key in keys: - record[key] = getattr(mail, key) - - # Create the container (outer) email message. - msg = MIMEMultipart() - # subject - msg['Subject'] = record['Subject'].encode('utf-8') - - # sender - sender = str(record['SenderName'].encode('utf-8')) #SenderEmailAddress - msg['From'] = sender - - #recipients - recipients = [] - for rec in range(record['Recipients'].__len__()): - recipients.append(getattr(record['Recipients'].Item(rec+1), 'Address')) - msg['To'] = COMMASPACE.join(recipients) - - # message - msg.preamble = record['Body'].encode('utf-8') - - # add the email message to the list - self.mailList.append(msg) - self.records.append(record) - - - """ - * Fetch the mails of the inbox subfolders - """ - if subfolders: + + # fetch mails from inbox + if self.inbox: + self.loadEmail(ofInbox) + + # fetch mails of inbox subfolders + if self.subfolders and self.pattern is None: if DEBUG_WRITELINE: print 'MSOutlook.loadInbox() ===> fetch emails of subfolders' lInboxSubfolders = getattr(ofInbox, 'Folders') for of in range(lInboxSubfolders.__len__()): - # get a MAPI-folder object - oFolder = lInboxSubfolders.Item(of + 1) - # get items of the folder - folderItems = getattr(oFolder, 'Items') - for item in range(len(folderItems)): - mail = folderItems.Item(item+1) - if mail.Class == win32com.client.constants.olMail: - if keys is None: - keys = [] - for key in mail._prop_map_get_: - if isinstance(getattr(mail, key), (int, str, unicode)): - keys.append(key) - - if DEBUG: - keys.sort() - print 'Fiels\n=======================================' - for key in keys: - print key - - record = {} - for key in keys: - record[key] = getattr(mail, key) - if DEBUG: - print of - - # Create the container (outer) email message. - msg = MIMEMultipart() - # subject - msg['Subject'] = record['Subject'].encode('utf-8') - - # sender - sender == record['SenderName'].encode('utf-8') #SenderEmailAddress - msg['From'] = sender - - # recipients - for rec in range(record['Recipients'].__len__()): - recipients.append(getattr(record['Recipients'].Item(rec+1), 'Address')) - msg['To'] = COMMASPACE.join(recipients) - - # message - msg.preamble = record['Body'].encode('utf-8') - - # add the email message to the list - self.mailList.append(msg) - self.records.append(record) + # get a MAPI-folder object and load its emails + self.loadEmail(lInboxSubfolders.Item(of + 1)) + + # pattern, just read the specified subfolder + elif self.subfolders and self.pattern: + + if DEBUG_WRITELINE: + print 'MSOutlook.loadInbox() ===> fetch emails of specified subfolder' + lInboxSubfolders = getattr(ofInbox, 'Folders') + for of in range(lInboxSubfolders.__len__()): + # get a MAPI-folder object and load its emails + if self.pattern.match(getattr(lInboxSubfolders.Item(of + 1), 'Name')): + self.loadEmail(lInboxSubfolders.Item(of + 1)) #oFolder + if DEBUG: print 'number of mails in Inbox:', len(ofInbox.Items) @@ -157,7 +118,7 @@ class MSOutlook: # get Count-Attribute of _Folders class iInboxSubfoldersCount = getattr(lInboxSubfolders, 'Count') # the Item-Method of the _Folders class returns a MAPIFolder object - oFolder = lInboxSubfolders.Item(0) #1 + oFolder = lInboxSubfolders.Item(1) print 'Count of Inbox-SubFolders:', iInboxSubfoldersCount print 'Inbox sub folders (Folder/Mails):' @@ -168,45 +129,78 @@ class MSOutlook: if DEBUG_WRITELINE: print 'MSOutlook.loadInbox() ===> ending' + yield '1' + + + def loadEmail(self, oFolder): + # get items of the folder + folderItems = getattr(oFolder, 'Items') + for item in range(len(folderItems)): + mail = folderItems.Item(item+1) + if mail.Class == win32com.client.constants.olMail: + if self.keys is None: + self.keys = [] + for key in mail._prop_map_get_: + if isinstance(getattr(mail, key), (int, str, unicode)): + self.keys.append(key) + + if DEBUG: + self.keys.sort() + print 'Fiels\n=======================================' + for key in self.keys: + print key + + record = {} + for key in self.keys: + record[key] = getattr(mail, key) + if DEBUG: + print str(item) - return self.mailList - - + # Create the mime email object + msg = self.createEmailMime(record) + + # list with mime objects + self.collected.append((OutlookResource(msg))) + + + def createEmailMime(self, emails): + # Create the container (outer) email message. + msg = MIMEMultipart() + # subject + msg['Subject'] = emails['Subject'].encode('utf-8') + + # sender + if emails.has_key('SenderEmailAddress'): + sender = str(emails['SenderEmailAddress'].encode('utf-8')) + else: + sender = str(emails['SenderName'].encode('utf-8')) + msg['From'] = sender + + #recipients + recipients = [] -if __name__ == '__main__': - if DEBUG: - print 'attempting to load Outlook' - oOutlook = MSOutlook() - # delayed check for Outlook on win32 box - if not oOutlook.outlookFound: - print 'Outlook not found' - sys.exit(1) + if emails.has_key('Recipients'): + for rec in range(emails['Recipients'].__len__()): + recipients.append(getattr(emails['Recipients'].Item(rec+1), 'Address')) + msg['To'] = COMMASPACE.join(recipients) + else: + recipients.append(emails['To']) + msg['To'] = COMMASPACE.join(recipients) + + # message + msg.preamble = emails['Body'].encode('utf-8') + + return msg + + +class OutlookResource(object): - fieldsMail = ['Body', - 'HTMLBody', - 'CC', - 'SenderName', - 'Recipients', - 'To', - 'Attachments', - 'Subject' - ] - # 'BodyFormat', removed BodyFormat temporarily because it is not available in Outlook.9 (Office2000) - # 'SenderEmailAddress', replaced by SenderName + implements(IResource) - if DEBUG: - import time - print 'loading records...' - startTime = time.time() - - mails = oOutlook.loadInbox(fieldsMail) + def __init__(self, oEmail): + self.oEmail = oEmail - for elem in mails: - print str(elem) + @property + def data(self): + return self.oEmail - if DEBUG_WRITELINE: - print '***Back in main() with some emails in a list....***' - print 'Mails fetched from MSOutlook inbox folder:', mails.__len__() - - if DEBUG: - print 'loading took %f seconds' % (time.time() - startTime) diff --git a/agent/ui/templates/filecrawl.html b/agent/ui/templates/filecrawl.html new file mode 100644 index 0000000..8e6e82c --- /dev/null +++ b/agent/ui/templates/filecrawl.html @@ -0,0 +1,228 @@ + + + + + + + + + +
+
+
+ + + +
+
+
+ +
+
+

+
+
+

Agent: Create Filesystem Crawler Job

+ +
    +
  • Configuration page for Filesystem Crawler Jobs
    • +
+
+
+

Overview

+
    +
  • File Collection
    • +
+
+ +

+

+

+
+
+
+ + Filesystem Crawl Settings + +
+ Directories to crawl + + + + + + + + + +
+ +

+ please use ';' as delimiters +

+

+ you can also use patterns like regular expressions +

+ 		+ +
+ +

+ please use ';' as delimiters +

+

+ you can also use patterns like regular expressions +

+ 		+ +
+
+
+ + Filter criteria patterns + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ + + +
+ + + +
+ +

+ no size means that transferred files might be very large! +

+ 		+ +
+ + + +
+ + + +
+ + + +
+ + + +
+
+
+ + Job Interval + + + + + + + + + + +
+ + + +
+ + + +
+
+ +
+
+
+
+
+
+
+ +
+
+ +
+
+ +
+ + + + \ No newline at end of file diff --git a/agent/ui/templates/mail.html b/agent/ui/templates/mail_detailed.html similarity index 73% rename from agent/ui/templates/mail.html rename to agent/ui/templates/mail_detailed.html index 0805686..482222d 100644 --- a/agent/ui/templates/mail.html +++ b/agent/ui/templates/mail_detailed.html @@ -47,36 +47,29 @@

Overview

    -
  • Mail Collection
    • +
  • Mail in Detail
- +
- - - -

+

- From + Field - CC - - Subject - - Date - - Source Folder + Value
+
+ [back to Ressources overview]
diff --git a/agent/ui/templates/mailcrawl.html b/agent/ui/templates/mailcrawl.html index 18a7fb4..c7201a9 100644 --- a/agent/ui/templates/mailcrawl.html +++ b/agent/ui/templates/mailcrawl.html @@ -65,22 +65,29 @@ + + + +
- + - +
- + - + +
+ + +
diff --git a/agent/ui/templates/navigation.html b/agent/ui/templates/navigation.html index df46da4..c2026f8 100644 --- a/agent/ui/templates/navigation.html +++ b/agent/ui/templates/navigation.html @@ -16,19 +16,19 @@
- collect Outlook Mails + add outlook crawl job
- - view Outlook Mails - + + add filesystem crawl job
- - add job + + view collected ressources +
diff --git a/agent/ui/templates/ressourceview.html b/agent/ui/templates/ressourceview.html new file mode 100644 index 0000000..0de0f0a --- /dev/null +++ b/agent/ui/templates/ressourceview.html @@ -0,0 +1,79 @@ + + + + + + + + + +
+
+
+ + + +
+
+
+ +
+
+

+
+
+

Agent: collected ressources

+ +
    +
  • All currently available objects that were collected by loops jobs
    • +
+
+
+

Overview

+
    +
  • Ressource Collection
    • +
+
+
+ + +

+

+ + +

+

+ +
+
+
+
+
+ +
+
+ +
+
+ +
+ + + + \ No newline at end of file diff --git a/agent/ui/web.py b/agent/ui/web.py index e6bc962..8070685 100644 --- a/agent/ui/web.py +++ b/agent/ui/web.py @@ -5,21 +5,28 @@ # version: 0.1 #------------------------------------------------------ +import sys import os +import tempfile +import cPickle +import traceback +import time +import email + from nevow import loaders, rend, static, url, inevow, tags from nevow.inevow import IRequest from twisted.internet import defer -from loops.agent.crawl.outlook import MSOutlook +from loops.agent.crawl import outlook from loops.agent import core +from loops.agent.crawl.outlook import OutlookResource +from loops.agent.crawl.filesystem import FileResource # ---- global definitions and settings --------------------------------- os.environ['HOME'] = os.path.dirname(core.__file__) OUTLOOK2000 = 1 -JOBCONFIGS = os.path.join(os.path.expanduser('~'),'jobconf.txt') -MAIL_DIR = os.path.join(os.path.expanduser('~'),'mails') -agent = core.Agent() -config = agent.config +PICKLE_MAILS = 0 +DEBUG = 1 resourcesDirectory = 'resources' templatesDirectory = 'templates' #---------------------------------------------------------- @@ -28,17 +35,10 @@ templatesDirectory = 'templates' def template(fn): return loaders.xmlfile(os.path.join(templatesDirectory, fn)) -def getConfigIndex(index_name, default_value, fn=None): - iterator = 0 - index_used = True - #check what crawling jobs exist to create a valid job index - while index_used == True: - if config.crawl[iterator].setdefault(index_name,default_value) == default_value: - index_used = False - else: - index_used = True - iterator = iterator + 1 - return iterator +def getConfigIndex(agentobj, index_name, default_value, fn=None): + """get the number of jobs currently in config file""" + index = len(agentobj.config.crawl) + return index # AgentHome @@ -50,10 +50,11 @@ class AgentHome(rend.Page): Instance variables: usermode -- can be 'Simple' or 'Advanced' + agent -- agent object where config and tempdir attributes can be accessed Class methods: - At the moment all methods of this class except the __init__ although public, - are intended to be called directly or stand-alone. + At the moment all methods of this class except the __init__ although + public, are intended to be called directly or stand-alone. __init__ -- Load the initial start-up settings from config @@ -62,11 +63,12 @@ class AgentHome(rend.Page): child_resources = static.File(resourcesDirectory) docFactory = template('agent.html') - def __init__(self,first_start=1): + def __init__(self, agent=None, first_start=1): """ Initialize the AgentHome object. - If first_start = 1 then the initalization procedure is run, which mainly - loads the stored settings from the agent ini-file into instance variables of the class. + If first_start = 1 then the initalization procedure is run, which + mainly loads the stored settings from the agent ini-file into instance + variables of the class. Keyword arguments: first_start -- defaults to 1 @@ -74,8 +76,11 @@ class AgentHome(rend.Page): """ print "[PAGECLASS] AgentHome" if first_start == 1: - self.usermode = config.ui.web.setdefault('usermode','Simple') - print "[AgentHome] setting self.usermode: ", config.ui.web.usermode + self.agent = core.Agent() + self.usermode = self.agent.config.ui.web.setdefault('usermode', 'Simple') + print "[AgentHome] setting self.usermode: ", self.agent.config.ui.web.usermode + else: + self.agent = agent """ def locateChild(self, ctx, segments): @@ -84,44 +89,49 @@ class AgentHome(rend.Page): # calls to child pages - def child_joboverview(self,context): + def child_joboverview(self, context): """ User requested page from menue: 'job overview' """ - return JobOverView() + return JobOverView(self.agent) - def child_collectOutlookMails(self,context): + def child_collectOutlookMails(self, context): """Display page for starting Outlook Crawler""" - return AgentOutlookMailCrawl() + return AgentOutlookMailCrawl(self.agent) - def child_viewOutlookMails(self,context): + def child_collectFilesystem(self, context): + """Display page for starting Filesystem Crawler""" + return AgentFilesystemCrawl(self.agent) + + def child_viewRessources(self, context): """Display page that shows all currently collected Outlook Mails.""" - return AgentOutlookMailView() + return RessourceView(self.agent) # "Startpage" methods (class AgentHome) - def child_changeUserMode(self,context): + def child_changeUserMode(self, context): """Change user mode. User has klicked the Change User Mode button, so change UserMode from Simple <-> Professional """ - print "[child_changeUserMode] UserMode: ", config.ui.web.usermode - print "[child_changeUserMode] ----retrieving form values----" + if DEBUG: + print "[child_changeUserMode] UserMode: ", self.agent.config.ui.web.usermode form = IRequest(context).args if form != {}: for elem in form: print "[child_changeUserMode] ", form[elem] - if config.ui.web.usermode == "Simple": - config.ui.web.usermode = "Advanced" + if self.agent.config.ui.web.usermode == "Simple": + self.agent.config.ui.web.usermode = "Advanced" else: - config.ui.web.usermode = "Simple" - print "[child_changeUserMode] : ", config.ui.web.usermode - config.save() - return AgentHome(first_start=0) + self.agent.config.ui.web.usermode = "Simple" + self.agent.config.save() + if DEBUG: + print "====> changed UserMode: ", self.agent.config.ui.web.usermode + return AgentHome(self.agent, first_start=0) # "job overview" methods (class JobOverView) - def child_ViewJobDetails(self,context): + def child_ViewJobDetails(self, context): """Get details for the selected job. Reads all information about the selected job from file/ database. @@ -129,47 +139,44 @@ class AgentHome(rend.Page): """ selected_job = ((IRequest(context).uri).split("?"))[1] - JOBDETAILS = 'joblist.txt' - file_pointer = open(JOBDETAILS,"r") - job_details = "" - job_list = file_pointer.readlines() - for line in job_list: - if line.startswith(selected_job): - job_details = line - break - file_pointer.close() - job_details = job_details.split(";") - return JobOverViewDetails(job_details) + crawl_index = selected_job.split(".")[1] + + return JobOverViewDetails(self.agent, crawl_index) - # "collect Outlook Mails" methods (class AgentOutlookMailCrawl) + # "add outlook crawl job" methods (class AgentOutlookMailCrawl) def child_submitOutlookCrawlJob(self,context): """Initiate Outlook crawling job as requested by user.""" - - ag = core.Agent() - config2 = ag.config - conf = config2 + ag = self.agent + conf = ag.config crawlSubfolder = False form = IRequest(context).args if form != {}: - iterator = getConfigIndex('type','unused') + index = getConfigIndex(self.agent, 'type', 'unused') #save job configuration if form['mailCrawlInterval'][0] == "oneTime": - conf.load('crawl[%i].state = "completed"' %(iterator)) + conf.crawl[index].state = 'completed' else: - conf.load('crawl[%i].state = "active"' %(iterator)) - conf.load('crawl[%i].jobid = "outlook%i"' %(iterator, iterator)) - conf.load('crawl[%i].type = "OutlookMail"' %(iterator)) - conf.load('crawl[%i].folder = "%s"'%(iterator,form["searchFolder"][0])) #always Inbox, ev. read from Outlook which folders are available? - conf.load('crawl[%i].getsubfolder = "%s"'%(iterator,form["selectSubfolder"][0])) - conf.load('crawl[%i].latest = ""'%(iterator)) #TODO: who does parsing? - conf.load('crawl[%i].filter_criteria = "%s"'%(iterator,form["selectFilterCriteria"][0])) - conf.load('crawl[%i].filter_pattern = "%s"'%(iterator,form["filterPattern"][0])) - conf.load('crawl[%i].content_format = "%s"'%(iterator,form["selectContentFormat"][0])) - conf.load('crawl[%i].include_attachements = "%s"'%(iterator,form["selectAttachements"][0])) - conf.load('crawl[%i].interval = "%s"'%(iterator,form["mailCrawlInterval"][0])) + conf.crawl[index].state = 'active' + conf.crawl[index].jobid = 'outlook.%i' %(index) + conf.crawl[index].type = 'OutlookMail' + if form.has_key('inbox'): + conf.crawl[index].getinbox = '%s'%(str(form["inbox"][0])) + else: + conf.crawl[index].getinbox = 'False' + if form.has_key('subfolders'): + conf.crawl[index].getsubfolder = '%s'%(str(form["subfolders"][0])) + else: + conf.crawl[index].getsubfolder = 'False' + conf.crawl[index].subfolderpat = '%s'%(str(form["pattern"][0])) + conf.crawl[index].latest = '%i'%(index) + #TODO: how to find out what the most actual element is? + conf.crawl[index].filter_criteria = '%s'%(form["selectFilterCriteria"][0]) + conf.crawl[index].filter_pattern = '%s'%(form["filterPattern"][0]) + conf.crawl[index].content_format = '%s'%(form["selectContentFormat"][0]) + conf.crawl[index].include_attachements = '%s'%(form["selectAttachements"][0]) + conf.crawl[index].interval = '%s'%(form["mailCrawlInterval"][0]) conf.save() - if form["mailCrawlInterval"][0] == 'oneTime': if OUTLOOK2000 == 1: #Outlook2000 (= Outlook 9) has different attributes @@ -193,57 +200,157 @@ class AgentHome(rend.Page): 'Attachments', 'Subject' ] - if form["selectSubfolder"][0] == 'Yes': - crawlSubfolder = True - - OutlookObj = MSOutlook() - mails = OutlookObj.loadInbox(fieldsMail,crawlSubfolder) - # currently deferreds are not implemented in outlook.py - #deferred.addCallback(self.defMailCrawl,context) - #deferred.addErrback(self.defMailCrawlError,context) - #return deferred - return self.defMailCrawl(mails,context) + # all form fileds in the html template are named according to + # the dictionary name if expected to be in params, + # this way it is not necessary to alter the code if another + # parameter is added + params = {} + for elem in form.items(): + params[elem[0]] = elem[1][0] + outlook_job = outlook.CrawlingJob() + outlook_job.params = params + deferred = outlook_job.collect() + deferred.addCallback(self.defMailCrawl, index, params, context) + deferred.addErrback(self.defMailCrawlError,context) + return deferred else: #TODO implement forwarding to next form (scheduler) - return AgentOutlookMailCrawl("Scheduled Mail Crawling not implemented yet.") + return AgentOutlookMailCrawl(self.agent, "Scheduled Mail Crawling not implemented yet.") else: - return AgentOutlookMailCrawl("An error occurred: form data has been empty.") - return AgentOutlookMailCrawl("Crawl Job settings have been saved.") + return AgentOutlookMailCrawl(self.agent, "An error occurred: form data has been empty.") + return AgentOutlookMailCrawl(self.agent, "Crawl Job settings have been saved.") - def defMailCrawl(self,mail_collection,context): + def defMailCrawl(self, mail_collection, index, params, context): """Save and Forward the mail collection to a page view.""" - files = os.listdir(MAIL_DIR) - filename = "Mail%i" %(len(files)) - print "Writing emails to files." + if DEBUG: + print "====> seting and saving mails to disk" + print "====> agent base dir: ",self.agent.tempdir + tmpdir = tempfile.mkdtemp(prefix='mailJob_%i_'%(index), dir=self.agent.tempdir) + if DEBUG: + print "====> outlook job dir: ", tmpdir + files = os.listdir(tmpdir) + filenum = len(files) for elem in mail_collection: - fp = open((os.path.join(MAIL_DIR,filename)),'w') - fp.write(str(elem)) - fp.close() - return AgentOutlookMailView(mail_collection) + tmpfile = tempfile.mkstemp(prefix='%i_mail_'%(filenum), dir=tmpdir) + if PICKLE_MAILS == 0: + # write data as MIME string into a text file + os.write(tmpfile[0], elem.data.as_string()) + else: + # alternatively also writing the list as a whole might be + # more performant (less disc I/O) but how to store the + # path so that the user can have a look at the mail view + # page and the page knows in which directory to look? + # Idea could be a object that stores all current paths and + # their crawlIDs + os.write(tmpfile[0], cPickle.dumps(elem)) + os.close(tmpfile[0]) + filenum = filenum + 1 + return RessourceView(self.agent, mail_collection, "The collected mails have been saved in %s"%(tmpdir), tmpdir) - def defMailCrawlError(self,ErrorMessage,context): + def defMailCrawlError(self, ErrorMessage, context): """Handles errors that ocurred in the MailCrawler.""" - return AgentHome(first_start=0) + #TODO: handle whole exception information + return traceback.format_exc() + + # "view collected ressources" methods (class RessourceViewView) + + def child_viewRessourceDetails(self,context): + """Get details for the selected ressource object. + + Reads all information about the selected ressource from file/ or the + mail_collection list. + Returns page object which displays the available information. + + """ + selected_item = ((IRequest(context).uri).split("?"))[1] + form = IRequest(context).args + if form != {}: + requested_file="" + if DEBUG: + print "[child_viewRessourceDetails]" + print "====> selected item: ", selected_item + print "====> printing items in directory" + fileslist = os.listdir(form['ressourceObjData'][0]) + for elem in fileslist: + if DEBUG: + print elem + if elem.startswith(("%i_mail_"%(int(selected_item))), 0): + requested_file = elem + requested_file = os.path.join(form['ressourceObjData'][0], requested_file) + if requested_file.find("mail") > 0: + fp = open(requested_file, "rb") + mail_parser = email.Parser.Parser() + mailobj = OutlookResource(mail_parser.parse(fp)) + fp.close() + mail = [] + for elem in mailobj.data.items(): + if DEBUG: + print "====> building MIMEObject" + print "====> appending attribute: ", elem + mail.append(elem) + if hasattr(mailobj.data, 'preamble'): + if DEBUG: + print "====> copying preamble contents" + mail.append(['Preamble', mailobj.data.preamble]) + else: + mail.append(['Preamble','']) + if hasattr(mailobj.data,'epilogue'): + if DEBUG: + print "====> copying epilogue contents" + mail.append(['Epilogue', mailobj.data.epilogue]) + else: + mail.append(['Epilogue', '']) + return OutlookMailDetail(agent=self.agent, pagemessage="", mail=mail, filename=requested_file) + elif requested_file.find("file") > 0: + #TODO implement file analyzing + return FileObjectDetail(agent=self.agent, pagemessage="", fileobj=fileobj, filename=requested_file) + if os.path.isdir(selected_item): + # selected item is a folder -> change to that folder + if DEBUG: + print "====> selected item is a directory! changing into directory" + return RessourceView(self.agent, ressource_collection=[], pagemessage="changed to folder %s" %(selected_item), tmpdir=selected_item) + elif os.path.isfile(selected_item): + if selected_item.find("mail") > 0: + fp = open(selected_item, "rb") + mail_parser = email.Parser.Parser() + mailobj = OutlookResource(mail_parser.parse(fp)) + fp.close() + mail = [] + for elem in mailobj.data.items(): + mail.append(elem) + if hasattr(mailobj.data,'preamble'): + mail.append(['Preamble', mailobj.data.preamble]) + else: + mail.append(['Preamble','']) + if hasattr(mailobj.data,'epilogue'): + mail.append(['Epilogue',mailobj.data.epilogue]) + else: + mail.append(['Epilogue','']) + requested_file = selected_item + return OutlookMailDetail(agent=self.agent, pagemessage="", mail=mail, filename=requested_file) + elif selected_item.find("file") > 0: + #TODO implement file analyzing + return FileObjectDetail(agent=self.agent, pagemessage="", fileobj=fileobj, filename=requested_file) # rendering methods of Startpage - def render_getActiveUserMode(self,context,data): - return config.ui.web.usermode + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode - def render_getAgentVersion(self,context,data): + def render_getAgentVersion(self, context, data): return "0.1 alpha" - def render_footer_fragment(self,context,data): + def render_footer_fragment(self, context, data): return context.tag[FooterFragment(data)] - def render_navigation_fragment(self,context,data): + def render_navigation_fragment(self, context, data): return context.tag[NavigationFragment(data)] - def render_top_fragment(self,context,data): + def render_top_fragment(self, context, data): return context.tag[TopFragment(data)] - def render_header_fragment(self,context,data): + def render_header_fragment(self, context, data): return context.tag[HeaderFragment(data)] @@ -270,11 +377,11 @@ class JobOverView(rend.Page): """Builds page that lists all currently registered jobs. Instance variables: - currently none in use + agent -- agent object where config and tempdir attributes can be accessed Class methods: - At the moment all methods of this class except the __init__ although public, - are intended to be called directly or stand-alone. + At the moment all methods of this class except the __init__ + although public, are intended to be called directly or stand-alone. __init__ -- Store the initial settings retrieved from AgentHome. @@ -282,60 +389,81 @@ class JobOverView(rend.Page): docFactory = template('joblisting.html') - def __init__(self): + def __init__(self, agent=None): print "[PAGECLASS] JobOverView" + self.agent = agent # rendering methods of job overview - def data_displayViewForm(self,context,data): + def data_displayViewForm(self, context, data): return "Overview of all running Crawling jobs" - def render_getActiveUserMode(self,context,data): - return config.ui.web.usermode + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode - def render_fillJobTable(self,ctx,data): + def render_fillJobTable(self, ctx, data): """Build table which displays all registered jobs.""" #---- get the registered jobs from the jobfile ---- joblist = [] - iterator = 0 - config.load() - while iterator < len(config.crawl): - joblist.append(dict(config.crawl[iterator].items())) - iterator = iterator + 1 - for elem in joblist: - print elem - job_table = [tags.tr - [ - tags.td - [ - tags.a(href="ViewJobDetails?%s" %(job_details['jobid'])) + index = 0 + self.agent.config.load() + while index < len(self.agent.config.crawl): + joblist.append(dict(self.agent.config.crawl[index].items())) + index = index + 1 + job_table = [] + for job_details in joblist: + if job_details['jobid'].split(".")[0] == "outlook": + job_table.append(tags.tr [ - tags.b - [ - "[" + job_details['jobid'] +"]" - ] + tags.td + [ + tags.a(href="ViewJobDetails?%s" %(job_details['jobid'])) + [ + tags.b + [ + "[" + job_details['jobid'] +"]" + ] + ] + ], + tags.td[job_details['state']], + tags.td[job_details['interval']], + tags.td[job_details['filter_criteria']], + tags.td["Inbox: %s, Subfolders: %s" %(job_details['getinbox'], job_details['getsubfolder'])] ] - ], - tags.td[job_details['state']], - tags.td[job_details['interval']], - tags.td[job_details['filter_criteria']], - tags.td[job_details['folder']] - ] - for job_details in joblist - ] + ) + elif job_details['jobid'].split(".")[0] == "filesystem": + job_table.append(tags.tr + [ + tags.td + [ + tags.a(href="ViewJobDetails?%s" %(job_details['jobid'])) + [ + tags.b + [ + "[" + job_details['jobid'] +"]" + ] + ] + ], + tags.td[job_details['state']], + tags.td[job_details['interval']], + tags.td[job_details['filter_criteria']], + tags.td["directories"] + ] + ) + return job_table - def render_footer_fragment(self,context,data): + def render_footer_fragment(self, context, data): return context.tag[FooterFragment(data)] - def render_navigation_fragment(self,context,data): + def render_navigation_fragment(self, context, data): return context.tag[NavigationFragment(data)] - def render_top_fragment(self,context,data): + def render_top_fragment(self, context, data): return context.tag[TopFragment(data)] - def render_header_fragment(self,context,data): + def render_header_fragment(self, context, data): return context.tag[HeaderFragment(data)] @@ -345,6 +473,7 @@ class JobOverViewDetails(rend.Page): Instance variables: jobdetails -- list that contains all available job information + agent -- agent object where config and tempdir attributes can be accessed Class methods: At the moment all methods of this class except the __init__ although public, @@ -356,84 +485,59 @@ class JobOverViewDetails(rend.Page): docFactory = template('jobdetail.html') - def __init__(self, selectedJob=[]): + def __init__(self, agent=None, selected_jobindex=None): print "[PAGECLASS] JobOverViewDetails" - self.jobdetails = selectedJob + self.selected_index = selected_jobindex + self.agent = agent # rendering methods of job view details - def data_displayViewForm(self,context,data): + def data_displayViewForm(self, context, data): return "Detailed view of crawling job." - def render_getActiveUserMode(self,context,data): - return config.ui.web.usermode + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode - def render_displayJobDetails(self,ctx,data): + def render_displayJobDetails(self, ctx, data): """Build the table containing information about the selected job.""" print "*******************************************************" - print "[render_displayJobDetails] received form: ", str(self.jobdetails) - print "[render_displayJobDetails] usermode: ", config.ui.web.usermode + print "[render_displayJobDetails] received form: ", str(self.selected_index) + print "[render_displayJobDetails] usermode: ", self.agent.config.ui.web.usermode print "*******************************************************" - job_detail = [] - job_detail = [tags.tr - [ - tags.td["PID"], - tags.td[self.jobdetails[0]] - ], - tags.tr - [ - tags.td["State"], - tags.td[self.jobdetails[1]] - ], - tags.tr - [ - tags.td["Interval"], - tags.td[self.jobdetails[2]] - ], - tags.tr - [ - tags.td["Search Criteria"], - tags.td[self.jobdetails[3]] - ], - tags.tr - [ - tags.td["Job Scope"], - tags.td[self.jobdetails[4]] - ], - tags.tr - [ - tags.td["Filter"], - tags.td[self.jobdetails[5]] - ], - tags.tr - [ - tags.td["Follow Up Tasks"], - tags.td[self.jobdetails[6]] - ] - ] - return job_detail - def render_footer_fragment(self,context,data): + if self.selected_index != None: + job_detailtable = [tags.tr + [ + tags.td[tags.b[parameter[0]]], + tags.td[parameter[1]] + ] + for parameter in list(self.agent.config.crawl[int(self.selected_index)].items()) + ] + return job_detailtable + + def render_footer_fragment(self, context, data): return context.tag[FooterFragment(data)] - def render_navigation_fragment(self,context,data): + def render_navigation_fragment(self, context, data): return context.tag[NavigationFragment(data)] - def render_top_fragment(self,context,data): + def render_top_fragment(self, context, data): return context.tag[TopFragment(data)] - def render_header_fragment(self,context,data): + def render_header_fragment(self, context, data): return context.tag[HeaderFragment(data)] -class AgentOutlookMailView(rend.Page): +class RessourceView(rend.Page): """Builds page that displays an overview of all collected mails. Instance variables: page_message -- string that is displayed on the page - mail_collection -- collection containing all currently collected mails. + ressource_collection -- collection containing all currently collected ressources. + temp_dir -- actual directory whose ressources have to be displayed + agent -- the agent object which holds the config data and temp directory Class methods: At the moment all methods of this class except the __init__ although public, @@ -443,84 +547,253 @@ class AgentOutlookMailView(rend.Page): """ - docFactory = template('mail.html') + mail_collection = None + docFactory = template('ressourceview.html') - def __init__(self, mail_collection=[], pagemessage=""): - print "[PAGECLASS] AgentOutlookMailView" - self.mail_collection = mail_collection + def __init__(self, agent=None, ressource_collection=[], pagemessage="", tmpdir=""): + print "[PAGECLASS] RessourceView" + self.ressource_collection = ressource_collection self.page_message = pagemessage + self.temp_dir = tmpdir + self.agent = agent - # rendering methods of Collect Outlook Mails + # rendering methods of view collected ressources - def data_displayViewForm(self,context,data): - return "Detailed view of all collected Outlook Mails." + def data_displayViewForm(self, context, data): + return "View of all collected ressources." - def render_getActiveUserMode(self,context,data): - return config.ui.web.usermode + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode - def render_systemMessage(self,context,data): + def render_systemMessage(self, context, data): """Displays messages from the system to the user.""" message = tags.b[self.page_message] return message - def render_displayOutlookMails(self,ctx,data): - """Builds table containing all currently collected mails.""" - if self.mail_collection != []: - mail_table = [tags.tr - [ - tags.td - [ - tags.a(href="ViewMailDetails?%s" %(str(mailObject.get('SenderName')))) - [ - tags.b - [ - "[" + str(mailObject.get('SenderName')) +"]" - ] - ] - ], - tags.td[str(mailObject.get('CC'))], - tags.td[str(mailObject.get('Subject'))], - tags.td[str(mailObject.get('To'))], - tags.td['SourceFolder'] - ] - for mailObject in self.mail_collection - ] - return mail_table + def render_displayRessourceHeaders(self, context, data): + """Displays the column headings according to the ressource type""" + #TODO: switch between different objecttypes + if DEBUG: + print "====> creating html columns for Ressource table" + current_dir="" + if self.temp_dir == "": + current_dir = self.agent.tempdir else: - mailFiles = os.listdir(MAIL_DIR) - #TODO: read mail data from files - mail_table = [tags.tr - [ - tags.td - [ - tags.a(href="ViewMailDetails?%s" %(str(mail))) - [ - tags.b - [ - "[" + str(mail) +"]" - ] - ] - ], - tags.td[str(mail)], - tags.td[str(mail)], - tags.td[str(mail)], - tags.td[str(mail)] - ] - for mail in mailFiles - ] - return mail_table - + current_dir = self.temp_dir + if self.ressource_collection == []: + files_found = 0 + for elem in os.listdir(current_dir): + if os.path.isfile(os.path.join(current_dir,elem)): + files_found = 1 + break + if files_found == 0: + if current_dir.find("mail"): + if DEBUG: + print "====> just mail subdirs in current directory" + return [tags.tr + [ + tags.th["Directory Name"], + tags.th["Created"], + tags.th["Modified"], + tags.th["Mails inside"], + ] + ] + elif current_dir.find("file"): + if DEBUG: + print "====> just file subdirs in current directory" + return [tags.tr + [ + tags.th["Directory Name"], + tags.th["Created"], + tags.th["Modified"], + tags.th["Files inside"], + ] + ] + if files_found >= 1: + if current_dir.find("mail"): + if DEBUG: + print "====> current directory has mails" + return [tags.tr + [ + tags.th["From"], + tags.th["CC"], + tags.th["Subject"], + tags.th["Recipient"], + tags.th["Date"] + ] + ] + elif current_dir.find("file"): + if DEBUG: + print "====> current directory has files" + return [tags.tr + [ + tags.th["Filename"], + tags.th["Filesize"], + tags.th["Date created"], + tags.th["Date modified"], + tags.th["Filetype"] + ] + ] + elif isinstance(self.ressource_collection[0], FileResource): + if DEBUG: + print "====> instance is a FileResource" + return [tags.tr + [ + tags.th["Filename"], + tags.th["Filesize"], + tags.th["Date created"], + tags.th["Date modified"], + tags.th["Filetype"] + ] + ] + elif isinstance(self.ressource_collection[0], OutlookResource): + if DEBUG: + print "====> instance is a OutlookResource" + return [tags.tr + [ + tags.th["From"], + tags.th["CC"], + tags.th["Subject"], + tags.th["Recipient"], + tags.th["Date"] + ] + ] + # raise exception here? + return "could not find a matching object type" - def render_footer_fragment(self,context,data): + def render_displayRessources(self, ctx, data): + """Builds table containing all currently collected ressources.""" + if self.ressource_collection != []: + if DEBUG: + print "====> building ressource with ressource_collection" + index = 0 + ressource_table = [] + if isinstance(self.ressource_collection[0],OutlookResource): + for ressourceObject in self.ressource_collection: + ressource_table.append(tags.form(name="ressourceEntry%i"%(index), action="viewRessourceDetails?%i"%(index), method="POST")[ + tags.input(name="ressourceObjData", type="hidden", value="%s"%(self.temp_dir)), + tags.tr[ + tags.td[ + tags.a(href="javascript:document.getElementsByName('ressourceEntry%i')[0].submit()"%(index))[ + tags.b[ + "[" + str(ressourceObject.data.get('SenderName')) +"]" + ] + ] + ], + tags.td[ + str(ressourceObject.data.get('CC')) + ], + tags.td[ + str(ressourceObject.data.get('Subject')) + ], + tags.td[ + #TODO: proper handling of unicode characters + ressourceObject.data.get('To').encode('utf-8') + ], + tags.td['SourceFolder']]]) + index = index + 1 + return ressource_table + + if isinstance(self.ressource_collection[0],FileResource): + #TODO: implement building the table for file objects + return ressource_table + + else: + if DEBUG: + print "====> building ressource by analyzing submitted dir" + if self.temp_dir == "": + current_dir = self.agent.tempdir + else: + current_dir = self.temp_dir + ressource_files= os.listdir(current_dir) + ressource_table = [] + #check whether there are directories, files or both in the actual directory + for elem in ressource_files: + element = os.path.join(current_dir,elem) + if os.path.isdir(element): + files_in_subdir = len(os.listdir(element)) + ressource_table.append(tags.tr + [ + tags.td + [ + tags.a(href="viewRessourceDetails?%s" %(element)) + [ + tags.b + [ + "[" + str(elem) +"]" + ] + ] + ], + tags.td[time.asctime(time.localtime(os.path.getctime(element)))], + tags.td[time.asctime(time.localtime(os.path.getmtime(element)))], + tags.td[files_in_subdir] + ] + ) + + elif os.path.isfile(element): + if elem.find("file") > 0: + if DEBUG: + print "====> %s is of type crawl file" %(elem) + ressource_table.append(tags.tr + [ + tags.td + [ + tags.a(href="viewRessourceDetails?%s" %(element)) + [ + tags.b + [ + "[" + str(elem) +"]" + ] + ] + ], + tags.td[os.path.getsize(element)], + tags.td[time.asctime(time.localtime(os.path.getctime(element)))], + tags.td[time.asctime(time.localtime(os.path.getmtime(element)))], + tags.td[str(elem.split(".")[-1])] + ] + ) + elif elem.find("mail") > 0: + if DEBUG: + print "====> %s is of type mail" %(elem) + fp = open(element,"rb") + mail_parser = email.Parser.Parser() + mail = OutlookResource(mail_parser.parse(fp)) + fp.close() + mail_from = mail.data.get('From', "not available") + mail_cc = mail.data.get('CC', "not available") + mail_subject = mail.data.get('Subject', "not available") + mail_recipient = mail.data.get('TO', "not available") + mail_date = mail.data.get('SENT', "not available") + ressource_table.append(tags.tr + [ + tags.td + [ + tags.a(href="viewRessourceDetails?%s" %(element)) + [ + tags.b + [ + "[" + mail_from +"]" + ] + ] + ], + tags.td[mail_cc], + tags.td[mail_subject], + tags.td[mail_recipient], + tags.td[mail_date] + ] + ) + return ressource_table + + def render_footer_fragment(self, context, data): return context.tag[FooterFragment(data)] - def render_navigation_fragment(self,context,data): + def render_navigation_fragment(self, context, data): return context.tag[NavigationFragment(data)] - def render_top_fragment(self,context,data): + def render_top_fragment(self, context, data): return context.tag[TopFragment(data)] - def render_header_fragment(self,context,data): + def render_header_fragment(self, context, data): return context.tag[HeaderFragment(data)] @@ -541,13 +814,14 @@ class AgentOutlookMailCrawl(rend.Page): docFactory = template('mailcrawl.html') - def __init__(self, pagemessage=""): + def __init__(self, agent=None, pagemessage=""): print "[PAGECLASS] AgentOutlookMailCrawl" self.page_message = pagemessage + self.agent = agent - # rendering methods of Collect Outlook Mails + # rendering methods of add outlook crawl job - def data_displayViewForm(self,context,data): + def data_displayViewForm(self, context, data): return "Configure your MailCrawl Job in the form below.\ You then can either choose to start it as a\ single job, or as scheduled job (further input\ @@ -556,27 +830,150 @@ class AgentOutlookMailCrawl(rend.Page): mails received in Outlook after the recent Crawl\ Job has been run, will be collected." - def render_getActiveUserMode(self,context,data): - return config.ui.web.usermode + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode - def render_displayOutlookMails(self,ctx,data): + def render_displayOutlookMails(self, ctx, data): """Currently no implementation""" return "" - def render_systemMessage(self,context,data): + def render_systemMessage(self, context, data): """Displays messages from the system to the user.""" message = tags.b[self.page_message] return message - def render_footer_fragment(self,context,data): + def render_footer_fragment(self, context, data): return context.tag[FooterFragment(data)] - def render_navigation_fragment(self,context,data): + def render_navigation_fragment(self, context, data): return context.tag[NavigationFragment(data)] - def render_top_fragment(self,context,data): + def render_top_fragment(self, context, data): return context.tag[TopFragment(data)] - def render_header_fragment(self,context,data): + def render_header_fragment(self, context, data): + return context.tag[HeaderFragment(data)] + + +class AgentFilesystemCrawl(rend.Page): + + """Builds page where an Filesystem Crawler can be configured and run. + + Instance variables: + page_message -- string that is displayed on the page + + Class methods: + At the moment all methods of this class except the __init__ although public, + are intended to be called directly or stand-alone. + + __init__ -- store pagemessage in member variable + + """ + + docFactory = template('filecrawl.html') + + def __init__(self, agent=None, pagemessage=""): + print "[PAGECLASS] AgentFilesystemCrawl" + self.page_message = pagemessage + self.agent = agent + + # rendering methods of ad filesystem crawl job + + def data_displayViewForm(self, context, data): + return "Configure your FileCrawl Job in the form below.\ + You then can either choose to start it as a\ + single job, or as scheduled job (further input\ + for scheduler necessary). For each new collect\ + procedure a filter will be applied, so that just\ + new objects will be collected." + + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode + + def render_displayFiles(self, ctx, data): + """Currently no implementation""" + return "" + + def render_systemMessage(self, context, data): + """Displays messages from the system to the user.""" + message = tags.b[self.page_message] + return message + + def render_footer_fragment(self, context, data): + return context.tag[FooterFragment(data)] + + def render_navigation_fragment(self, context, data): + return context.tag[NavigationFragment(data)] + + def render_top_fragment(self, context, data): + return context.tag[TopFragment(data)] + + def render_header_fragment(self, context, data): + return context.tag[HeaderFragment(data)] + + +class OutlookMailDetail(rend.Page): + + """Builds page that displays the selected mail in detail. + + Instance variables: + page_message -- string that is displayed on the page + mail_elements -- received mailobject as a list. + agent -- agent object that holds the config data + mail_file -- filename of the selected mail to view + + Class methods: + At the moment all methods of this class except the __init__ although public, + are intended to be called directly or stand-alone. + + __init__ -- Store the mail collection object and the pagemessage + + """ + + docFactory = template('mail_detailed.html') + + def __init__(self, agent=None, pagemessage="", mail=[], filename=""): + print "[PAGECLASS] OutlookMailDetail" + self.mail_elements = mail + self.page_message = pagemessage + self.agent = agent + self.mail_file = filename + + # rendering methods of Collect Outlook Mails + + def data_displayViewForm(self, context, data): + return (tags.b["Filename of the selected mail: "], tags.i[self.mail_file]) + + def render_getActiveUserMode(self, context, data): + return self.agent.config.ui.web.usermode + + def render_systemMessage(self, context, data): + """Displays messages from the system to the user.""" + message = tags.b[self.page_message] + return message + + def render_displayOutlookMail(self, ctx, data): + """Builds the layout for the mail.""" + + if self.mail_elements != []: + mail_view = [] + for elem in self.mail_elements: + mail_view.append(tags.tr[ + tags.td[elem[0]], + tags.td[elem[1]] + ] + ) + return mail_view + + def render_footer_fragment(self, context, data): + return context.tag[FooterFragment(data)] + + def render_navigation_fragment(self, context, data): + return context.tag[NavigationFragment(data)] + + def render_top_fragment(self, context, data): + return context.tag[TopFragment(data)] + + def render_header_fragment(self, context, data): return context.tag[HeaderFragment(data)]