cco/loops
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

make runnable on Linux (ignore win imports conditionally); avoid hard-coded path and port setting

Browse source 
git-svn-id: svn://svn.cy55.de/Zope3/src/loops/trunk@2105 fd906abe-77d9-0310-91a1-e0d9ade77398
This commit is contained in:
helmutm 2007-10-09 07:19:17 +00:00
parent 999c25bf0b
commit eee4e198c0
3 changed files with 96 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
agent/loops.tac
View file

@ -1,15 +1,22 @@
"""
Start with ``twistd -noy loops/agent/loops.tac``.
$Id$
"""
from twisted.application import internet, service from twisted.application import internet, service
from nevow import appserver from nevow import appserver
from loops.agent.core import Agent from loops.agent.core import Agent
from loops.agent.ui.web import AgentHome from loops.agent.ui.web import AgentHome
from loops.agent.config import conf
agent = Agent()
conf = agent.config
port = conf.ui.web.port or 10095 port = conf.ui.web.port or 10095
application = service.Application('LoopsAgent') application = service.Application('LoopsAgent')
site = appserver.NevowSite(resource=AgentHome(Agent())) site = appserver.NevowSite(resource=AgentHome(agent))
webServer = internet.TCPServer(port, site) webServer = internet.TCPServer(port, site)
webServer.setServiceParent(application) webServer.setServiceParent(application)

12
agent/ui/templates/navigation.html
View file

@ -3,36 +3,36 @@
<div class="body"> <div class="body">
<div class="content even menu-1"> <div class="content even menu-1">
<a href="http://localhost:8080" class="">Startpage</a> <a href="/" class="">Startpage</a>
</div> </div>
<div class="content odd menu-2">Agent configuration</div> <div class="content odd menu-2">Agent configuration</div>
<div class="content odd menu-3"> <div class="content odd menu-3">
<a href="http://localhost:8080/joboverview" class=""> <a href="/joboverview" class="">
job overview job overview
</a> </a>
</div> </div>
<div class="content odd menu-3"> <div class="content odd menu-3">
<a href="http://localhost:8080/collectOutlookMails" class=""> <a href="/collectOutlookMails" class="">
add outlook crawl job add outlook crawl job
</a> </a>
</div> </div>
<div class="content odd menu-3"> <div class="content odd menu-3">
<a href="http://localhost:8080/collectFilesystem" class=""> <a href="/collectFilesystem" class="">
add filesystem crawl job</a> add filesystem crawl job</a>
</div> </div>
<div class="content odd menu-3"> <div class="content odd menu-3">
<a href="http://localhost:8080/viewRessources" class=""> <a href="/viewRessources" class="">
view collected ressources view collected ressources
</a> </a>
</div> </div>
<div class="content odd menu-3"> <div class="content odd menu-3">
<a href="http://localhost:8080/loggingoptions" class="" title=""> <a href="/loggingoptions" class="" title="">
logging options logging options
</a> </a>
</div> </div>

150
agent/ui/web.py
View file

@ -5,6 +5,11 @@
# version: 0.1 # version: 0.1
#------------------------------------------------------ #------------------------------------------------------
"""
$Id$
"""
import sys import sys
import os import os
import tempfile import tempfile
@ -12,15 +17,20 @@ import cPickle
import traceback import traceback
import time import time
import email import email
import _winreg as winreg try:
import _winreg as winreg
USE_WINDOWS = True
except ImportError:
USE_WINDOWS = False
from nevow import loaders, rend, static, url, inevow, tags from nevow import loaders, rend, static, url, inevow, tags
from nevow.inevow import IRequest from nevow.inevow import IRequest
from twisted.internet import defer from twisted.internet import defer
from loops.agent.crawl import outlook
from loops.agent import core from loops.agent import core
from loops.agent.crawl.outlook import OutlookResource if USE_WINDOWS:
from loops.agent.crawl import outlook
from loops.agent.crawl.outlook import OutlookResource
from loops.agent.crawl.filesystem import FileResource from loops.agent.crawl.filesystem import FileResource
# ---- global definitions and settings --------------------------------- # ---- global definitions and settings ---------------------------------
@ -30,7 +40,7 @@ PICKLE_MAILS = 0
DEBUG = 1 DEBUG = 1
resourcesDirectory = 'resources' resourcesDirectory = 'resources'
templatesDirectory = 'templates' templatesDirectory = 'templates'
# some constatnts # some constatnts
_REG_KEY_CONST_ = "SOFTWARE\\Classes\\mailto\\shell\\open\\command" _REG_KEY_CONST_ = "SOFTWARE\\Classes\\mailto\\shell\\open\\command"
_OUTLOOK_2007_ = "Office12" _OUTLOOK_2007_ = "Office12"
@ -41,7 +51,8 @@ _OUTLOOK_EXPRESS_ = "Office10"
def template(fn): def template(fn):
return loaders.xmlfile(os.path.join(templatesDirectory, fn)) return loaders.xmlfile(os.path.join(
os.path.dirname(__file__), templatesDirectory, fn))
def getConfigIndex(agentobj, index_name, default_value, fn=None): def getConfigIndex(agentobj, index_name, default_value, fn=None):
@ -55,7 +66,7 @@ def getOutlookVersion():
checks what outlook version we have to handle checks what outlook version we have to handle
Returns the standard email application on this machine. Returns the standard email application on this machine.
Therefor we have to read out the registry key Therefor we have to read out the registry key
"HKEY_LOCAL_MACHINE\SOFTWARE\Classes\mailto\shell\open\command" "HKEY_LOCAL_MACHINE\SOFTWARE\Classes\mailto\shell\open\command"
@ -68,44 +79,44 @@ def getOutlookVersion():
Outlook versions installed and modify, Outlook versions installed and modify,
if needed the defined constants and return if needed the defined constants and return
values of the getOutlookVersion function values of the getOutlookVersion function
""" """
# open registry key # open registry key
try: try:
key = winreg.OpenKey( key = winreg.OpenKey(
winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_LOCAL_MACHINE,
_REG_KEY_CONST_ ) _REG_KEY_CONST_ )
except WindowsError: except WindowsError:
return None return None
try: try:
try: try:
# read out current standard outlook version # read out current standard outlook version
version = winreg.QueryValueEx(key, "")[0] version = winreg.QueryValueEx(key, "")[0]
# check what outlook we have # check what outlook we have
if version: if version:
if _OUTLOOK_2007_ in version: if _OUTLOOK_2007_ in version:
version = _OUTLOOK_2007_ version = _OUTLOOK_2007_
elif _OUTLOOK_2003_ in version: elif _OUTLOOK_2003_ in version:
version = _OUTLOOK_2003_ version = _OUTLOOK_2003_
elif _OUTLOOK_EXPRESS_ in version: elif _OUTLOOK_EXPRESS_ in version:
version = _OUTLOOK_EXPRESS_ version = _OUTLOOK_EXPRESS_
except WindowsError: except WindowsError:
version = "" version = ""
finally: finally:
# close key # close key
winreg.CloseKey(key) winreg.CloseKey(key)
#print '--> getOutlookVersion(): Outlook version found, version is: ', version #print '--> getOutlookVersion(): Outlook version found, version is: ', version
# return key value # return key value
return version return version
# AgentHome # AgentHome
# root page of the Agent UI # root page of the Agent UI
class AgentHome(rend.Page): class AgentHome(rend.Page):
@ -123,10 +134,11 @@ class AgentHome(rend.Page):
__init__ -- Load the initial start-up settings from config __init__ -- Load the initial start-up settings from config
""" """
child_resources = static.File(resourcesDirectory) child_resources = static.File(os.path.join(
os.path.dirname(__file__),resourcesDirectory))
docFactory = template('agent.html') docFactory = template('agent.html')
def __init__(self, agent=None, first_start=1): def __init__(self, agent=None, first_start=1):
""" Initialize the AgentHome object. """ Initialize the AgentHome object.
@ -145,14 +157,14 @@ class AgentHome(rend.Page):
print "[AgentHome] setting self.usermode: ", self.agent.config.ui.web.usermode print "[AgentHome] setting self.usermode: ", self.agent.config.ui.web.usermode
else: else:
self.agent = agent self.agent = agent
""" """
def locateChild(self, ctx, segments): def locateChild(self, ctx, segments):
return self, () return self, ()
""" """
# calls to child pages # calls to child pages
def child_joboverview(self, context): def child_joboverview(self, context):
""" User requested page from menue: 'job overview' """ """ User requested page from menue: 'job overview' """
return JobOverView(self.agent) return JobOverView(self.agent)
@ -204,7 +216,7 @@ class AgentHome(rend.Page):
""" """
selected_job = ((IRequest(context).uri).split("?"))[1] selected_job = ((IRequest(context).uri).split("?"))[1]
crawl_index = selected_job.split(".")[1] crawl_index = selected_job.split(".")[1]
return JobOverViewDetails(self.agent, crawl_index) return JobOverViewDetails(self.agent, crawl_index)
# "add outlook crawl job" methods (class AgentOutlookMailCrawl) # "add outlook crawl job" methods (class AgentOutlookMailCrawl)
@ -216,12 +228,12 @@ class AgentHome(rend.Page):
crawlSubfolder = False crawlSubfolder = False
form = IRequest(context).args form = IRequest(context).args
if form != {}: if form != {}:
index = getConfigIndex(self.agent, 'type', 'unused') index = getConfigIndex(self.agent, 'type', 'unused')
#save job configuration #save job configuration
if form['mailCrawlInterval'][0] == "oneTime": if form['mailCrawlInterval'][0] == "oneTime":
conf.crawl[index].state = 'completed' conf.crawl[index].state = 'completed'
else: else:
conf.crawl[index].state = 'active' conf.crawl[index].state = 'active'
conf.crawl[index].jobid = 'outlook.%i' %(index) conf.crawl[index].jobid = 'outlook.%i' %(index)
conf.crawl[index].type = 'OutlookMail' conf.crawl[index].type = 'OutlookMail'
if form.has_key('inbox'): if form.has_key('inbox'):
@ -283,17 +295,17 @@ class AgentHome(rend.Page):
return deferred return deferred
else: else:
#TODO implement forwarding to next form (scheduler) #TODO implement forwarding to next form (scheduler)
return AgentOutlookMailCrawl(self.agent, "Scheduled Mail Crawling not implemented yet.") return AgentOutlookMailCrawl(self.agent, "Scheduled Mail Crawling not implemented yet.")
else: else:
return AgentOutlookMailCrawl(self.agent, "An error occurred: form data has been empty.") return AgentOutlookMailCrawl(self.agent, "An error occurred: form data has been empty.")
return AgentOutlookMailCrawl(self.agent, "Crawl Job settings have been saved.") return AgentOutlookMailCrawl(self.agent, "Crawl Job settings have been saved.")
def defMailCrawl(self, mail_collection, index, params, context): def defMailCrawl(self, mail_collection, index, params, context):
"""Save and Forward the mail collection to a page view.""" """Save and Forward the mail collection to a page view."""
if DEBUG: if DEBUG:
print "====> seting and saving mails to disk" print "====> seting and saving mails to disk"
print "====> agent base dir: ",self.agent.tempdir print "====> agent base dir: ",self.agent.tempdir
tmpdir = tempfile.mkdtemp(prefix='mailJob_%i_'%(index), dir=self.agent.tempdir) tmpdir = tempfile.mkdtemp(prefix='mailJob_%i_'%(index), dir=self.agent.tempdir)
if DEBUG: if DEBUG:
print "====> outlook job dir: ", tmpdir print "====> outlook job dir: ", tmpdir
@ -312,7 +324,7 @@ class AgentHome(rend.Page):
# Idea could be a object that stores all current paths and # Idea could be a object that stores all current paths and
# their crawlIDs # their crawlIDs
os.write(tmpfile[0], cPickle.dumps(elem)) os.write(tmpfile[0], cPickle.dumps(elem))
os.close(tmpfile[0]) os.close(tmpfile[0])
filenum = filenum + 1 filenum = filenum + 1
return RessourceView(self.agent, mail_collection, "The collected mails have been saved in %s"%(tmpdir), tmpdir) return RessourceView(self.agent, mail_collection, "The collected mails have been saved in %s"%(tmpdir), tmpdir)
@ -331,7 +343,7 @@ class AgentHome(rend.Page):
Returns page object which displays the available information. Returns page object which displays the available information.
""" """
selected_item = ((IRequest(context).uri).split("?"))[1] selected_item = ((IRequest(context).uri).split("?"))[1]
form = IRequest(context).args form = IRequest(context).args
if form != {}: if form != {}:
requested_file="" requested_file=""
@ -346,7 +358,7 @@ class AgentHome(rend.Page):
if elem.startswith(("%i_mail_"%(int(selected_item))), 0): if elem.startswith(("%i_mail_"%(int(selected_item))), 0):
requested_file = elem requested_file = elem
requested_file = os.path.join(form['ressourceObjData'][0], requested_file) requested_file = os.path.join(form['ressourceObjData'][0], requested_file)
if requested_file.find("mail") > 0: if requested_file.find("mail") > 0:
fp = open(requested_file, "rb") fp = open(requested_file, "rb")
mail_parser = email.Parser.Parser() mail_parser = email.Parser.Parser()
mailobj = OutlookResource(mail_parser.parse(fp)) mailobj = OutlookResource(mail_parser.parse(fp))
@ -359,7 +371,7 @@ class AgentHome(rend.Page):
mail.append(elem) mail.append(elem)
if hasattr(mailobj.data, 'preamble'): if hasattr(mailobj.data, 'preamble'):
if DEBUG: if DEBUG:
print "====> copying preamble contents" print "====> copying preamble contents"
mail.append(['Preamble', mailobj.data.preamble]) mail.append(['Preamble', mailobj.data.preamble])
else: else:
mail.append(['Preamble','']) mail.append(['Preamble',''])
@ -372,7 +384,7 @@ class AgentHome(rend.Page):
return OutlookMailDetail(agent=self.agent, pagemessage="", mail=mail, filename=requested_file) return OutlookMailDetail(agent=self.agent, pagemessage="", mail=mail, filename=requested_file)
elif requested_file.find("file") > 0: elif requested_file.find("file") > 0:
#TODO implement file analyzing #TODO implement file analyzing
return FileObjectDetail(agent=self.agent, pagemessage="", fileobj=fileobj, filename=requested_file) return FileObjectDetail(agent=self.agent, pagemessage="", fileobj=fileobj, filename=requested_file)
if os.path.isdir(selected_item): if os.path.isdir(selected_item):
# selected item is a folder -> change to that folder # selected item is a folder -> change to that folder
if DEBUG: if DEBUG:
@ -421,7 +433,7 @@ class AgentHome(rend.Page):
def render_header_fragment(self, context, data): def render_header_fragment(self, context, data):
return context.tag[HeaderFragment(data)] return context.tag[HeaderFragment(data)]
class FooterFragment(rend.Fragment): class FooterFragment(rend.Fragment):
docFactory = template('footer.html') docFactory = template('footer.html')
@ -437,7 +449,7 @@ class TopFragment(rend.Fragment):
class HeaderFragment(rend.Fragment): class HeaderFragment(rend.Fragment):
docFactory = template('header.html') docFactory = template('header.html')
# subpages of AgentHome # subpages of AgentHome
class JobOverView(rend.Page): class JobOverView(rend.Page):
@ -445,7 +457,7 @@ class JobOverView(rend.Page):
"""Builds page that lists all currently registered jobs. """Builds page that lists all currently registered jobs.
Instance variables: Instance variables:
agent -- agent object where config and tempdir attributes can be accessed agent -- agent object where config and tempdir attributes can be accessed
Class methods: Class methods:
At the moment all methods of this class except the __init__ At the moment all methods of this class except the __init__
@ -454,7 +466,7 @@ class JobOverView(rend.Page):
__init__ -- Store the initial settings retrieved from AgentHome. __init__ -- Store the initial settings retrieved from AgentHome.
""" """
docFactory = template('joblisting.html') docFactory = template('joblisting.html')
def __init__(self, agent=None): def __init__(self, agent=None):
@ -462,7 +474,7 @@ class JobOverView(rend.Page):
self.agent = agent self.agent = agent
# rendering methods of job overview # rendering methods of job overview
def data_displayViewForm(self, context, data): def data_displayViewForm(self, context, data):
return "Overview of all running Crawling jobs" return "Overview of all running Crawling jobs"
@ -536,7 +548,7 @@ class JobOverView(rend.Page):
class JobOverViewDetails(rend.Page): class JobOverViewDetails(rend.Page):
"""Builds page that displays detailed information about a selected job. """Builds page that displays detailed information about a selected job.
Instance variables: Instance variables:
@ -574,7 +586,7 @@ class JobOverViewDetails(rend.Page):
print "[render_displayJobDetails] usermode: ", self.agent.config.ui.web.usermode print "[render_displayJobDetails] usermode: ", self.agent.config.ui.web.usermode
print "*******************************************************" print "*******************************************************"
if self.selected_index != None: if self.selected_index != None:
job_detailtable = [tags.tr job_detailtable = [tags.tr
[ [
tags.td[tags.b[parameter[0]]], tags.td[tags.b[parameter[0]]],
@ -598,7 +610,7 @@ class JobOverViewDetails(rend.Page):
class RessourceView(rend.Page): class RessourceView(rend.Page):
"""Builds page that displays an overview of all collected mails. """Builds page that displays an overview of all collected mails.
Instance variables: Instance variables:
@ -701,7 +713,7 @@ class RessourceView(rend.Page):
tags.th["Date modified"], tags.th["Date modified"],
tags.th["Filetype"] tags.th["Filetype"]
] ]
] ]
elif isinstance(self.ressource_collection[0], FileResource): elif isinstance(self.ressource_collection[0], FileResource):
if DEBUG: if DEBUG:
print "====> instance is a FileResource" print "====> instance is a FileResource"
@ -713,7 +725,7 @@ class RessourceView(rend.Page):
tags.th["Date modified"], tags.th["Date modified"],
tags.th["Filetype"] tags.th["Filetype"]
] ]
] ]
elif isinstance(self.ressource_collection[0], OutlookResource): elif isinstance(self.ressource_collection[0], OutlookResource):
if DEBUG: if DEBUG:
print "====> instance is a OutlookResource" print "====> instance is a OutlookResource"
@ -725,7 +737,7 @@ class RessourceView(rend.Page):
tags.th["Recipient"], tags.th["Recipient"],
tags.th["Date"] tags.th["Date"]
] ]
] ]
# raise exception here? # raise exception here?
return "could not find a matching object type" return "could not find a matching object type"
@ -736,7 +748,7 @@ class RessourceView(rend.Page):
print "====> building ressource with ressource_collection" print "====> building ressource with ressource_collection"
index = 0 index = 0
ressource_table = [] ressource_table = []
if isinstance(self.ressource_collection[0],OutlookResource): if isinstance(self.ressource_collection[0],OutlookResource):
for ressourceObject in self.ressource_collection: for ressourceObject in self.ressource_collection:
ressource_table.append(tags.form(name="ressourceEntry%i"%(index), action="viewRessourceDetails?%i"%(index), method="POST")[ ressource_table.append(tags.form(name="ressourceEntry%i"%(index), action="viewRessourceDetails?%i"%(index), method="POST")[
tags.input(name="ressourceObjData", type="hidden", value="%s"%(self.temp_dir)), tags.input(name="ressourceObjData", type="hidden", value="%s"%(self.temp_dir)),
@ -765,7 +777,7 @@ class RessourceView(rend.Page):
if isinstance(self.ressource_collection[0],FileResource): if isinstance(self.ressource_collection[0],FileResource):
#TODO: implement building the table for file objects #TODO: implement building the table for file objects
return ressource_table return ressource_table
else: else:
if DEBUG: if DEBUG:
print "====> building ressource by analyzing submitted dir" print "====> building ressource by analyzing submitted dir"
@ -797,7 +809,7 @@ class RessourceView(rend.Page):
tags.td[files_in_subdir] tags.td[files_in_subdir]
] ]
) )
elif os.path.isfile(element): elif os.path.isfile(element):
if elem.find("file") > 0: if elem.find("file") > 0:
if DEBUG: if DEBUG:
@ -866,7 +878,7 @@ class RessourceView(rend.Page):
class AgentOutlookMailCrawl(rend.Page): class AgentOutlookMailCrawl(rend.Page):
"""Builds page where an Outlook Mail Crawler can be configured and run. """Builds page where an Outlook Mail Crawler can be configured and run.
Instance variables: Instance variables:
@ -924,7 +936,7 @@ class AgentOutlookMailCrawl(rend.Page):
class AgentFilesystemCrawl(rend.Page): class AgentFilesystemCrawl(rend.Page):
"""Builds page where an Filesystem Crawler can be configured and run. """Builds page where an Filesystem Crawler can be configured and run.
Instance variables: Instance variables:
@ -981,7 +993,7 @@ class AgentFilesystemCrawl(rend.Page):
class OutlookMailDetail(rend.Page): class OutlookMailDetail(rend.Page):
"""Builds page that displays the selected mail in detail. """Builds page that displays the selected mail in detail.
Instance variables: Instance variables:
@ -997,7 +1009,7 @@ class OutlookMailDetail(rend.Page):
__init__ -- Store the mail collection object and the pagemessage __init__ -- Store the mail collection object and the pagemessage
""" """
docFactory = template('mail_detailed.html') docFactory = template('mail_detailed.html')
def __init__(self, agent=None, pagemessage="", mail=[], filename=""): def __init__(self, agent=None, pagemessage="", mail=[], filename=""):
@ -1032,7 +1044,7 @@ class OutlookMailDetail(rend.Page):
] ]
) )
return mail_view return mail_view
def render_footer_fragment(self, context, data): def render_footer_fragment(self, context, data):
return context.tag[FooterFragment(data)] return context.tag[FooterFragment(data)]
@ -1044,4 +1056,4 @@ class OutlookMailDetail(rend.Page):
def render_header_fragment(self, context, data): def render_header_fragment(self, context, data):
return context.tag[HeaderFragment(data)] return context.tag[HeaderFragment(data)]